a4ad8935f2f2902a54b45554093e1265b284d790
[WebKit-https.git] / Source / WebCore / platform / network / curl / ResourceHandleCurl.cpp
1 /*
2  * Copyright (C) 2004, 2006 Apple Inc.  All rights reserved.
3  * Copyright (C) 2005, 2006 Michael Emmel mike.emmel@gmail.com
4  * Copyright (C) 2017 Sony Interactive Entertainment Inc.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
17  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
20  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
24  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28
29 #include "config.h"
30 #include "ResourceHandle.h"
31
32 #if USE(CURL)
33
34 #include "CachedResourceLoader.h"
35 #include "CredentialStorage.h"
36 #include "CurlCacheManager.h"
37 #include "CurlContext.h"
38 #include "CurlJobManager.h"
39 #include "FileSystem.h"
40 #include "Logging.h"
41 #include "MIMETypeRegistry.h"
42 #include "NetworkingContext.h"
43 #include "ResourceHandleInternal.h"
44 #include "SSLHandle.h"
45 #include "SynchronousLoaderClient.h"
46 #include <wtf/text/Base64.h>
47
48 namespace WebCore {
49
50 ResourceHandleInternal::~ResourceHandleInternal()
51 {
52 }
53
54 ResourceHandle::~ResourceHandle()
55 {
56 }
57
58 bool ResourceHandle::start()
59 {
60     ASSERT(isMainThread());
61
62     // The frame could be null if the ResourceHandle is not associated to any
63     // Frame, e.g. if we are downloading a file.
64     // If the frame is not null but the page is null this must be an attempted
65     // load from an unload handler, so let's just block it.
66     // If both the frame and the page are not null the context is valid.
67     if (d->m_context && !d->m_context->isValid())
68         return false;
69
70     d->initialize();
71
72     d->m_job = CurlJobManager::singleton().add(d->m_curlHandle, [this, protectedThis = makeRef(*this)](CurlJobResult result) {
73         ASSERT(isMainThread());
74
75         switch (result) {
76         case CurlJobResult::Done:
77             d->didFinish();
78             break;
79
80         case CurlJobResult::Error:
81             d->didFail();
82             break;
83
84         case CurlJobResult::Cancelled:
85             break;
86         }
87     });
88     ASSERT(d->m_job);
89
90     return true;
91 }
92
93 void ResourceHandle::cancel()
94 {
95     d->m_cancelled = true;
96     CurlJobManager::singleton().cancel(d->m_job);
97 }
98
99 void ResourceHandleInternal::initialize()
100 {
101     CurlContext& context = CurlContext::singleton();
102
103     URL url = m_firstRequest.url();
104
105     // Remove any fragment part, otherwise curl will send it as part of the request.
106     url.removeFragmentIdentifier();
107
108     String urlString = url.string();
109
110     if (url.isLocalFile()) {
111         // Remove any query part sent to a local file.
112         if (!url.query().isEmpty()) {
113             // By setting the query to a null string it'll be removed.
114             url.setQuery(String());
115             urlString = url.string();
116         }
117         // Determine the MIME type based on the path.
118         m_response.setMimeType(MIMETypeRegistry::getMIMETypeForPath(url));
119     }
120
121     if (m_defersLoading) {
122         CURLcode error = m_curlHandle.pause(CURLPAUSE_ALL);
123         // If we did not pause the handle, we would ASSERT in the
124         // header callback. So just assert here.
125         ASSERT_UNUSED(error, error == CURLE_OK);
126     }
127
128 #ifndef NDEBUG
129     m_curlHandle.enableVerboseIfUsed();
130     m_curlHandle.enableStdErrIfUsed();
131 #endif
132
133     m_curlHandle.setSslVerifyPeer(CurlHandle::VerifyPeerEnable);
134     m_curlHandle.setSslVerifyHost(CurlHandle::VerifyHostStrictNameCheck);
135     m_curlHandle.setPrivateData(this);
136     m_curlHandle.setWriteCallbackFunction(writeCallback, this);
137     m_curlHandle.setHeaderCallbackFunction(headerCallback, this);
138     m_curlHandle.enableAutoReferer();
139     m_curlHandle.enableFollowLocation();
140     m_curlHandle.enableHttpAuthentication(CURLAUTH_ANY);
141     m_curlHandle.enableShareHandle();
142     m_curlHandle.enableTimeout();
143     m_curlHandle.enableAllowedProtocols();
144     setSSLClientCertificate(m_handle);
145
146     if (CurlContext::singleton().shouldIgnoreSSLErrors())
147         m_curlHandle.setSslVerifyPeer(CurlHandle::VerifyPeerDisable);
148     else
149         setSSLVerifyOptions(m_handle);
150
151     m_curlHandle.enableCAInfoIfExists();
152
153     m_curlHandle.enableAcceptEncoding();
154     m_curlHandle.setUrl(urlString);
155     m_curlHandle.enableCookieJarIfExists();
156
157     if (m_firstRequest.httpHeaderFields().size()) {
158         auto customHeaders = m_firstRequest.httpHeaderFields();
159         auto& cache = CurlCacheManager::getInstance();
160
161         bool hasCacheHeaders = customHeaders.contains(HTTPHeaderName::IfModifiedSince) || customHeaders.contains(HTTPHeaderName::IfNoneMatch);
162         if (!hasCacheHeaders && cache.isCached(url)) {
163             cache.addCacheEntryClient(url, m_handle);
164
165             // append additional cache information
166             for (auto entry : cache.requestHeaders(url))
167                 customHeaders.set(entry.key, entry.value);
168
169             m_addedCacheValidationHeaders = true;
170         }
171
172         m_curlHandle.appendRequestHeaders(customHeaders);
173     }
174
175     String method = m_firstRequest.httpMethod();
176     if ("GET" == method)
177         m_curlHandle.enableHttpGetRequest();
178     else if ("POST" == method)
179         setupPOST();
180     else if ("PUT" == method)
181         setupPUT();
182     else if ("HEAD" == method)
183         m_curlHandle.enableHttpHeadRequest();
184     else {
185         m_curlHandle.setHttpCustomRequest(method);
186         setupPUT();
187     }
188
189     m_curlHandle.enableRequestHeaders();
190
191     applyAuthentication();
192
193     m_curlHandle.enableProxyIfExists();
194 }
195
196 void ResourceHandleInternal::applyAuthentication()
197 {
198     // m_user/m_pass are credentials given manually, for instance, by the arguments passed to XMLHttpRequest.open().
199     String partition = m_firstRequest.cachePartition();
200
201     if (m_handle->shouldUseCredentialStorage()) {
202         if (m_user.isEmpty() && m_pass.isEmpty()) {
203             // <rdar://problem/7174050> - For URLs that match the paths of those previously challenged for HTTP Basic authentication, 
204             // try and reuse the credential preemptively, as allowed by RFC 2617.
205             m_initialCredential = CredentialStorage::defaultCredentialStorage().get(partition, m_firstRequest.url());
206         } else {
207             // If there is already a protection space known for the URL, update stored credentials
208             // before sending a request. This makes it possible to implement logout by sending an
209             // XMLHttpRequest with known incorrect credentials, and aborting it immediately (so that
210             // an authentication dialog doesn't pop up).
211             CredentialStorage::defaultCredentialStorage().set(partition, Credential(m_user, m_pass, CredentialPersistenceNone), m_firstRequest.url());
212         }
213     }
214
215     String user = m_user;
216     String password = m_pass;
217
218     if (!m_initialCredential.isEmpty()) {
219         user = m_initialCredential.user();
220         password = m_initialCredential.password();
221         m_curlHandle.enableHttpAuthentication(CURLAUTH_BASIC);
222     }
223
224     // It seems we need to set CURLOPT_USERPWD even if username and password is empty.
225     // Otherwise cURL will not automatically continue with a new request after a 401 response.
226
227     // curl CURLOPT_USERPWD expects username:password
228     m_curlHandle.setHttpAuthUserPass(user, password);
229 }
230
231 static inline size_t getFormElementsCount(ResourceHandle* job)
232 {
233     RefPtr<FormData> formData = job->firstRequest().httpBody();
234
235     if (!formData)
236         return 0;
237
238     // Resolve the blob elements so the formData can correctly report it's size.
239     formData = formData->resolveBlobReferences();
240     size_t size = formData->elements().size();
241     job->firstRequest().setHTTPBody(WTFMove(formData));
242
243     return size;
244 }
245
246 void ResourceHandleInternal::setupPUT()
247 {
248     m_curlHandle.enableHttpPutRequest();
249
250     // Disable the Expect: 100 continue header
251     m_curlHandle.appendRequestHeader("Expect:");
252
253     size_t numElements = getFormElementsCount(m_handle);
254     if (!numElements)
255         return;
256
257     setupFormData(false);
258 }
259
260 void ResourceHandleInternal::setupPOST()
261 {
262     m_curlHandle.enableHttpPostRequest();
263
264     size_t numElements = getFormElementsCount(m_handle);
265     if (!numElements)
266         return;
267
268     // Do not stream for simple POST data
269     if (numElements == 1) {
270         m_firstRequest.httpBody()->flatten(m_postBytes);
271         if (m_postBytes.size())
272             m_curlHandle.setPostFields(m_postBytes.data(), m_postBytes.size());
273         return;
274     }
275
276     setupFormData(true);
277 }
278
279 void ResourceHandleInternal::setupFormData(bool isPostRequest)
280 {
281     Vector<FormDataElement> elements = m_firstRequest.httpBody()->elements();
282     size_t numElements = elements.size();
283
284     static const long long maxCurlOffT = m_curlHandle.maxCurlOffT();
285
286     // Obtain the total size of the form data
287     curl_off_t size = 0;
288     bool chunkedTransfer = false;
289     for (size_t i = 0; i < numElements; i++) {
290         FormDataElement element = elements[i];
291         if (element.m_type == FormDataElement::Type::EncodedFile) {
292             long long fileSizeResult;
293             if (getFileSize(element.m_filename, fileSizeResult)) {
294                 if (fileSizeResult > maxCurlOffT) {
295                     // File size is too big for specifying it to cURL
296                     chunkedTransfer = true;
297                     break;
298                 }
299                 size += fileSizeResult;
300             } else {
301                 chunkedTransfer = true;
302                 break;
303             }
304         } else
305             size += elements[i].m_data.size();
306     }
307
308     // cURL guesses that we want chunked encoding as long as we specify the header
309     if (chunkedTransfer)
310         m_curlHandle.appendRequestHeader("Transfer-Encoding: chunked");
311     else {
312         if (isPostRequest)
313             m_curlHandle.setPostFieldLarge(size);
314         else
315             m_curlHandle.setInFileSizeLarge(size);
316     }
317
318     m_curlHandle.setReadCallbackFunction(readCallback, this);
319 }
320
321 #if OS(WINDOWS)
322
323 void ResourceHandle::setHostAllowsAnyHTTPSCertificate(const String& host)
324 {
325     ASSERT(isMainThread());
326
327     allowsAnyHTTPSCertificateHosts(host);
328 }
329
330 void ResourceHandle::setClientCertificateInfo(const String& host, const String& certificate, const String& key)
331 {
332     ASSERT(isMainThread());
333
334     if (fileExists(certificate))
335         addAllowedClientCertificate(host, certificate, key);
336     else
337         LOG(Network, "Invalid client certificate file: %s!\n", certificate.latin1().data());
338 }
339
340 #endif
341
342 #if OS(WINDOWS) && USE(CF)
343
344 void ResourceHandle::setClientCertificate(const String&, CFDataRef)
345 {
346 }
347
348 #endif
349
350 void ResourceHandle::platformSetDefersLoading(bool defers)
351 {
352     ASSERT(isMainThread());
353
354     auto action = [defers, this, protectedThis = makeRef(*this)]() {
355         if (defers) {
356             CURLcode error = d->m_curlHandle.pause(CURLPAUSE_ALL);
357             // If we could not defer the handle, so don't do it.
358             if (error != CURLE_OK)
359                 return;
360         } else {
361             CURLcode error = d->m_curlHandle.pause(CURLPAUSE_CONT);
362             if (error != CURLE_OK) {
363                 // Restarting the handle has failed so just cancel it.
364                 cancel();
365             }
366         }
367     };
368
369     if (d->m_job) {
370         CurlJobManager::singleton().callOnJobThread(WTFMove(action));
371     } else {
372         action();
373     }
374 }
375
376 void ResourceHandleInternal::didFinish()
377 {
378     calculateWebTimingInformations();
379
380     if (m_cancelled)
381         return;
382
383     if (!m_response.responseFired()) {
384         handleLocalReceiveResponse();
385         if (m_cancelled)
386             return;
387     }
388
389     if (m_multipartHandle)
390         m_multipartHandle->contentEnded();
391
392     if (client()) {
393         client()->didFinishLoading(m_handle);
394         CurlCacheManager::getInstance().didFinishLoading(*m_handle);
395     }
396 }
397
398 void ResourceHandleInternal::didFail()
399 {
400     if (m_cancelled)
401         return;
402     URL url = m_curlHandle.getEffectiveURL();
403     if (client()) {
404         client()->didFail(m_handle, ResourceError(m_curlHandle, m_sslErrors));
405         CurlCacheManager::getInstance().didFail(*m_handle);
406     }
407 }
408
409 bool ResourceHandle::shouldUseCredentialStorage()
410 {
411     return (!client() || client()->shouldUseCredentialStorage(this)) && firstRequest().url().protocolIsInHTTPFamily();
412 }
413
414 void ResourceHandle::didReceiveAuthenticationChallenge(const AuthenticationChallenge& challenge)
415 {
416     ASSERT(isMainThread());
417
418     String partition = firstRequest().cachePartition();
419
420     if (!d->m_user.isNull() && !d->m_pass.isNull()) {
421         Credential credential(d->m_user, d->m_pass, CredentialPersistenceNone);
422
423         URL urlToStore;
424         if (challenge.failureResponse().httpStatusCode() == 401)
425             urlToStore = challenge.failureResponse().url();
426         CredentialStorage::defaultCredentialStorage().set(partition, credential, challenge.protectionSpace(), urlToStore);
427         
428         d->m_curlHandle.setHttpAuthUserPass(credential.user(), credential.password());
429
430         d->m_user = String();
431         d->m_pass = String();
432         // FIXME: Per the specification, the user shouldn't be asked for credentials if there were incorrect ones provided explicitly.
433         return;
434     }
435
436     if (shouldUseCredentialStorage()) {
437         if (!d->m_initialCredential.isEmpty() || challenge.previousFailureCount()) {
438             // The stored credential wasn't accepted, stop using it.
439             // There is a race condition here, since a different credential might have already been stored by another ResourceHandle,
440             // but the observable effect should be very minor, if any.
441             CredentialStorage::defaultCredentialStorage().remove(partition, challenge.protectionSpace());
442         }
443
444         if (!challenge.previousFailureCount()) {
445             Credential credential = CredentialStorage::defaultCredentialStorage().get(partition, challenge.protectionSpace());
446             if (!credential.isEmpty() && credential != d->m_initialCredential) {
447                 ASSERT(credential.persistence() == CredentialPersistenceNone);
448                 if (challenge.failureResponse().httpStatusCode() == 401) {
449                     // Store the credential back, possibly adding it as a default for this directory.
450                     CredentialStorage::defaultCredentialStorage().set(partition, credential, challenge.protectionSpace(), challenge.failureResponse().url());
451                 }
452
453                 d->m_curlHandle.setHttpAuthUserPass(credential.user(), credential.password());
454                 return;
455             }
456         }
457     }
458
459     d->m_currentWebChallenge = challenge;
460     
461     if (client())
462         client()->didReceiveAuthenticationChallenge(this, d->m_currentWebChallenge);
463 }
464
465 void ResourceHandle::receivedCredential(const AuthenticationChallenge& challenge, const Credential& credential)
466 {
467     ASSERT(isMainThread());
468
469     if (challenge != d->m_currentWebChallenge)
470         return;
471
472     if (credential.isEmpty()) {
473         receivedRequestToContinueWithoutCredential(challenge);
474         return;
475     }
476
477     String partition = firstRequest().cachePartition();
478
479     if (shouldUseCredentialStorage()) {
480         if (challenge.failureResponse().httpStatusCode() == 401) {
481             URL urlToStore = challenge.failureResponse().url();
482             CredentialStorage::defaultCredentialStorage().set(partition, credential, challenge.protectionSpace(), urlToStore);
483         }
484     }
485
486     d->m_curlHandle.setHttpAuthUserPass(credential.user(), credential.password());
487     clearAuthentication();
488 }
489
490 void ResourceHandle::receivedRequestToContinueWithoutCredential(const AuthenticationChallenge& challenge)
491 {
492     ASSERT(isMainThread());
493
494     if (challenge != d->m_currentWebChallenge)
495         return;
496
497     d->m_curlHandle.setHttpAuthUserPass("", "");
498     clearAuthentication();
499 }
500
501 void ResourceHandle::receivedCancellation(const AuthenticationChallenge& challenge)
502 {
503     ASSERT(isMainThread());
504
505     if (challenge != d->m_currentWebChallenge)
506         return;
507
508     if (client())
509         client()->receivedCancellation(this, challenge);
510 }
511
512 void ResourceHandle::receivedRequestToPerformDefaultHandling(const AuthenticationChallenge&)
513 {
514     ASSERT_NOT_REACHED();
515 }
516
517 void ResourceHandle::receivedChallengeRejection(const AuthenticationChallenge&)
518 {
519     ASSERT_NOT_REACHED();
520 }
521
522 void ResourceHandleInternal::calculateWebTimingInformations()
523 {
524     double preTransferTime = 0;
525     double dnslookupTime = 0;
526     double connectTime = 0;
527     double appConnectTime = 0;
528
529     m_curlHandle.getTimes(preTransferTime, dnslookupTime, connectTime, appConnectTime);
530
531     m_response.deprecatedNetworkLoadMetrics().domainLookupStart = Seconds(0);
532     m_response.deprecatedNetworkLoadMetrics().domainLookupEnd = Seconds(dnslookupTime);
533
534     m_response.deprecatedNetworkLoadMetrics().connectStart = Seconds(dnslookupTime);
535     m_response.deprecatedNetworkLoadMetrics().connectEnd = Seconds(connectTime);
536
537     m_response.deprecatedNetworkLoadMetrics().requestStart = Seconds(connectTime);
538     m_response.deprecatedNetworkLoadMetrics().responseStart = Seconds(preTransferTime);
539
540     if (appConnectTime)
541         m_response.deprecatedNetworkLoadMetrics().secureConnectionStart = Seconds(connectTime);
542 }
543
544 void ResourceHandleInternal::handleLocalReceiveResponse()
545 {
546     ASSERT(isMainThread());
547
548     // since the code in headerCallback will not have run for local files
549     // the code to set the URL and fire didReceiveResponse is never run,
550     // which means the ResourceLoader's response does not contain the URL.
551     // Run the code here for local files to resolve the issue.
552     // TODO: See if there is a better approach for handling this.
553     URL url = m_curlHandle.getEffectiveURL();
554     ASSERT(url.isValid());
555     m_response.setURL(url);
556     if (client())
557         client()->didReceiveResponse(m_handle, ResourceResponse(m_response));
558     m_response.setResponseFired(true);
559 }
560
561 inline static bool isHttpInfo(int statusCode)
562 {
563     return 100 <= statusCode && statusCode < 200;
564 }
565
566 inline static bool isHttpRedirect(int statusCode)
567 {
568     return 300 <= statusCode && statusCode < 400 && statusCode != 304;
569 }
570
571 inline static bool isHttpAuthentication(int statusCode)
572 {
573     return statusCode == 401;
574 }
575
576 inline static bool isHttpNotModified(int statusCode)
577 {
578     return statusCode == 304;
579 }
580
581 static bool isAppendableHeader(const String &key)
582 {
583     static const char* appendableHeaders[] = {
584         "access-control-allow-headers",
585         "access-control-allow-methods",
586         "access-control-allow-origin",
587         "access-control-expose-headers",
588         "allow",
589         "cache-control",
590         "connection",
591         "content-encoding",
592         "content-language",
593         "if-match",
594         "if-none-match",
595         "keep-alive",
596         "pragma",
597         "proxy-authenticate",
598         "public",
599         "server",
600         "set-cookie",
601         "te",
602         "trailer",
603         "transfer-encoding",
604         "upgrade",
605         "user-agent",
606         "vary",
607         "via",
608         "warning",
609         "www-authenticate"
610     };
611
612     // Custom headers start with 'X-', and need no further checking.
613     if (key.startsWith("x-", /* caseSensitive */ false))
614         return true;
615
616     for (auto& header : appendableHeaders) {
617         if (equalIgnoringASCIICase(key, header))
618             return true;
619     }
620
621     return false;
622 }
623
624 static void removeLeadingAndTrailingQuotes(String& value)
625 {
626     unsigned length = value.length();
627     if (value.startsWith('"') && value.endsWith('"') && length > 1)
628         value = value.substring(1, length - 2);
629 }
630
631 static bool getProtectionSpace(ResourceHandle* job, const ResourceResponse& response, ProtectionSpace& protectionSpace)
632 {
633     ResourceHandleInternal* d = job->getInternal();
634
635     CURLcode err;
636
637     long port = 0;
638     err = d->m_curlHandle.getPrimaryPort(port);
639     if (err != CURLE_OK)
640         return false;
641
642     long availableAuth = CURLAUTH_NONE;
643     err = d->m_curlHandle.getHttpAuthAvail(availableAuth);
644     if (err != CURLE_OK)
645         return false;
646
647     URL url = d->m_curlHandle.getEffectiveURL();
648     if (!url.isValid())
649         return false;
650
651     String host = url.host();
652     StringView protocol = url.protocol();
653
654     String realm;
655
656     const String authHeader = response.httpHeaderField(HTTPHeaderName::Authorization);
657     const String realmString = "realm=";
658     int realmPos = authHeader.find(realmString);
659     if (realmPos > 0) {
660         realm = authHeader.substring(realmPos + realmString.length());
661         realm = realm.left(realm.find(','));
662         removeLeadingAndTrailingQuotes(realm);
663     }
664
665     ProtectionSpaceServerType serverType = ProtectionSpaceServerHTTP;
666     if (protocol == "https")
667         serverType = ProtectionSpaceServerHTTPS;
668
669     ProtectionSpaceAuthenticationScheme authScheme = ProtectionSpaceAuthenticationSchemeUnknown;
670
671     if (availableAuth & CURLAUTH_BASIC)
672         authScheme = ProtectionSpaceAuthenticationSchemeHTTPBasic;
673     if (availableAuth & CURLAUTH_DIGEST)
674         authScheme = ProtectionSpaceAuthenticationSchemeHTTPDigest;
675     if (availableAuth & CURLAUTH_GSSNEGOTIATE)
676         authScheme = ProtectionSpaceAuthenticationSchemeNegotiate;
677     if (availableAuth & CURLAUTH_NTLM)
678         authScheme = ProtectionSpaceAuthenticationSchemeNTLM;
679
680     protectionSpace = ProtectionSpace(host, port, serverType, realm, authScheme);
681
682     return true;
683 }
684
685 size_t ResourceHandleInternal::willPrepareSendData(char* ptr, size_t blockSize, size_t numberOfBlocks)
686 {
687     if (!m_formDataStream.hasMoreElements())
688         return 0;
689
690     size_t size = m_formDataStream.read(ptr, blockSize, numberOfBlocks);
691
692     // Something went wrong so cancel the job.
693     if (!size) {
694         m_handle->cancel();
695         return 0;
696     }
697
698     return size;
699
700 }
701
702 void ResourceHandleInternal::didReceiveHeaderLine(const String& header)
703 {
704     ASSERT(isMainThread());
705
706     auto splitPosition = header.find(":");
707     if (splitPosition != notFound) {
708         String key = header.left(splitPosition).stripWhiteSpace();
709         String value = header.substring(splitPosition + 1).stripWhiteSpace();
710
711         if (isAppendableHeader(key))
712             m_response.addHTTPHeaderField(key, value);
713         else
714             m_response.setHTTPHeaderField(key, value);
715     } else if (header.startsWith("HTTP", false)) {
716         // This is the first line of the response.
717         // Extract the http status text from this.
718         //
719         // If the FOLLOWLOCATION option is enabled for the curl handle then
720         // curl will follow the redirections internally. Thus this header callback
721         // will be called more than one time with the line starting "HTTP" for one job.
722         long httpCode = 0;
723         m_curlHandle.getResponseCode(httpCode);
724
725         String httpCodeString = String::number(httpCode);
726         int statusCodePos = header.find(httpCodeString);
727
728         if (statusCodePos != notFound) {
729             // The status text is after the status code.
730             String status = header.substring(statusCodePos + httpCodeString.length());
731             m_response.setHTTPStatusText(status.stripWhiteSpace());
732         }
733     }
734 }
735
736 void ResourceHandleInternal::didReceiveAllHeaders(long httpCode, long long contentLength)
737 {
738     ASSERT(isMainThread());
739
740     m_response.setExpectedContentLength(contentLength);
741
742     m_response.setURL(m_curlHandle.getEffectiveURL());
743
744     m_response.setHTTPStatusCode(httpCode);
745     m_response.setMimeType(extractMIMETypeFromMediaType(m_response.httpHeaderField(HTTPHeaderName::ContentType)).convertToASCIILowercase());
746     m_response.setTextEncodingName(extractCharsetFromMediaType(m_response.httpHeaderField(HTTPHeaderName::ContentType)));
747
748     if (m_response.isMultipart()) {
749         String boundary;
750         bool parsed = MultipartHandle::extractBoundary(m_response.httpHeaderField(HTTPHeaderName::ContentType), boundary);
751         if (parsed)
752             m_multipartHandle = std::make_unique<MultipartHandle>(m_handle, boundary);
753     }
754
755     // HTTP redirection
756     if (isHttpRedirect(httpCode)) {
757         String location = m_response.httpHeaderField(HTTPHeaderName::Location);
758         if (!location.isEmpty()) {
759             URL newURL = URL(m_firstRequest.url(), location);
760
761             ResourceRequest redirectedRequest = m_firstRequest;
762             redirectedRequest.setURL(newURL);
763             ResourceResponse response = m_response;
764             if (client())
765                 client()->willSendRequest(m_handle, WTFMove(redirectedRequest), WTFMove(response));
766
767             m_firstRequest.setURL(newURL);
768
769             return;
770         }
771     } else if (isHttpAuthentication(httpCode)) {
772         ProtectionSpace protectionSpace;
773         if (getProtectionSpace(m_handle, m_response, protectionSpace)) {
774             Credential credential;
775             AuthenticationChallenge challenge(protectionSpace, credential, m_authFailureCount, m_response, ResourceError());
776             challenge.setAuthenticationClient(m_handle);
777             m_handle->didReceiveAuthenticationChallenge(challenge);
778             m_authFailureCount++;
779             return;
780         }
781     }
782
783     if (client()) {
784         if (isHttpNotModified(httpCode)) {
785             const String& url = m_firstRequest.url().string();
786             if (CurlCacheManager::getInstance().getCachedResponse(url, m_response)) {
787                 if (m_addedCacheValidationHeaders) {
788                     m_response.setHTTPStatusCode(200);
789                     m_response.setHTTPStatusText("OK");
790                 }
791             }
792         }
793         client()->didReceiveResponse(m_handle, ResourceResponse(m_response));
794         CurlCacheManager::getInstance().didReceiveResponse(*m_handle, m_response);
795     }
796
797     m_response.setResponseFired(true);
798 }
799
800 void ResourceHandleInternal::didReceiveContentData()
801 {
802     ASSERT(isMainThread());
803
804     if (!m_response.responseFired())
805         handleLocalReceiveResponse();
806
807     Vector<char> buffer;
808     {
809         LockHolder locker { m_receivedBufferMutex };
810         buffer = WTFMove(m_receivedBuffer);
811     }
812
813     char* ptr = buffer.begin();
814     size_t size = buffer.size();
815
816     if (m_multipartHandle)
817         m_multipartHandle->contentReceived(static_cast<const char*>(ptr), size);
818     else if (client()) {
819         client()->didReceiveData(m_handle, ptr, size, 0);
820         CurlCacheManager::getInstance().didReceiveData(*m_handle, ptr, size);
821     }
822 }
823
824 /* This is called to obtain HTTP POST or PUT data.
825 Iterate through FormData elements and upload files.
826 Carefully respect the given buffer size and fill the rest of the data at the next calls.
827 */
828 size_t ResourceHandleInternal::readCallback(char* ptr, size_t size, size_t nmemb, void* data)
829 {
830     ASSERT(!isMainThread());
831
832     ResourceHandleInternal* d = static_cast<ResourceHandleInternal*>(data);
833
834     if (d->m_cancelled)
835         return 0;
836
837     // We should never be called when deferred loading is activated.
838     ASSERT(!d->m_defersLoading);
839
840     if (!size || !nmemb)
841         return 0;
842
843     return d->willPrepareSendData(ptr, size, nmemb);
844 }
845
846 /*
847 * This is being called for each HTTP header in the response. This includes '\r\n'
848 * for the last line of the header.
849 *
850 * We will add each HTTP Header to the ResourceResponse and on the termination
851 * of the header (\r\n) we will parse Content-Type and Content-Disposition and
852 * update the ResourceResponse and then send it away.
853 *
854 */
855 size_t ResourceHandleInternal::headerCallback(char* ptr, size_t size, size_t nmemb, void* data)
856 {
857     ASSERT(!isMainThread());
858
859     ResourceHandleInternal* d = static_cast<ResourceHandleInternal*>(data);
860     ResourceHandle* job = d->m_handle;
861
862     if (d->m_cancelled)
863         return 0;
864
865     // We should never be called when deferred loading is activated.
866     ASSERT(!d->m_defersLoading);
867
868     size_t totalSize = size * nmemb;
869
870     String header(static_cast<const char*>(ptr), totalSize);
871
872     /*
873     * a) We can finish and send the ResourceResponse
874     * b) We will add the current header to the HTTPHeaderMap of the ResourceResponse
875     *
876     * The HTTP standard requires to use \r\n but for compatibility it recommends to
877     * accept also \n.
878     */
879     if (header == AtomicString("\r\n") || header == AtomicString("\n")) {
880         long httpCode = 0;
881         d->m_curlHandle.getResponseCode(httpCode);
882
883         if (!httpCode) {
884             // Comes here when receiving 200 Connection Established. Just return.
885             return totalSize;
886         }
887         if (isHttpInfo(httpCode)) {
888             // Just return when receiving http info, e.g. HTTP/1.1 100 Continue.
889             // If not, the request might be cancelled, because the MIME type will be empty for this response.
890             return totalSize;
891         }
892
893         long long contentLength = 0;
894         d->m_curlHandle.getContentLenghtDownload(contentLength);
895
896         callOnMainThread([job = RefPtr<ResourceHandle>(job), d, httpCode, contentLength] {
897             if (!d->m_cancelled)
898                 d->didReceiveAllHeaders(httpCode, contentLength);
899         });
900     } else {
901         callOnMainThread([job = RefPtr<ResourceHandle>(job), d, header] {
902             if (!d->m_cancelled)
903                 d->didReceiveHeaderLine(header);
904         });
905     }
906
907     return totalSize;
908 }
909
910 // called with data after all headers have been processed via headerCallback
911 size_t ResourceHandleInternal::writeCallback(char* ptr, size_t size, size_t nmemb, void* data)
912 {
913     ASSERT(!isMainThread());
914
915     ResourceHandleInternal* d = static_cast<ResourceHandleInternal*>(data);
916     ResourceHandle* job = d->m_handle;
917
918     if (d->m_cancelled)
919         return 0;
920
921     // We should never be called when deferred loading is activated.
922     ASSERT(!d->m_defersLoading);
923
924     size_t totalSize = size * nmemb;
925
926     // this shouldn't be necessary but apparently is. CURL writes the data
927     // of html page even if it is a redirect that was handled internally
928     // can be observed e.g. on gmail.com
929     long httpCode = 0;
930     CURLcode errCd = d->m_curlHandle.getResponseCode(httpCode);
931     if (CURLE_OK == errCd && httpCode >= 300 && httpCode < 400)
932         return totalSize;
933
934     bool shouldCall { false };
935     {
936         LockHolder locker(d->m_receivedBufferMutex);
937         
938         if (d->m_receivedBuffer.isEmpty())
939             shouldCall = true;
940         
941         d->m_receivedBuffer.append(ptr, totalSize);
942     }
943
944     if (shouldCall) {
945         callOnMainThread([job = RefPtr<ResourceHandle>(job), d] {
946             if (!d->m_cancelled)
947                 d->didReceiveContentData();
948         });
949     }
950
951     return totalSize;
952 }
953
954 // sync loader
955
956 void ResourceHandle::platformLoadResourceSynchronously(NetworkingContext* context, const ResourceRequest& request, StoredCredentials, ResourceError& error, ResourceResponse& response, Vector<char>& data)
957 {
958     ASSERT(isMainThread());
959
960     SynchronousLoaderClient client;
961     RefPtr<ResourceHandle> handle = adoptRef(new ResourceHandle(context, request, &client, false, false));
962
963     handle->d->dispatchSynchronousJob();
964
965     error = client.error();
966     data.swap(client.mutableData());
967     response = client.response();
968 }
969
970 void ResourceHandleInternal::dispatchSynchronousJob()
971 {
972     URL kurl = m_firstRequest.url();
973
974     if (kurl.protocolIsData()) {
975         handleDataURL();
976         return;
977     }
978
979     // If defersLoading is true and we call curl_easy_perform
980     // on a paused handle, libcURL would do the transfert anyway
981     // and we would assert so force defersLoading to be false.
982     m_defersLoading = false;
983
984     initialize();
985
986     // curl_easy_perform blocks until the transfert is finished.
987     CURLcode ret = m_curlHandle.perform();
988
989     calculateWebTimingInformations();
990
991     if (client()) {
992         if (ret != CURLE_OK)
993             client()->didFail(m_handle, ResourceError(m_curlHandle, m_sslErrors));
994         else
995             client()->didReceiveResponse(m_handle, ResourceResponse(m_response));
996     }
997 }
998
999 void ResourceHandleInternal::handleDataURL()
1000 {
1001     ASSERT(m_firstRequest.url().protocolIsData());
1002     String url = m_firstRequest.url().string();
1003
1004     ASSERT(client());
1005
1006     int index = url.find(',');
1007     if (index == -1) {
1008         client()->cannotShowURL(m_handle);
1009         return;
1010     }
1011
1012     String mediaType = url.substring(5, index - 5);
1013     String data = url.substring(index + 1);
1014
1015     bool base64 = mediaType.endsWith(";base64", false);
1016     if (base64)
1017         mediaType = mediaType.left(mediaType.length() - 7);
1018
1019     if (mediaType.isEmpty())
1020         mediaType = "text/plain";
1021
1022     String mimeType = extractMIMETypeFromMediaType(mediaType);
1023     String charset = extractCharsetFromMediaType(mediaType);
1024
1025     if (charset.isEmpty())
1026         charset = "US-ASCII";
1027
1028     ResourceResponse response;
1029     response.setMimeType(mimeType);
1030     response.setTextEncodingName(charset);
1031     response.setURL(m_firstRequest.url());
1032
1033     if (base64) {
1034         data = decodeURLEscapeSequences(data);
1035         client()->didReceiveResponse(m_handle, WTFMove(response));
1036
1037         // didReceiveResponse might cause the client to be deleted.
1038         if (client()) {
1039             Vector<char> out;
1040             if (base64Decode(data, out, Base64IgnoreSpacesAndNewLines) && out.size() > 0)
1041                 client()->didReceiveData(m_handle, out.data(), out.size(), 0);
1042         }
1043     } else {
1044         TextEncoding encoding(charset);
1045         data = decodeURLEscapeSequences(data, encoding);
1046         client()->didReceiveResponse(m_handle, WTFMove(response));
1047
1048         // didReceiveResponse might cause the client to be deleted.
1049         if (client()) {
1050             CString encodedData = encoding.encode(data, URLEncodedEntitiesForUnencodables);
1051             if (encodedData.length())
1052                 client()->didReceiveData(m_handle, encodedData.data(), encodedData.length(), 0);
1053         }
1054     }
1055
1056     if (client())
1057         client()->didFinishLoading(m_handle);
1058 }
1059
1060 } // namespace WebCore
1061
1062 #endif