2 * Copyright (C) 2018 Sony Interactive Entertainment Inc.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23 * THE POSSIBILITY OF SUCH DAMAGE.
27 #include "CurlRequest.h"
31 #include "CurlRequestClient.h"
32 #include "CurlRequestScheduler.h"
33 #include "MIMETypeRegistry.h"
34 #include "ResourceError.h"
35 #include "SharedBuffer.h"
36 #include <wtf/Language.h>
37 #include <wtf/MainThread.h>
41 CurlRequest::CurlRequest(const ResourceRequest&request, CurlRequestClient* client, bool shouldSuspend, bool enableMultipart)
42 : m_request(request.isolatedCopy())
44 , m_shouldSuspend(shouldSuspend)
45 , m_enableMultipart(enableMultipart)
46 , m_formDataStream(m_request.httpBody())
48 ASSERT(isMainThread());
51 void CurlRequest::setUserPass(const String& user, const String& password)
53 ASSERT(isMainThread());
55 m_user = user.isolatedCopy();
56 m_password = password.isolatedCopy();
59 void CurlRequest::start(bool isSyncRequest)
61 // The pausing of transfer does not work with protocols, like file://.
62 // Therefore, PAUSE can not be done in didReceiveData().
63 // It means that the same logic as http:// can not be used.
64 // In the file scheme, invokeDidReceiveResponse() is done first.
65 // Then StartWithJobManager is called with completeDidReceiveResponse and start transfer with libcurl.
67 // http : didReceiveHeader => didReceiveData[PAUSE] => invokeDidReceiveResponse => (MainThread)curlDidReceiveResponse => completeDidReceiveResponse[RESUME] => didReceiveData
68 // file : invokeDidReceiveResponseForFile => (MainThread)curlDidReceiveResponse => completeDidReceiveResponse => didReceiveData
70 ASSERT(isMainThread());
72 m_isSyncRequest = isSyncRequest;
74 auto url = m_request.url().isolatedCopy();
76 if (!m_isSyncRequest) {
77 // For asynchronous, use CurlRequestScheduler. Curl processes runs on sub thread.
78 if (url.isLocalFile())
79 invokeDidReceiveResponseForFile(url);
81 startWithJobManager();
83 // For synchronous, does not use CurlRequestScheduler. Curl processes runs on main thread.
84 // curl_easy_perform blocks until the transfer is finished.
86 if (url.isLocalFile())
87 invokeDidReceiveResponseForFile(url);
90 CURLcode resultCode = m_curlHandle->perform();
91 didCompleteTransfer(resultCode);
96 void CurlRequest::startWithJobManager()
98 ASSERT(isMainThread());
100 CurlContext::singleton().scheduler().add(this);
103 void CurlRequest::cancel()
105 ASSERT(isMainThread());
107 if (isCompletedOrCancelled())
112 if (!m_isSyncRequest) {
113 auto& scheduler = CurlContext::singleton().scheduler();
115 if (needToInvokeDidCancelTransfer()) {
116 scheduler.callOnWorkerThread([protectedThis = makeRef(*this)]() {
117 protectedThis->didCancelTransfer();
120 scheduler.cancel(this);
122 if (needToInvokeDidCancelTransfer())
126 setRequestPaused(false);
127 setCallbackPaused(false);
131 void CurlRequest::suspend()
133 ASSERT(isMainThread());
135 setRequestPaused(true);
138 void CurlRequest::resume()
140 ASSERT(isMainThread());
142 setRequestPaused(false);
145 /* `this` is protected inside this method. */
146 void CurlRequest::callClient(WTF::Function<void(CurlRequest&, CurlRequestClient&)> task)
148 if (isMainThread()) {
149 if (CurlRequestClient* client = m_client) {
150 RefPtr<CurlRequestClient> protectedClient(client);
151 task(*this, *client);
154 callOnMainThread([this, protectedThis = makeRef(*this), task = WTFMove(task)]() mutable {
155 if (CurlRequestClient* client = protectedThis->m_client) {
156 RefPtr<CurlRequestClient> protectedClient(client);
157 task(*this, *client);
163 CURL* CurlRequest::setupTransfer()
165 auto& sslHandle = CurlContext::singleton().sslHandle();
167 auto httpHeaderFields = m_request.httpHeaderFields();
168 appendAcceptLanguageHeader(httpHeaderFields);
170 m_curlHandle = std::make_unique<CurlHandle>();
172 m_curlHandle->initialize();
173 m_curlHandle->setUrl(m_request.url());
174 m_curlHandle->appendRequestHeaders(httpHeaderFields);
176 const auto& method = m_request.httpMethod();
178 m_curlHandle->enableHttpGetRequest();
179 else if (method == "POST")
180 setupPOST(m_request);
181 else if (method == "PUT")
183 else if (method == "HEAD")
184 m_curlHandle->enableHttpHeadRequest();
186 m_curlHandle->setHttpCustomRequest(method);
190 if (!m_user.isEmpty() || !m_password.isEmpty()) {
191 m_curlHandle->enableHttpAuthentication(CURLAUTH_ANY);
192 m_curlHandle->setHttpAuthUserPass(m_user, m_password);
195 m_curlHandle->setHeaderCallbackFunction(didReceiveHeaderCallback, this);
196 m_curlHandle->setWriteCallbackFunction(didReceiveDataCallback, this);
198 m_curlHandle->enableShareHandle();
199 m_curlHandle->enableAllowedProtocols();
200 m_curlHandle->enableAcceptEncoding();
202 m_curlHandle->setTimeout(Seconds(m_request.timeoutInterval()));
203 m_curlHandle->setDnsCacheTimeout(CurlContext::singleton().dnsCacheTimeout());
204 m_curlHandle->setConnectTimeout(CurlContext::singleton().connectTimeout());
206 m_curlHandle->enableProxyIfExists();
208 m_curlHandle->setSslVerifyPeer(CurlHandle::VerifyPeer::Enable);
209 m_curlHandle->setSslVerifyHost(CurlHandle::VerifyHost::StrictNameCheck);
211 auto sslClientCertificate = sslHandle.getSSLClientCertificate(m_request.url().host());
212 if (sslClientCertificate) {
213 m_curlHandle->setSslCert(sslClientCertificate->first.utf8().data());
214 m_curlHandle->setSslCertType("P12");
215 m_curlHandle->setSslKeyPassword(sslClientCertificate->second.utf8().data());
218 if (sslHandle.shouldIgnoreSSLErrors())
219 m_curlHandle->setSslVerifyPeer(CurlHandle::VerifyPeer::Disable);
221 m_curlHandle->setSslCtxCallbackFunction(willSetupSslCtxCallback, this);
223 m_curlHandle->setCACertPath(sslHandle.getCACertPath());
229 m_curlHandle->enableVerboseIfUsed();
230 m_curlHandle->enableStdErrIfUsed();
233 return m_curlHandle->handle();
236 CURLcode CurlRequest::willSetupSslCtx(void* sslCtx)
238 m_sslVerifier = std::make_unique<CurlSSLVerifier>(m_curlHandle.get(), m_request.url().host(), sslCtx);
243 // This is called to obtain HTTP POST or PUT data.
244 // Iterate through FormData elements and upload files.
245 // Carefully respect the given buffer size and fill the rest of the data at the next calls.
247 size_t CurlRequest::willSendData(char* buffer, size_t blockSize, size_t numberOfBlocks)
249 if (isCompletedOrCancelled())
250 return CURL_READFUNC_ABORT;
252 if (!blockSize || !numberOfBlocks)
253 return CURL_READFUNC_ABORT;
255 // Check for overflow.
256 if (blockSize > (std::numeric_limits<size_t>::max() / numberOfBlocks))
257 return CURL_READFUNC_ABORT;
259 size_t bufferSize = blockSize * numberOfBlocks;
260 auto sendBytes = m_formDataStream.read(buffer, bufferSize);
262 // Something went wrong so error the job.
263 return CURL_READFUNC_ABORT;
266 callClient([totalReadSize = m_formDataStream.totalReadSize(), totalSize = m_formDataStream.totalSize()](CurlRequest& request, CurlRequestClient& client) {
267 client.curlDidSendData(request, totalReadSize, totalSize);
273 // This is being called for each HTTP header in the response. This includes '\r\n'
274 // for the last line of the header.
276 size_t CurlRequest::didReceiveHeader(String&& header)
278 static const auto emptyLineCRLF = "\r\n";
279 static const auto emptyLineLF = "\n";
281 if (isCompletedOrCancelled())
284 // libcurl sends all headers that libcurl received to application.
285 // So, in digest authentication, a block of response headers are received twice consecutively from libcurl.
286 // For example, when authentication succeeds, the first block is "401 Authorization", and the second block is "200 OK".
287 // Also, "100 Continue" and "200 Connection Established" do the same behavior.
288 // In this process, deletes the first block to send a correct headers to WebCore.
289 if (m_didReceiveResponse) {
290 m_didReceiveResponse = false;
291 m_response = CurlResponse { };
292 m_multipartHandle = nullptr;
295 auto receiveBytes = static_cast<size_t>(header.length());
297 // The HTTP standard requires to use \r\n but for compatibility it recommends to accept also \n.
298 if ((header != emptyLineCRLF) && (header != emptyLineLF)) {
299 m_response.headers.append(WTFMove(header));
304 if (auto code = m_curlHandle->getResponseCode())
307 long httpConnectCode = 0;
308 if (auto code = m_curlHandle->getHttpConnectCode())
309 httpConnectCode = *code;
311 m_didReceiveResponse = true;
313 m_response.url = m_request.url();
314 m_response.statusCode = statusCode;
315 m_response.httpConnectCode = httpConnectCode;
317 if (auto length = m_curlHandle->getContentLength())
318 m_response.expectedContentLength = *length;
320 if (auto port = m_curlHandle->getPrimaryPort())
321 m_response.connectPort = *port;
323 if (auto auth = m_curlHandle->getHttpAuthAvail())
324 m_response.availableHttpAuth = *auth;
326 if (auto version = m_curlHandle->getHttpVersion())
327 m_response.httpVersion = *version;
329 if (auto metrics = m_curlHandle->getNetworkLoadMetrics())
330 m_networkLoadMetrics = *metrics;
332 if (m_enableMultipart)
333 m_multipartHandle = CurlMultipartHandle::createIfNeeded(*this, m_response);
335 // Response will send at didReceiveData() or didCompleteTransfer()
336 // to receive continueDidRceiveResponse() for asynchronously.
341 // called with data after all headers have been processed via headerCallback
343 size_t CurlRequest::didReceiveData(Ref<SharedBuffer>&& buffer)
345 if (isCompletedOrCancelled())
348 if (needToInvokeDidReceiveResponse()) {
349 if (!m_isSyncRequest) {
350 // For asynchronous, pause until completeDidReceiveResponse() is called.
351 setCallbackPaused(true);
352 invokeDidReceiveResponse(m_response, Action::ReceiveData);
353 return CURL_WRITEFUNC_PAUSE;
356 // For synchronous, completeDidReceiveResponse() is called in invokeDidReceiveResponse().
357 // In this case, pause is unnecessary.
358 invokeDidReceiveResponse(m_response, Action::None);
361 auto receiveBytes = buffer->size();
363 writeDataToDownloadFileIfEnabled(buffer);
366 if (m_multipartHandle)
367 m_multipartHandle->didReceiveData(buffer);
369 callClient([buffer = WTFMove(buffer)](CurlRequest& request, CurlRequestClient& client) mutable {
370 client.curlDidReceiveBuffer(request, WTFMove(buffer));
378 void CurlRequest::didReceiveHeaderFromMultipart(const Vector<String>& headers)
380 if (isCompletedOrCancelled())
383 CurlResponse response = m_response.isolatedCopy();
384 response.expectedContentLength = 0;
385 response.headers.clear();
387 for (auto header : headers)
388 response.headers.append(header);
390 invokeDidReceiveResponse(response, Action::None);
393 void CurlRequest::didReceiveDataFromMultipart(Ref<SharedBuffer>&& buffer)
395 if (isCompletedOrCancelled())
398 auto receiveBytes = buffer->size();
401 callClient([buffer = WTFMove(buffer)](CurlRequest& request, CurlRequestClient& client) mutable {
402 client.curlDidReceiveBuffer(request, WTFMove(buffer));
407 void CurlRequest::didCompleteTransfer(CURLcode result)
410 m_curlHandle = nullptr;
414 if (needToInvokeDidReceiveResponse()) {
415 // Processing of didReceiveResponse() has not been completed. (For example, HEAD method)
416 // When completeDidReceiveResponse() is called, didCompleteTransfer() will be called again.
418 m_finishedResultCode = result;
419 invokeDidReceiveResponse(m_response, Action::FinishTransfer);
423 if (result == CURLE_OK) {
424 if (m_multipartHandle)
425 m_multipartHandle->didComplete();
427 if (auto metrics = m_curlHandle->getNetworkLoadMetrics())
428 m_networkLoadMetrics = *metrics;
431 callClient([](CurlRequest& request, CurlRequestClient& client) {
432 client.curlDidComplete(request);
435 auto type = (result == CURLE_OPERATION_TIMEDOUT && m_request.timeoutInterval() > 0.0) ? ResourceError::Type::Timeout : ResourceError::Type::General;
436 auto resourceError = ResourceError::httpError(result, m_request.url(), type);
437 if (m_sslVerifier && m_sslVerifier->sslErrors())
438 resourceError.setSslErrors(m_sslVerifier->sslErrors());
441 callClient([error = resourceError.isolatedCopy()](CurlRequest& request, CurlRequestClient& client) {
442 client.curlDidFailWithError(request, error);
447 void CurlRequest::didCancelTransfer()
450 cleanupDownloadFile();
453 void CurlRequest::finalizeTransfer()
456 m_formDataStream.clean();
457 m_sslVerifier = nullptr;
458 m_multipartHandle = nullptr;
459 m_curlHandle = nullptr;
462 void CurlRequest::appendAcceptLanguageHeader(HTTPHeaderMap& header)
464 for (const auto& language : userPreferredLanguages())
465 header.add(HTTPHeaderName::AcceptLanguage, language);
468 void CurlRequest::setupPUT(ResourceRequest& request)
470 m_curlHandle->enableHttpPutRequest();
472 // Disable the Expect: 100 continue header
473 m_curlHandle->removeRequestHeader("Expect");
475 auto elementSize = m_formDataStream.elementSize();
482 void CurlRequest::setupPOST(ResourceRequest& request)
484 m_curlHandle->enableHttpPostRequest();
486 auto elementSize = m_formDataStream.elementSize();
490 // Do not stream for simple POST data
491 if (elementSize == 1) {
492 const auto* postData = m_formDataStream.getPostData();
493 if (postData && postData->size())
494 m_curlHandle->setPostFields(postData->data(), postData->size());
496 setupSendData(false);
499 void CurlRequest::setupSendData(bool forPutMethod)
501 // curl guesses that we want chunked encoding as long as we specify the header
502 if (m_formDataStream.shouldUseChunkTransfer())
503 m_curlHandle->appendRequestHeader("Transfer-Encoding: chunked");
506 m_curlHandle->setInFileSizeLarge(static_cast<curl_off_t>(m_formDataStream.totalSize()));
508 m_curlHandle->setPostFieldLarge(static_cast<curl_off_t>(m_formDataStream.totalSize()));
511 m_curlHandle->setReadCallbackFunction(willSendDataCallback, this);
514 void CurlRequest::invokeDidReceiveResponseForFile(URL& url)
516 // Since the code in didReceiveHeader() will not have run for local files
517 // the code to set the URL and fire didReceiveResponse is never run,
518 // which means the ResourceLoader's response does not contain the URL.
519 // Run the code here for local files to resolve the issue.
521 ASSERT(isMainThread());
522 ASSERT(url.isLocalFile());
524 m_response.url = url;
525 m_response.statusCode = 200;
527 // Determine the MIME type based on the path.
528 m_response.headers.append(String("Content-Type: " + MIMETypeRegistry::getMIMETypeForPath(m_response.url.path())));
530 if (!m_isSyncRequest) {
531 // DidReceiveResponse must not be called immediately
532 CurlContext::singleton().scheduler().callOnWorkerThread([protectedThis = makeRef(*this)]() {
533 protectedThis->invokeDidReceiveResponse(protectedThis->m_response, Action::StartTransfer);
536 // For synchronous, completeDidReceiveResponse() is called in platformContinueSynchronousDidReceiveResponse().
537 invokeDidReceiveResponse(m_response, Action::None);
541 void CurlRequest::invokeDidReceiveResponse(const CurlResponse& response, Action behaviorAfterInvoke)
543 ASSERT(!m_didNotifyResponse || m_multipartHandle);
545 m_didNotifyResponse = true;
546 m_actionAfterInvoke = behaviorAfterInvoke;
548 callClient([response = response.isolatedCopy()](CurlRequest& request, CurlRequestClient& client) {
549 client.curlDidReceiveResponse(request, response);
553 void CurlRequest::completeDidReceiveResponse()
555 ASSERT(isMainThread());
556 ASSERT(m_didNotifyResponse);
557 ASSERT(!m_didReturnFromNotify || m_multipartHandle);
562 if (m_actionAfterInvoke != Action::StartTransfer && isCompleted())
565 m_didReturnFromNotify = true;
567 if (m_actionAfterInvoke == Action::ReceiveData) {
569 setCallbackPaused(false);
570 } else if (m_actionAfterInvoke == Action::StartTransfer) {
571 // Start transfer for file scheme
572 startWithJobManager();
573 } else if (m_actionAfterInvoke == Action::FinishTransfer) {
574 if (!m_isSyncRequest) {
575 CurlContext::singleton().scheduler().callOnWorkerThread([protectedThis = makeRef(*this), finishedResultCode = m_finishedResultCode]() {
576 protectedThis->didCompleteTransfer(finishedResultCode);
579 didCompleteTransfer(m_finishedResultCode);
583 void CurlRequest::setRequestPaused(bool paused)
585 auto wasPaused = isPaused();
587 m_isPausedOfRequest = paused;
589 if (isPaused() == wasPaused)
592 pausedStatusChanged();
595 void CurlRequest::setCallbackPaused(bool paused)
597 auto wasPaused = isPaused();
599 m_isPausedOfCallback = paused;
601 if (isPaused() == wasPaused)
604 // In this case, PAUSE will be executed within didReceiveData(). Change pause state and return.
608 pausedStatusChanged();
611 void CurlRequest::pausedStatusChanged()
613 if (isCompletedOrCancelled())
616 if (!m_isSyncRequest && isMainThread()) {
617 CurlContext::singleton().scheduler().callOnWorkerThread([protectedThis = makeRef(*this), paused = isPaused()]() {
618 if (protectedThis->isCompletedOrCancelled())
621 auto error = protectedThis->m_curlHandle->pause(paused ? CURLPAUSE_ALL : CURLPAUSE_CONT);
622 if ((error != CURLE_OK) && !paused) {
623 // Restarting the handle has failed so just cancel it.
624 callOnMainThread([protectedThis = makeRef(protectedThis.get())]() {
625 protectedThis->cancel();
630 auto error = m_curlHandle->pause(isPaused() ? CURLPAUSE_ALL : CURLPAUSE_CONT);
631 if ((error != CURLE_OK) && !isPaused())
636 void CurlRequest::enableDownloadToFile()
638 LockHolder locker(m_downloadMutex);
639 m_isEnabledDownloadToFile = true;
642 const String& CurlRequest::getDownloadedFilePath()
644 LockHolder locker(m_downloadMutex);
645 return m_downloadFilePath;
648 void CurlRequest::writeDataToDownloadFileIfEnabled(const SharedBuffer& buffer)
651 LockHolder locker(m_downloadMutex);
653 if (!m_isEnabledDownloadToFile)
656 if (m_downloadFilePath.isEmpty())
657 m_downloadFilePath = FileSystem::openTemporaryFile("download", m_downloadFileHandle);
660 if (m_downloadFileHandle != FileSystem::invalidPlatformFileHandle)
661 FileSystem::writeToFile(m_downloadFileHandle, buffer.data(), buffer.size());
664 void CurlRequest::closeDownloadFile()
666 LockHolder locker(m_downloadMutex);
668 if (m_downloadFileHandle == FileSystem::invalidPlatformFileHandle)
671 FileSystem::closeFile(m_downloadFileHandle);
672 m_downloadFileHandle = FileSystem::invalidPlatformFileHandle;
675 void CurlRequest::cleanupDownloadFile()
677 LockHolder locker(m_downloadMutex);
679 if (!m_downloadFilePath.isEmpty()) {
680 FileSystem::deleteFile(m_downloadFilePath);
681 m_downloadFilePath = String();
685 CURLcode CurlRequest::willSetupSslCtxCallback(CURL*, void* sslCtx, void* userData)
687 return static_cast<CurlRequest*>(userData)->willSetupSslCtx(sslCtx);
690 size_t CurlRequest::willSendDataCallback(char* ptr, size_t blockSize, size_t numberOfBlocks, void* userData)
692 return static_cast<CurlRequest*>(userData)->willSendData(ptr, blockSize, numberOfBlocks);
695 size_t CurlRequest::didReceiveHeaderCallback(char* ptr, size_t blockSize, size_t numberOfBlocks, void* userData)
697 return static_cast<CurlRequest*>(userData)->didReceiveHeader(String(ptr, blockSize * numberOfBlocks));
700 size_t CurlRequest::didReceiveDataCallback(char* ptr, size_t blockSize, size_t numberOfBlocks, void* userData)
702 return static_cast<CurlRequest*>(userData)->didReceiveData(SharedBuffer::create(ptr, blockSize * numberOfBlocks));