2 * Copyright (C) 2017 Sony Interactive Entertainment Inc.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23 * THE POSSIBILITY OF SUCH DAMAGE.
27 #include "CurlRequest.h"
31 #include "CurlRequestClient.h"
32 #include "CurlRequestScheduler.h"
33 #include "MIMETypeRegistry.h"
34 #include "ResourceError.h"
35 #include "SharedBuffer.h"
36 #include <wtf/MainThread.h>
40 CurlRequest::CurlRequest(const ResourceRequest&request, CurlRequestClient* client, bool shouldSuspend)
41 : m_request(request.isolatedCopy())
42 , m_shouldSuspend(shouldSuspend)
44 ASSERT(isMainThread());
47 resolveBlobReferences(m_request);
50 void CurlRequest::setUserPass(const String& user, const String& password)
52 ASSERT(isMainThread());
54 m_user = user.isolatedCopy();
55 m_password = password.isolatedCopy();
58 void CurlRequest::start(bool isSyncRequest)
60 // The pausing of transfer does not work with protocols, like file://.
61 // Therefore, PAUSE can not be done in didReceiveData().
62 // It means that the same logic as http:// can not be used.
63 // In the file scheme, invokeDidReceiveResponse() is done first.
64 // Then StartWithJobManager is called with completeDidReceiveResponse and start transfer with libcurl.
66 // http : didReceiveHeader => didReceiveData[PAUSE] => invokeDidReceiveResponse => (MainThread)curlDidReceiveResponse => completeDidReceiveResponse[RESUME] => didReceiveData
67 // file : invokeDidReceiveResponseForFile => (MainThread)curlDidReceiveResponse => completeDidReceiveResponse => didReceiveData
69 ASSERT(isMainThread());
71 m_isSyncRequest = isSyncRequest;
73 auto url = m_request.url().isolatedCopy();
75 if (!m_isSyncRequest) {
76 // For asynchronous, use CurlRequestScheduler. Curl processes runs on sub thread.
77 if (url.isLocalFile())
78 invokeDidReceiveResponseForFile(url);
80 startWithJobManager();
82 // For synchronous, does not use CurlRequestScheduler. Curl processes runs on main thread.
83 // curl_easy_perform blocks until the transfer is finished.
85 if (url.isLocalFile())
86 invokeDidReceiveResponseForFile(url);
89 CURLcode resultCode = m_curlHandle->perform();
90 didCompleteTransfer(resultCode);
95 void CurlRequest::startWithJobManager()
97 ASSERT(isMainThread());
99 CurlRequestScheduler::singleton().add(this);
102 void CurlRequest::cancel()
104 ASSERT(isMainThread());
106 if (isCompletedOrCancelled())
111 if (!m_isSyncRequest) {
112 auto& scheduler = CurlRequestScheduler::singleton();
114 if (needToInvokeDidCancelTransfer()) {
115 scheduler.callOnWorkerThread([protectedThis = makeRef(*this)]() {
116 protectedThis->didCancelTransfer();
119 scheduler.cancel(this);
121 if (needToInvokeDidCancelTransfer())
125 setRequestPaused(false);
126 setCallbackPaused(false);
129 void CurlRequest::suspend()
131 ASSERT(isMainThread());
133 setRequestPaused(true);
136 void CurlRequest::resume()
138 ASSERT(isMainThread());
140 setRequestPaused(false);
143 /* `this` is protected inside this method. */
144 void CurlRequest::callClient(WTF::Function<void(CurlRequestClient*)> task)
146 if (isMainThread()) {
147 if (CurlRequestClient* client = m_client)
150 callOnMainThread([protectedThis = makeRef(*this), task = WTFMove(task)]() mutable {
151 if (CurlRequestClient* client = protectedThis->m_client)
157 CURL* CurlRequest::setupTransfer()
159 auto& sslHandle = CurlContext::singleton().sslHandle();
161 m_curlHandle = std::make_unique<CurlHandle>();
163 m_curlHandle->initialize();
164 m_curlHandle->setUrl(m_request.url());
165 m_curlHandle->appendRequestHeaders(m_request.httpHeaderFields());
167 const auto& method = m_request.httpMethod();
169 m_curlHandle->enableHttpGetRequest();
170 else if (method == "POST")
171 setupPOST(m_request);
172 else if (method == "PUT")
174 else if (method == "HEAD")
175 m_curlHandle->enableHttpHeadRequest();
177 m_curlHandle->setHttpCustomRequest(method);
181 if (!m_user.isEmpty() || !m_password.isEmpty()) {
182 m_curlHandle->enableHttpAuthentication(CURLAUTH_ANY);
183 m_curlHandle->setHttpAuthUserPass(m_user, m_password);
186 m_curlHandle->setHeaderCallbackFunction(didReceiveHeaderCallback, this);
187 m_curlHandle->setWriteCallbackFunction(didReceiveDataCallback, this);
189 m_curlHandle->enableShareHandle();
190 m_curlHandle->enableAllowedProtocols();
191 m_curlHandle->enableAcceptEncoding();
192 m_curlHandle->enableTimeout();
194 m_curlHandle->enableProxyIfExists();
195 m_curlHandle->enableCookieJarIfExists();
197 m_curlHandle->setSslVerifyPeer(CurlHandle::VerifyPeer::Enable);
198 m_curlHandle->setSslVerifyHost(CurlHandle::VerifyHost::StrictNameCheck);
200 auto sslClientCertificate = sslHandle.getSSLClientCertificate(m_request.url().host());
201 if (sslClientCertificate) {
202 m_curlHandle->setSslCert(sslClientCertificate->first.utf8().data());
203 m_curlHandle->setSslCertType("P12");
204 m_curlHandle->setSslKeyPassword(sslClientCertificate->second.utf8().data());
207 if (sslHandle.shouldIgnoreSSLErrors())
208 m_curlHandle->setSslVerifyPeer(CurlHandle::VerifyPeer::Disable);
210 m_curlHandle->setSslCtxCallbackFunction(willSetupSslCtxCallback, this);
212 m_curlHandle->setCACertPath(sslHandle.getCACertPath());
218 m_curlHandle->enableVerboseIfUsed();
219 m_curlHandle->enableStdErrIfUsed();
222 return m_curlHandle->handle();
225 CURLcode CurlRequest::willSetupSslCtx(void* sslCtx)
227 m_sslVerifier.setCurlHandle(m_curlHandle.get());
228 m_sslVerifier.setHostName(m_request.url().host());
229 m_sslVerifier.setSslCtx(sslCtx);
234 // This is called to obtain HTTP POST or PUT data.
235 // Iterate through FormData elements and upload files.
236 // Carefully respect the given buffer size and fill the rest of the data at the next calls.
238 size_t CurlRequest::willSendData(char* ptr, size_t blockSize, size_t numberOfBlocks)
240 if (isCompletedOrCancelled())
241 return CURL_READFUNC_ABORT;
243 if (!blockSize || !numberOfBlocks)
246 if (!m_formDataStream || !m_formDataStream->hasMoreElements())
249 auto sendBytes = m_formDataStream->read(ptr, blockSize, numberOfBlocks);
251 // Something went wrong so error the job.
252 return CURL_READFUNC_ABORT;
258 // This is being called for each HTTP header in the response. This includes '\r\n'
259 // for the last line of the header.
261 size_t CurlRequest::didReceiveHeader(String&& header)
263 static const auto emptyLineCRLF = "\r\n";
264 static const auto emptyLineLF = "\n";
266 if (isCompletedOrCancelled())
269 // libcurl sends all headers that libcurl received to application.
270 // So, in digest authentication, a block of response headers are received twice consecutively from libcurl.
271 // For example, when authentication succeeds, the first block is "401 Authorization", and the second block is "200 OK".
272 // Also, "100 Continue" and "200 Connection Established" do the same behavior.
273 // In this process, deletes the first block to send a correct headers to WebCore.
274 if (m_didReceiveResponse) {
275 m_didReceiveResponse = false;
276 m_response = CurlResponse { };
279 auto receiveBytes = static_cast<size_t>(header.length());
281 // The HTTP standard requires to use \r\n but for compatibility it recommends to accept also \n.
282 if ((header != emptyLineCRLF) && (header != emptyLineLF)) {
283 m_response.headers.append(WTFMove(header));
288 if (auto code = m_curlHandle->getResponseCode())
291 long httpConnectCode = 0;
292 if (auto code = m_curlHandle->getHttpConnectCode())
293 httpConnectCode = *code;
295 m_didReceiveResponse = true;
297 m_response.url = m_request.url();
298 m_response.statusCode = statusCode;
300 if (auto length = m_curlHandle->getContentLength())
301 m_response.expectedContentLength = *length;
303 if (auto port = m_curlHandle->getPrimaryPort())
304 m_response.connectPort = *port;
306 if (auto auth = m_curlHandle->getHttpAuthAvail())
307 m_response.availableHttpAuth = *auth;
309 if (auto metrics = m_curlHandle->getNetworkLoadMetrics())
310 m_networkLoadMetrics = *metrics;
312 // Response will send at didReceiveData() or didCompleteTransfer()
313 // to receive continueDidRceiveResponse() for asynchronously.
318 // called with data after all headers have been processed via headerCallback
320 size_t CurlRequest::didReceiveData(Ref<SharedBuffer>&& buffer)
322 if (isCompletedOrCancelled())
325 if (needToInvokeDidReceiveResponse()) {
326 if (!m_isSyncRequest) {
327 // For asynchronous, pause until completeDidReceiveResponse() is called.
328 setCallbackPaused(true);
329 invokeDidReceiveResponse(Action::ReceiveData);
330 return CURL_WRITEFUNC_PAUSE;
333 // For synchronous, completeDidReceiveResponse() is called in invokeDidReceiveResponse().
334 // In this case, pause is unnecessary.
335 invokeDidReceiveResponse(Action::None);
338 auto receiveBytes = buffer->size();
340 writeDataToDownloadFileIfEnabled(buffer);
343 callClient([this, buffer = WTFMove(buffer)](CurlRequestClient* client) mutable {
345 client->curlDidReceiveBuffer(WTFMove(buffer));
352 void CurlRequest::didCompleteTransfer(CURLcode result)
355 m_curlHandle = nullptr;
359 if (result == CURLE_OK) {
360 if (needToInvokeDidReceiveResponse()) {
361 // Processing of didReceiveResponse() has not been completed. (For example, HEAD method)
362 // When completeDidReceiveResponse() is called, didCompleteTransfer() will be called again.
364 m_finishedResultCode = result;
365 invokeDidReceiveResponse(Action::FinishTransfer);
367 if (auto metrics = m_curlHandle->getNetworkLoadMetrics())
368 m_networkLoadMetrics = *metrics;
371 callClient([this](CurlRequestClient* client) {
373 client->curlDidComplete();
377 auto resourceError = ResourceError::httpError(result, m_request.url());
378 if (m_sslVerifier.sslErrors())
379 resourceError.setSslErrors(m_sslVerifier.sslErrors());
382 callClient([this, error = resourceError.isolatedCopy()](CurlRequestClient* client) {
384 client->curlDidFailWithError(error);
389 void CurlRequest::didCancelTransfer()
392 cleanupDownloadFile();
395 void CurlRequest::finalizeTransfer()
397 m_formDataStream = nullptr;
399 m_curlHandle = nullptr;
402 void CurlRequest::resolveBlobReferences(ResourceRequest& request)
404 ASSERT(isMainThread());
406 auto body = request.httpBody();
407 if (!body || body->isEmpty())
410 // Resolve the blob elements so the formData can correctly report it's size.
411 RefPtr<FormData> formData = body->resolveBlobReferences();
412 request.setHTTPBody(WTFMove(formData));
415 void CurlRequest::setupPUT(ResourceRequest& request)
417 m_curlHandle->enableHttpPutRequest();
419 // Disable the Expect: 100 continue header
420 m_curlHandle->removeRequestHeader("Expect");
422 auto body = request.httpBody();
423 if (!body || body->isEmpty())
426 setupFormData(request, false);
429 void CurlRequest::setupPOST(ResourceRequest& request)
431 m_curlHandle->enableHttpPostRequest();
433 auto body = request.httpBody();
434 if (!body || body->isEmpty())
437 auto numElements = body->elements().size();
441 // Do not stream for simple POST data
442 if (numElements == 1) {
443 m_postBuffer = body->flatten();
444 if (m_postBuffer.size())
445 m_curlHandle->setPostFields(m_postBuffer.data(), m_postBuffer.size());
447 setupFormData(request, true);
450 void CurlRequest::setupFormData(ResourceRequest& request, bool isPostRequest)
452 static auto maxCurlOffT = CurlHandle::maxCurlOffT();
454 // Obtain the total size of the form data
456 bool chunkedTransfer = false;
457 auto elements = request.httpBody()->elements();
459 for (auto element : elements) {
460 if (element.m_type == FormDataElement::Type::EncodedFile) {
461 long long fileSizeResult;
462 if (FileSystem::getFileSize(element.m_filename, fileSizeResult)) {
463 if (fileSizeResult > maxCurlOffT) {
464 // File size is too big for specifying it to cURL
465 chunkedTransfer = true;
468 size += fileSizeResult;
470 chunkedTransfer = true;
474 size += element.m_data.size();
477 // cURL guesses that we want chunked encoding as long as we specify the header
479 m_curlHandle->appendRequestHeader("Transfer-Encoding: chunked");
482 m_curlHandle->setPostFieldLarge(size);
484 m_curlHandle->setInFileSizeLarge(size);
487 m_formDataStream = std::make_unique<FormDataStream>();
488 m_formDataStream->setHTTPBody(request.httpBody());
490 m_curlHandle->setReadCallbackFunction(willSendDataCallback, this);
493 void CurlRequest::invokeDidReceiveResponseForFile(URL& url)
495 // Since the code in didReceiveHeader() will not have run for local files
496 // the code to set the URL and fire didReceiveResponse is never run,
497 // which means the ResourceLoader's response does not contain the URL.
498 // Run the code here for local files to resolve the issue.
500 ASSERT(isMainThread());
501 ASSERT(url.isLocalFile());
503 m_response.url = url;
504 m_response.statusCode = 200;
506 // Determine the MIME type based on the path.
507 m_response.headers.append(String("Content-Type: " + MIMETypeRegistry::getMIMETypeForPath(m_response.url.path())));
509 if (!m_isSyncRequest) {
510 // DidReceiveResponse must not be called immediately
511 CurlRequestScheduler::singleton().callOnWorkerThread([protectedThis = makeRef(*this)]() {
512 protectedThis->invokeDidReceiveResponse(Action::StartTransfer);
515 // For synchronous, completeDidReceiveResponse() is called in platformContinueSynchronousDidReceiveResponse().
516 invokeDidReceiveResponse(Action::None);
520 void CurlRequest::invokeDidReceiveResponse(Action behaviorAfterInvoke)
522 ASSERT(!m_didNotifyResponse);
524 m_didNotifyResponse = true;
525 m_actionAfterInvoke = behaviorAfterInvoke;
527 callClient([this, response = m_response.isolatedCopy()](CurlRequestClient* client) {
529 client->curlDidReceiveResponse(response);
533 void CurlRequest::completeDidReceiveResponse()
535 ASSERT(isMainThread());
536 ASSERT(m_didNotifyResponse);
537 ASSERT(!m_didReturnFromNotify);
542 if (m_actionAfterInvoke != Action::StartTransfer && isCompleted())
545 m_didReturnFromNotify = true;
547 if (m_actionAfterInvoke == Action::ReceiveData) {
549 setCallbackPaused(false);
550 } else if (m_actionAfterInvoke == Action::StartTransfer) {
551 // Start transfer for file scheme
552 startWithJobManager();
553 } else if (m_actionAfterInvoke == Action::FinishTransfer) {
554 if (!m_isSyncRequest) {
555 CurlRequestScheduler::singleton().callOnWorkerThread([protectedThis = makeRef(*this), finishedResultCode = m_finishedResultCode]() {
556 protectedThis->didCompleteTransfer(finishedResultCode);
559 didCompleteTransfer(m_finishedResultCode);
563 void CurlRequest::setRequestPaused(bool paused)
565 auto wasPaused = isPaused();
567 m_isPausedOfRequest = paused;
569 if (isPaused() == wasPaused)
572 pausedStatusChanged();
575 void CurlRequest::setCallbackPaused(bool paused)
577 auto wasPaused = isPaused();
579 m_isPausedOfCallback = paused;
581 if (isPaused() == wasPaused)
584 // In this case, PAUSE will be executed within didReceiveData(). Change pause state and return.
588 pausedStatusChanged();
591 void CurlRequest::pausedStatusChanged()
593 if (isCompletedOrCancelled())
596 if (!m_isSyncRequest && isMainThread()) {
597 CurlRequestScheduler::singleton().callOnWorkerThread([protectedThis = makeRef(*this), paused = isPaused()]() {
598 if (protectedThis->isCompletedOrCancelled())
601 auto error = protectedThis->m_curlHandle->pause(paused ? CURLPAUSE_ALL : CURLPAUSE_CONT);
602 if ((error != CURLE_OK) && !paused) {
603 // Restarting the handle has failed so just cancel it.
604 callOnMainThread([protectedThis = makeRef(protectedThis.get())]() {
605 protectedThis->cancel();
610 auto error = m_curlHandle->pause(isPaused() ? CURLPAUSE_ALL : CURLPAUSE_CONT);
611 if ((error != CURLE_OK) && !isPaused())
616 void CurlRequest::enableDownloadToFile()
618 LockHolder locker(m_downloadMutex);
619 m_isEnabledDownloadToFile = true;
622 const String& CurlRequest::getDownloadedFilePath()
624 LockHolder locker(m_downloadMutex);
625 return m_downloadFilePath;
628 void CurlRequest::writeDataToDownloadFileIfEnabled(const SharedBuffer& buffer)
631 LockHolder locker(m_downloadMutex);
633 if (!m_isEnabledDownloadToFile)
636 if (m_downloadFilePath.isEmpty())
637 m_downloadFilePath = FileSystem::openTemporaryFile("download", m_downloadFileHandle);
640 if (m_downloadFileHandle != FileSystem::invalidPlatformFileHandle)
641 FileSystem::writeToFile(m_downloadFileHandle, buffer.data(), buffer.size());
644 void CurlRequest::closeDownloadFile()
646 LockHolder locker(m_downloadMutex);
648 if (m_downloadFileHandle == FileSystem::invalidPlatformFileHandle)
651 FileSystem::closeFile(m_downloadFileHandle);
652 m_downloadFileHandle = FileSystem::invalidPlatformFileHandle;
655 void CurlRequest::cleanupDownloadFile()
657 LockHolder locker(m_downloadMutex);
659 if (!m_downloadFilePath.isEmpty()) {
660 FileSystem::deleteFile(m_downloadFilePath);
661 m_downloadFilePath = String();
665 CURLcode CurlRequest::willSetupSslCtxCallback(CURL*, void* sslCtx, void* userData)
667 return static_cast<CurlRequest*>(userData)->willSetupSslCtx(sslCtx);
670 size_t CurlRequest::willSendDataCallback(char* ptr, size_t blockSize, size_t numberOfBlocks, void* userData)
672 return static_cast<CurlRequest*>(userData)->willSendData(ptr, blockSize, numberOfBlocks);
675 size_t CurlRequest::didReceiveHeaderCallback(char* ptr, size_t blockSize, size_t numberOfBlocks, void* userData)
677 return static_cast<CurlRequest*>(userData)->didReceiveHeader(String(ptr, blockSize * numberOfBlocks));
680 size_t CurlRequest::didReceiveDataCallback(char* ptr, size_t blockSize, size_t numberOfBlocks, void* userData)
682 return static_cast<CurlRequest*>(userData)->didReceiveData(SharedBuffer::create(ptr, blockSize * numberOfBlocks));