2 * Copyright (C) 2017 Sony Interactive Entertainment Inc.
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
13 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23 * THE POSSIBILITY OF SUCH DAMAGE.
27 #include "CurlRequest.h"
31 #include "CurlRequestClient.h"
32 #include "CurlRequestScheduler.h"
33 #include "MIMETypeRegistry.h"
34 #include "ResourceError.h"
35 #include "SharedBuffer.h"
36 #include <wtf/MainThread.h>
40 CurlRequest::CurlRequest(const ResourceRequest&request, CurlRequestClient* client, bool shouldSuspend)
41 : m_request(request.isolatedCopy())
42 , m_shouldSuspend(shouldSuspend)
43 , m_formDataStream(m_request.httpBody())
45 ASSERT(isMainThread());
50 void CurlRequest::setUserPass(const String& user, const String& password)
52 ASSERT(isMainThread());
54 m_user = user.isolatedCopy();
55 m_password = password.isolatedCopy();
58 void CurlRequest::start(bool isSyncRequest)
60 // The pausing of transfer does not work with protocols, like file://.
61 // Therefore, PAUSE can not be done in didReceiveData().
62 // It means that the same logic as http:// can not be used.
63 // In the file scheme, invokeDidReceiveResponse() is done first.
64 // Then StartWithJobManager is called with completeDidReceiveResponse and start transfer with libcurl.
66 // http : didReceiveHeader => didReceiveData[PAUSE] => invokeDidReceiveResponse => (MainThread)curlDidReceiveResponse => completeDidReceiveResponse[RESUME] => didReceiveData
67 // file : invokeDidReceiveResponseForFile => (MainThread)curlDidReceiveResponse => completeDidReceiveResponse => didReceiveData
69 ASSERT(isMainThread());
71 m_isSyncRequest = isSyncRequest;
73 auto url = m_request.url().isolatedCopy();
75 if (!m_isSyncRequest) {
76 // For asynchronous, use CurlRequestScheduler. Curl processes runs on sub thread.
77 if (url.isLocalFile())
78 invokeDidReceiveResponseForFile(url);
80 startWithJobManager();
82 // For synchronous, does not use CurlRequestScheduler. Curl processes runs on main thread.
83 // curl_easy_perform blocks until the transfer is finished.
85 if (url.isLocalFile())
86 invokeDidReceiveResponseForFile(url);
89 CURLcode resultCode = m_curlHandle->perform();
90 didCompleteTransfer(resultCode);
95 void CurlRequest::startWithJobManager()
97 ASSERT(isMainThread());
99 CurlRequestScheduler::singleton().add(this);
102 void CurlRequest::cancel()
104 ASSERT(isMainThread());
106 if (isCompletedOrCancelled())
111 if (!m_isSyncRequest) {
112 auto& scheduler = CurlRequestScheduler::singleton();
114 if (needToInvokeDidCancelTransfer()) {
115 scheduler.callOnWorkerThread([protectedThis = makeRef(*this)]() {
116 protectedThis->didCancelTransfer();
119 scheduler.cancel(this);
121 if (needToInvokeDidCancelTransfer())
125 setRequestPaused(false);
126 setCallbackPaused(false);
129 void CurlRequest::suspend()
131 ASSERT(isMainThread());
133 setRequestPaused(true);
136 void CurlRequest::resume()
138 ASSERT(isMainThread());
140 setRequestPaused(false);
143 /* `this` is protected inside this method. */
144 void CurlRequest::callClient(WTF::Function<void(CurlRequestClient*)> task)
146 if (isMainThread()) {
147 if (CurlRequestClient* client = m_client)
150 callOnMainThread([protectedThis = makeRef(*this), task = WTFMove(task)]() mutable {
151 if (CurlRequestClient* client = protectedThis->m_client)
157 CURL* CurlRequest::setupTransfer()
159 auto& sslHandle = CurlContext::singleton().sslHandle();
161 m_curlHandle = std::make_unique<CurlHandle>();
163 m_curlHandle->initialize();
164 m_curlHandle->setUrl(m_request.url());
165 m_curlHandle->appendRequestHeaders(m_request.httpHeaderFields());
167 const auto& method = m_request.httpMethod();
169 m_curlHandle->enableHttpGetRequest();
170 else if (method == "POST")
171 setupPOST(m_request);
172 else if (method == "PUT")
174 else if (method == "HEAD")
175 m_curlHandle->enableHttpHeadRequest();
177 m_curlHandle->setHttpCustomRequest(method);
181 if (!m_user.isEmpty() || !m_password.isEmpty()) {
182 m_curlHandle->enableHttpAuthentication(CURLAUTH_ANY);
183 m_curlHandle->setHttpAuthUserPass(m_user, m_password);
186 m_curlHandle->setHeaderCallbackFunction(didReceiveHeaderCallback, this);
187 m_curlHandle->setWriteCallbackFunction(didReceiveDataCallback, this);
189 m_curlHandle->enableShareHandle();
190 m_curlHandle->enableAllowedProtocols();
191 m_curlHandle->enableAcceptEncoding();
192 m_curlHandle->enableTimeout();
194 m_curlHandle->enableProxyIfExists();
195 m_curlHandle->enableCookieJarIfExists();
197 m_curlHandle->setSslVerifyPeer(CurlHandle::VerifyPeer::Enable);
198 m_curlHandle->setSslVerifyHost(CurlHandle::VerifyHost::StrictNameCheck);
200 auto sslClientCertificate = sslHandle.getSSLClientCertificate(m_request.url().host());
201 if (sslClientCertificate) {
202 m_curlHandle->setSslCert(sslClientCertificate->first.utf8().data());
203 m_curlHandle->setSslCertType("P12");
204 m_curlHandle->setSslKeyPassword(sslClientCertificate->second.utf8().data());
207 if (sslHandle.shouldIgnoreSSLErrors())
208 m_curlHandle->setSslVerifyPeer(CurlHandle::VerifyPeer::Disable);
210 m_curlHandle->setSslCtxCallbackFunction(willSetupSslCtxCallback, this);
212 m_curlHandle->setCACertPath(sslHandle.getCACertPath());
218 m_curlHandle->enableVerboseIfUsed();
219 m_curlHandle->enableStdErrIfUsed();
222 return m_curlHandle->handle();
225 CURLcode CurlRequest::willSetupSslCtx(void* sslCtx)
227 m_sslVerifier.setCurlHandle(m_curlHandle.get());
228 m_sslVerifier.setHostName(m_request.url().host());
229 m_sslVerifier.setSslCtx(sslCtx);
234 // This is called to obtain HTTP POST or PUT data.
235 // Iterate through FormData elements and upload files.
236 // Carefully respect the given buffer size and fill the rest of the data at the next calls.
238 size_t CurlRequest::willSendData(char* buffer, size_t blockSize, size_t numberOfBlocks)
240 if (isCompletedOrCancelled())
241 return CURL_READFUNC_ABORT;
243 if (!blockSize || !numberOfBlocks)
244 return CURL_READFUNC_ABORT;
246 // Check for overflow.
247 if (blockSize > (std::numeric_limits<size_t>::max() / numberOfBlocks))
248 return CURL_READFUNC_ABORT;
250 size_t bufferSize = blockSize * numberOfBlocks;
251 auto sendBytes = m_formDataStream.read(buffer, bufferSize);
253 // Something went wrong so error the job.
254 return CURL_READFUNC_ABORT;
260 // This is being called for each HTTP header in the response. This includes '\r\n'
261 // for the last line of the header.
263 size_t CurlRequest::didReceiveHeader(String&& header)
265 static const auto emptyLineCRLF = "\r\n";
266 static const auto emptyLineLF = "\n";
268 if (isCompletedOrCancelled())
271 // libcurl sends all headers that libcurl received to application.
272 // So, in digest authentication, a block of response headers are received twice consecutively from libcurl.
273 // For example, when authentication succeeds, the first block is "401 Authorization", and the second block is "200 OK".
274 // Also, "100 Continue" and "200 Connection Established" do the same behavior.
275 // In this process, deletes the first block to send a correct headers to WebCore.
276 if (m_didReceiveResponse) {
277 m_didReceiveResponse = false;
278 m_response = CurlResponse { };
281 auto receiveBytes = static_cast<size_t>(header.length());
283 // The HTTP standard requires to use \r\n but for compatibility it recommends to accept also \n.
284 if ((header != emptyLineCRLF) && (header != emptyLineLF)) {
285 m_response.headers.append(WTFMove(header));
290 if (auto code = m_curlHandle->getResponseCode())
293 long httpConnectCode = 0;
294 if (auto code = m_curlHandle->getHttpConnectCode())
295 httpConnectCode = *code;
297 m_didReceiveResponse = true;
299 m_response.url = m_request.url();
300 m_response.statusCode = statusCode;
302 if (auto length = m_curlHandle->getContentLength())
303 m_response.expectedContentLength = *length;
305 if (auto port = m_curlHandle->getPrimaryPort())
306 m_response.connectPort = *port;
308 if (auto auth = m_curlHandle->getHttpAuthAvail())
309 m_response.availableHttpAuth = *auth;
311 if (auto version = m_curlHandle->getHttpVersion())
312 m_response.httpVersion = *version;
314 if (auto metrics = m_curlHandle->getNetworkLoadMetrics())
315 m_networkLoadMetrics = *metrics;
317 // Response will send at didReceiveData() or didCompleteTransfer()
318 // to receive continueDidRceiveResponse() for asynchronously.
323 // called with data after all headers have been processed via headerCallback
325 size_t CurlRequest::didReceiveData(Ref<SharedBuffer>&& buffer)
327 if (isCompletedOrCancelled())
330 if (needToInvokeDidReceiveResponse()) {
331 if (!m_isSyncRequest) {
332 // For asynchronous, pause until completeDidReceiveResponse() is called.
333 setCallbackPaused(true);
334 invokeDidReceiveResponse(Action::ReceiveData);
335 return CURL_WRITEFUNC_PAUSE;
338 // For synchronous, completeDidReceiveResponse() is called in invokeDidReceiveResponse().
339 // In this case, pause is unnecessary.
340 invokeDidReceiveResponse(Action::None);
343 auto receiveBytes = buffer->size();
345 writeDataToDownloadFileIfEnabled(buffer);
348 callClient([this, buffer = WTFMove(buffer)](CurlRequestClient* client) mutable {
350 client->curlDidReceiveBuffer(WTFMove(buffer));
357 void CurlRequest::didCompleteTransfer(CURLcode result)
360 m_curlHandle = nullptr;
364 if (result == CURLE_OK) {
365 if (needToInvokeDidReceiveResponse()) {
366 // Processing of didReceiveResponse() has not been completed. (For example, HEAD method)
367 // When completeDidReceiveResponse() is called, didCompleteTransfer() will be called again.
369 m_finishedResultCode = result;
370 invokeDidReceiveResponse(Action::FinishTransfer);
372 if (auto metrics = m_curlHandle->getNetworkLoadMetrics())
373 m_networkLoadMetrics = *metrics;
376 callClient([this](CurlRequestClient* client) {
378 client->curlDidComplete();
382 auto resourceError = ResourceError::httpError(result, m_request.url());
383 if (m_sslVerifier.sslErrors())
384 resourceError.setSslErrors(m_sslVerifier.sslErrors());
387 callClient([this, error = resourceError.isolatedCopy()](CurlRequestClient* client) {
389 client->curlDidFailWithError(error);
394 void CurlRequest::didCancelTransfer()
397 cleanupDownloadFile();
400 void CurlRequest::finalizeTransfer()
403 m_formDataStream.clean();
404 m_curlHandle = nullptr;
407 void CurlRequest::setupPUT(ResourceRequest& request)
409 m_curlHandle->enableHttpPutRequest();
411 // Disable the Expect: 100 continue header
412 m_curlHandle->removeRequestHeader("Expect");
414 auto elementSize = m_formDataStream.elementSize();
421 void CurlRequest::setupPOST(ResourceRequest& request)
423 m_curlHandle->enableHttpPostRequest();
425 auto elementSize = m_formDataStream.elementSize();
429 // Do not stream for simple POST data
430 if (elementSize == 1) {
431 auto postData = m_formDataStream.getPostData();
432 if (postData && postData->size())
433 m_curlHandle->setPostFields(postData->data(), postData->size());
435 setupSendData(false);
438 void CurlRequest::setupSendData(bool forPutMethod)
440 // curl guesses that we want chunked encoding as long as we specify the header
441 if (m_formDataStream.shouldUseChunkTransfer())
442 m_curlHandle->appendRequestHeader("Transfer-Encoding: chunked");
445 m_curlHandle->setInFileSizeLarge(static_cast<curl_off_t>(m_formDataStream.totalSize()));
447 m_curlHandle->setPostFieldLarge(static_cast<curl_off_t>(m_formDataStream.totalSize()));
450 m_curlHandle->setReadCallbackFunction(willSendDataCallback, this);
453 void CurlRequest::invokeDidReceiveResponseForFile(URL& url)
455 // Since the code in didReceiveHeader() will not have run for local files
456 // the code to set the URL and fire didReceiveResponse is never run,
457 // which means the ResourceLoader's response does not contain the URL.
458 // Run the code here for local files to resolve the issue.
460 ASSERT(isMainThread());
461 ASSERT(url.isLocalFile());
463 m_response.url = url;
464 m_response.statusCode = 200;
466 // Determine the MIME type based on the path.
467 m_response.headers.append(String("Content-Type: " + MIMETypeRegistry::getMIMETypeForPath(m_response.url.path())));
469 if (!m_isSyncRequest) {
470 // DidReceiveResponse must not be called immediately
471 CurlRequestScheduler::singleton().callOnWorkerThread([protectedThis = makeRef(*this)]() {
472 protectedThis->invokeDidReceiveResponse(Action::StartTransfer);
475 // For synchronous, completeDidReceiveResponse() is called in platformContinueSynchronousDidReceiveResponse().
476 invokeDidReceiveResponse(Action::None);
480 void CurlRequest::invokeDidReceiveResponse(Action behaviorAfterInvoke)
482 ASSERT(!m_didNotifyResponse);
484 m_didNotifyResponse = true;
485 m_actionAfterInvoke = behaviorAfterInvoke;
487 callClient([this, response = m_response.isolatedCopy()](CurlRequestClient* client) {
489 client->curlDidReceiveResponse(response);
493 void CurlRequest::completeDidReceiveResponse()
495 ASSERT(isMainThread());
496 ASSERT(m_didNotifyResponse);
497 ASSERT(!m_didReturnFromNotify);
502 if (m_actionAfterInvoke != Action::StartTransfer && isCompleted())
505 m_didReturnFromNotify = true;
507 if (m_actionAfterInvoke == Action::ReceiveData) {
509 setCallbackPaused(false);
510 } else if (m_actionAfterInvoke == Action::StartTransfer) {
511 // Start transfer for file scheme
512 startWithJobManager();
513 } else if (m_actionAfterInvoke == Action::FinishTransfer) {
514 if (!m_isSyncRequest) {
515 CurlRequestScheduler::singleton().callOnWorkerThread([protectedThis = makeRef(*this), finishedResultCode = m_finishedResultCode]() {
516 protectedThis->didCompleteTransfer(finishedResultCode);
519 didCompleteTransfer(m_finishedResultCode);
523 void CurlRequest::setRequestPaused(bool paused)
525 auto wasPaused = isPaused();
527 m_isPausedOfRequest = paused;
529 if (isPaused() == wasPaused)
532 pausedStatusChanged();
535 void CurlRequest::setCallbackPaused(bool paused)
537 auto wasPaused = isPaused();
539 m_isPausedOfCallback = paused;
541 if (isPaused() == wasPaused)
544 // In this case, PAUSE will be executed within didReceiveData(). Change pause state and return.
548 pausedStatusChanged();
551 void CurlRequest::pausedStatusChanged()
553 if (isCompletedOrCancelled())
556 if (!m_isSyncRequest && isMainThread()) {
557 CurlRequestScheduler::singleton().callOnWorkerThread([protectedThis = makeRef(*this), paused = isPaused()]() {
558 if (protectedThis->isCompletedOrCancelled())
561 auto error = protectedThis->m_curlHandle->pause(paused ? CURLPAUSE_ALL : CURLPAUSE_CONT);
562 if ((error != CURLE_OK) && !paused) {
563 // Restarting the handle has failed so just cancel it.
564 callOnMainThread([protectedThis = makeRef(protectedThis.get())]() {
565 protectedThis->cancel();
570 auto error = m_curlHandle->pause(isPaused() ? CURLPAUSE_ALL : CURLPAUSE_CONT);
571 if ((error != CURLE_OK) && !isPaused())
576 void CurlRequest::enableDownloadToFile()
578 LockHolder locker(m_downloadMutex);
579 m_isEnabledDownloadToFile = true;
582 const String& CurlRequest::getDownloadedFilePath()
584 LockHolder locker(m_downloadMutex);
585 return m_downloadFilePath;
588 void CurlRequest::writeDataToDownloadFileIfEnabled(const SharedBuffer& buffer)
591 LockHolder locker(m_downloadMutex);
593 if (!m_isEnabledDownloadToFile)
596 if (m_downloadFilePath.isEmpty())
597 m_downloadFilePath = FileSystem::openTemporaryFile("download", m_downloadFileHandle);
600 if (m_downloadFileHandle != FileSystem::invalidPlatformFileHandle)
601 FileSystem::writeToFile(m_downloadFileHandle, buffer.data(), buffer.size());
604 void CurlRequest::closeDownloadFile()
606 LockHolder locker(m_downloadMutex);
608 if (m_downloadFileHandle == FileSystem::invalidPlatformFileHandle)
611 FileSystem::closeFile(m_downloadFileHandle);
612 m_downloadFileHandle = FileSystem::invalidPlatformFileHandle;
615 void CurlRequest::cleanupDownloadFile()
617 LockHolder locker(m_downloadMutex);
619 if (!m_downloadFilePath.isEmpty()) {
620 FileSystem::deleteFile(m_downloadFilePath);
621 m_downloadFilePath = String();
625 CURLcode CurlRequest::willSetupSslCtxCallback(CURL*, void* sslCtx, void* userData)
627 return static_cast<CurlRequest*>(userData)->willSetupSslCtx(sslCtx);
630 size_t CurlRequest::willSendDataCallback(char* ptr, size_t blockSize, size_t numberOfBlocks, void* userData)
632 return static_cast<CurlRequest*>(userData)->willSendData(ptr, blockSize, numberOfBlocks);
635 size_t CurlRequest::didReceiveHeaderCallback(char* ptr, size_t blockSize, size_t numberOfBlocks, void* userData)
637 return static_cast<CurlRequest*>(userData)->didReceiveHeader(String(ptr, blockSize * numberOfBlocks));
640 size_t CurlRequest::didReceiveDataCallback(char* ptr, size_t blockSize, size_t numberOfBlocks, void* userData)
642 return static_cast<CurlRequest*>(userData)->didReceiveData(SharedBuffer::create(ptr, blockSize * numberOfBlocks));