25a3da12cf946f44d6430770d83e8c3532f90044
[WebKit-https.git] / Source / WebCore / platform / network / curl / CurlRequest.cpp
1 /*
2  * Copyright (C) 2017 Sony Interactive Entertainment Inc.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  * 
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "CurlRequest.h"
28
29 #if USE(CURL)
30
31 #include "CurlRequestClient.h"
32 #include "CurlRequestScheduler.h"
33 #include "MIMETypeRegistry.h"
34 #include "ResourceError.h"
35 #include "SharedBuffer.h"
36 #include <wtf/MainThread.h>
37
38 namespace WebCore {
39
40 CurlRequest::CurlRequest(const ResourceRequest&request, CurlRequestClient* client, bool shouldSuspend)
41     : m_request(request.isolatedCopy())
42     , m_shouldSuspend(shouldSuspend)
43     , m_formDataStream(m_request.httpBody())
44 {
45     ASSERT(isMainThread());
46
47     setClient(client);
48 }
49
50 void CurlRequest::setUserPass(const String& user, const String& password)
51 {
52     ASSERT(isMainThread());
53
54     m_user = user.isolatedCopy();
55     m_password = password.isolatedCopy();
56 }
57
58 void CurlRequest::start(bool isSyncRequest)
59 {
60     // The pausing of transfer does not work with protocols, like file://.
61     // Therefore, PAUSE can not be done in didReceiveData().
62     // It means that the same logic as http:// can not be used.
63     // In the file scheme, invokeDidReceiveResponse() is done first. 
64     // Then StartWithJobManager is called with completeDidReceiveResponse and start transfer with libcurl.
65
66     // http : didReceiveHeader => didReceiveData[PAUSE] => invokeDidReceiveResponse => (MainThread)curlDidReceiveResponse => completeDidReceiveResponse[RESUME] => didReceiveData
67     // file : invokeDidReceiveResponseForFile => (MainThread)curlDidReceiveResponse => completeDidReceiveResponse => didReceiveData
68
69     ASSERT(isMainThread());
70
71     m_isSyncRequest = isSyncRequest;
72
73     auto url = m_request.url().isolatedCopy();
74
75     if (!m_isSyncRequest) {
76         // For asynchronous, use CurlRequestScheduler. Curl processes runs on sub thread.
77         if (url.isLocalFile())
78             invokeDidReceiveResponseForFile(url);
79         else
80             startWithJobManager();
81     } else {
82         // For synchronous, does not use CurlRequestScheduler. Curl processes runs on main thread.
83         // curl_easy_perform blocks until the transfer is finished.
84         retain();
85         if (url.isLocalFile())
86             invokeDidReceiveResponseForFile(url);
87
88         setupTransfer();
89         CURLcode resultCode = m_curlHandle->perform();
90         didCompleteTransfer(resultCode);
91         release();
92     }
93 }
94
95 void CurlRequest::startWithJobManager()
96 {
97     ASSERT(isMainThread());
98
99     CurlRequestScheduler::singleton().add(this);
100 }
101
102 void CurlRequest::cancel()
103 {
104     ASSERT(isMainThread());
105
106     if (isCompletedOrCancelled())
107         return;
108
109     m_cancelled = true;
110
111     if (!m_isSyncRequest) {
112         auto& scheduler = CurlRequestScheduler::singleton();
113
114         if (needToInvokeDidCancelTransfer()) {
115             scheduler.callOnWorkerThread([protectedThis = makeRef(*this)]() {
116                 protectedThis->didCancelTransfer();
117             });
118         } else
119             scheduler.cancel(this);
120     } else {
121         if (needToInvokeDidCancelTransfer())
122             didCancelTransfer();
123     }
124
125     setRequestPaused(false);
126     setCallbackPaused(false);
127 }
128
129 void CurlRequest::suspend()
130 {
131     ASSERT(isMainThread());
132
133     setRequestPaused(true);
134 }
135
136 void CurlRequest::resume()
137 {
138     ASSERT(isMainThread());
139
140     setRequestPaused(false);
141 }
142
143 /* `this` is protected inside this method. */
144 void CurlRequest::callClient(WTF::Function<void(CurlRequestClient*)> task)
145 {
146     if (isMainThread()) {
147         if (CurlRequestClient* client = m_client)
148             task(client);
149     } else {
150         callOnMainThread([protectedThis = makeRef(*this), task = WTFMove(task)]() mutable {
151             if (CurlRequestClient* client = protectedThis->m_client)
152                 task(client);
153         });
154     }
155 }
156
157 CURL* CurlRequest::setupTransfer()
158 {
159     auto& sslHandle = CurlContext::singleton().sslHandle();
160
161     m_curlHandle = std::make_unique<CurlHandle>();
162
163     m_curlHandle->initialize();
164     m_curlHandle->setUrl(m_request.url());
165     m_curlHandle->appendRequestHeaders(m_request.httpHeaderFields());
166
167     const auto& method = m_request.httpMethod();
168     if (method == "GET")
169         m_curlHandle->enableHttpGetRequest();
170     else if (method == "POST")
171         setupPOST(m_request);
172     else if (method == "PUT")
173         setupPUT(m_request);
174     else if (method == "HEAD")
175         m_curlHandle->enableHttpHeadRequest();
176     else {
177         m_curlHandle->setHttpCustomRequest(method);
178         setupPUT(m_request);
179     }
180
181     if (!m_user.isEmpty() || !m_password.isEmpty()) {
182         m_curlHandle->enableHttpAuthentication(CURLAUTH_ANY);
183         m_curlHandle->setHttpAuthUserPass(m_user, m_password);
184     }
185
186     m_curlHandle->setHeaderCallbackFunction(didReceiveHeaderCallback, this);
187     m_curlHandle->setWriteCallbackFunction(didReceiveDataCallback, this);
188
189     m_curlHandle->enableShareHandle();
190     m_curlHandle->enableAllowedProtocols();
191     m_curlHandle->enableAcceptEncoding();
192     m_curlHandle->enableTimeout();
193
194     m_curlHandle->enableProxyIfExists();
195     m_curlHandle->enableCookieJarIfExists();
196
197     m_curlHandle->setSslVerifyPeer(CurlHandle::VerifyPeer::Enable);
198     m_curlHandle->setSslVerifyHost(CurlHandle::VerifyHost::StrictNameCheck);
199
200     auto sslClientCertificate = sslHandle.getSSLClientCertificate(m_request.url().host());
201     if (sslClientCertificate) {
202         m_curlHandle->setSslCert(sslClientCertificate->first.utf8().data());
203         m_curlHandle->setSslCertType("P12");
204         m_curlHandle->setSslKeyPassword(sslClientCertificate->second.utf8().data());
205     }
206
207     if (sslHandle.shouldIgnoreSSLErrors())
208         m_curlHandle->setSslVerifyPeer(CurlHandle::VerifyPeer::Disable);
209     else
210         m_curlHandle->setSslCtxCallbackFunction(willSetupSslCtxCallback, this);
211
212     m_curlHandle->setCACertPath(sslHandle.getCACertPath());
213
214     if (m_shouldSuspend)
215         suspend();
216
217 #ifndef NDEBUG
218     m_curlHandle->enableVerboseIfUsed();
219     m_curlHandle->enableStdErrIfUsed();
220 #endif
221
222     return m_curlHandle->handle();
223 }
224
225 CURLcode CurlRequest::willSetupSslCtx(void* sslCtx)
226 {
227     m_sslVerifier.setCurlHandle(m_curlHandle.get());
228     m_sslVerifier.setHostName(m_request.url().host());
229     m_sslVerifier.setSslCtx(sslCtx);
230
231     return CURLE_OK;
232 }
233
234 // This is called to obtain HTTP POST or PUT data.
235 // Iterate through FormData elements and upload files.
236 // Carefully respect the given buffer size and fill the rest of the data at the next calls.
237
238 size_t CurlRequest::willSendData(char* buffer, size_t blockSize, size_t numberOfBlocks)
239 {
240     if (isCompletedOrCancelled())
241         return CURL_READFUNC_ABORT;
242
243     if (!blockSize || !numberOfBlocks)
244         return CURL_READFUNC_ABORT;
245
246     // Check for overflow.
247     if (blockSize > (std::numeric_limits<size_t>::max() / numberOfBlocks))
248         return CURL_READFUNC_ABORT;
249
250     size_t bufferSize = blockSize * numberOfBlocks;
251     auto sendBytes = m_formDataStream.read(buffer, bufferSize);
252     if (!sendBytes) {
253         // Something went wrong so error the job.
254         return CURL_READFUNC_ABORT;
255     }
256
257     return *sendBytes;
258 }
259
260 // This is being called for each HTTP header in the response. This includes '\r\n'
261 // for the last line of the header.
262
263 size_t CurlRequest::didReceiveHeader(String&& header)
264 {
265     static const auto emptyLineCRLF = "\r\n";
266     static const auto emptyLineLF = "\n";
267
268     if (isCompletedOrCancelled())
269         return 0;
270
271     // libcurl sends all headers that libcurl received to application.
272     // So, in digest authentication, a block of response headers are received twice consecutively from libcurl.
273     // For example, when authentication succeeds, the first block is "401 Authorization", and the second block is "200 OK".
274     // Also, "100 Continue" and "200 Connection Established" do the same behavior.
275     // In this process, deletes the first block to send a correct headers to WebCore.
276     if (m_didReceiveResponse) {
277         m_didReceiveResponse = false;
278         m_response = CurlResponse { };
279     }
280
281     auto receiveBytes = static_cast<size_t>(header.length());
282
283     // The HTTP standard requires to use \r\n but for compatibility it recommends to accept also \n.
284     if ((header != emptyLineCRLF) && (header != emptyLineLF)) {
285         m_response.headers.append(WTFMove(header));
286         return receiveBytes;
287     }
288
289     long statusCode = 0;
290     if (auto code = m_curlHandle->getResponseCode())
291         statusCode = *code;
292
293     long httpConnectCode = 0;
294     if (auto code = m_curlHandle->getHttpConnectCode())
295         httpConnectCode = *code;
296
297     m_didReceiveResponse = true;
298
299     m_response.url = m_request.url();
300     m_response.statusCode = statusCode;
301
302     if (auto length = m_curlHandle->getContentLength())
303         m_response.expectedContentLength = *length;
304
305     if (auto port = m_curlHandle->getPrimaryPort())
306         m_response.connectPort = *port;
307
308     if (auto auth = m_curlHandle->getHttpAuthAvail())
309         m_response.availableHttpAuth = *auth;
310
311     if (auto version = m_curlHandle->getHttpVersion())
312         m_response.httpVersion = *version;
313
314     if (auto metrics = m_curlHandle->getNetworkLoadMetrics())
315         m_networkLoadMetrics = *metrics;
316
317     // Response will send at didReceiveData() or didCompleteTransfer()
318     // to receive continueDidRceiveResponse() for asynchronously.
319
320     return receiveBytes;
321 }
322
323 // called with data after all headers have been processed via headerCallback
324
325 size_t CurlRequest::didReceiveData(Ref<SharedBuffer>&& buffer)
326 {
327     if (isCompletedOrCancelled())
328         return 0;
329
330     if (needToInvokeDidReceiveResponse()) {
331         if (!m_isSyncRequest) {
332             // For asynchronous, pause until completeDidReceiveResponse() is called.
333             setCallbackPaused(true);
334             invokeDidReceiveResponse(Action::ReceiveData);
335             return CURL_WRITEFUNC_PAUSE;
336         }
337
338         // For synchronous, completeDidReceiveResponse() is called in invokeDidReceiveResponse().
339         // In this case, pause is unnecessary.
340         invokeDidReceiveResponse(Action::None);
341     }
342
343     auto receiveBytes = buffer->size();
344
345     writeDataToDownloadFileIfEnabled(buffer);
346
347     if (receiveBytes) {
348         callClient([this, buffer = WTFMove(buffer)](CurlRequestClient* client) mutable {
349             if (client)
350                 client->curlDidReceiveBuffer(WTFMove(buffer));
351         });
352     }
353
354     return receiveBytes;
355 }
356
357 void CurlRequest::didCompleteTransfer(CURLcode result)
358 {
359     if (m_cancelled) {
360         m_curlHandle = nullptr;
361         return;
362     }
363
364     if (result == CURLE_OK) {
365         if (needToInvokeDidReceiveResponse()) {
366             // Processing of didReceiveResponse() has not been completed. (For example, HEAD method)
367             // When completeDidReceiveResponse() is called, didCompleteTransfer() will be called again.
368
369             m_finishedResultCode = result;
370             invokeDidReceiveResponse(Action::FinishTransfer);
371         } else {
372             if (auto metrics = m_curlHandle->getNetworkLoadMetrics())
373                 m_networkLoadMetrics = *metrics;
374
375             finalizeTransfer();
376             callClient([this](CurlRequestClient* client) {
377                 if (client)
378                     client->curlDidComplete();
379             });
380         }
381     } else {
382         auto resourceError = ResourceError::httpError(result, m_request.url());
383         if (m_sslVerifier.sslErrors())
384             resourceError.setSslErrors(m_sslVerifier.sslErrors());
385
386         finalizeTransfer();
387         callClient([this, error = resourceError.isolatedCopy()](CurlRequestClient* client) {
388             if (client)
389                 client->curlDidFailWithError(error);
390         });
391     }
392 }
393
394 void CurlRequest::didCancelTransfer()
395 {
396     finalizeTransfer();
397     cleanupDownloadFile();
398 }
399
400 void CurlRequest::finalizeTransfer()
401 {
402     closeDownloadFile();
403     m_formDataStream.clean();
404     m_curlHandle = nullptr;
405 }
406
407 void CurlRequest::setupPUT(ResourceRequest& request)
408 {
409     m_curlHandle->enableHttpPutRequest();
410
411     // Disable the Expect: 100 continue header
412     m_curlHandle->removeRequestHeader("Expect");
413
414     auto elementSize = m_formDataStream.elementSize();
415     if (!elementSize)
416         return;
417
418     setupSendData(true);
419 }
420
421 void CurlRequest::setupPOST(ResourceRequest& request)
422 {
423     m_curlHandle->enableHttpPostRequest();
424
425     auto elementSize = m_formDataStream.elementSize();
426     if (!elementSize)
427         return;
428
429     // Do not stream for simple POST data
430     if (elementSize == 1) {
431         auto postData = m_formDataStream.getPostData();
432         if (postData && postData->size())
433             m_curlHandle->setPostFields(postData->data(), postData->size());
434     } else
435         setupSendData(false);
436 }
437
438 void CurlRequest::setupSendData(bool forPutMethod)
439 {
440     // curl guesses that we want chunked encoding as long as we specify the header
441     if (m_formDataStream.shouldUseChunkTransfer())
442         m_curlHandle->appendRequestHeader("Transfer-Encoding: chunked");
443     else {
444         if (forPutMethod)
445             m_curlHandle->setInFileSizeLarge(static_cast<curl_off_t>(m_formDataStream.totalSize()));
446         else
447             m_curlHandle->setPostFieldLarge(static_cast<curl_off_t>(m_formDataStream.totalSize()));
448     }
449
450     m_curlHandle->setReadCallbackFunction(willSendDataCallback, this);
451 }
452
453 void CurlRequest::invokeDidReceiveResponseForFile(URL& url)
454 {
455     // Since the code in didReceiveHeader() will not have run for local files
456     // the code to set the URL and fire didReceiveResponse is never run,
457     // which means the ResourceLoader's response does not contain the URL.
458     // Run the code here for local files to resolve the issue.
459
460     ASSERT(isMainThread());
461     ASSERT(url.isLocalFile());
462
463     m_response.url = url;
464     m_response.statusCode = 200;
465
466     // Determine the MIME type based on the path.
467     m_response.headers.append(String("Content-Type: " + MIMETypeRegistry::getMIMETypeForPath(m_response.url.path())));
468
469     if (!m_isSyncRequest) {
470         // DidReceiveResponse must not be called immediately
471         CurlRequestScheduler::singleton().callOnWorkerThread([protectedThis = makeRef(*this)]() {
472             protectedThis->invokeDidReceiveResponse(Action::StartTransfer);
473         });
474     } else {
475         // For synchronous, completeDidReceiveResponse() is called in platformContinueSynchronousDidReceiveResponse().
476         invokeDidReceiveResponse(Action::None);
477     }
478 }
479
480 void CurlRequest::invokeDidReceiveResponse(Action behaviorAfterInvoke)
481 {
482     ASSERT(!m_didNotifyResponse);
483
484     m_didNotifyResponse = true;
485     m_actionAfterInvoke = behaviorAfterInvoke;
486
487     callClient([this, response = m_response.isolatedCopy()](CurlRequestClient* client) {
488         if (client)
489             client->curlDidReceiveResponse(response);
490     });
491 }
492
493 void CurlRequest::completeDidReceiveResponse()
494 {
495     ASSERT(isMainThread());
496     ASSERT(m_didNotifyResponse);
497     ASSERT(!m_didReturnFromNotify);
498
499     if (isCancelled())
500         return;
501
502     if (m_actionAfterInvoke != Action::StartTransfer && isCompleted())
503         return;
504
505     m_didReturnFromNotify = true;
506
507     if (m_actionAfterInvoke == Action::ReceiveData) {
508         // Resume transfer
509         setCallbackPaused(false);
510     } else if (m_actionAfterInvoke == Action::StartTransfer) {
511         // Start transfer for file scheme
512         startWithJobManager();
513     } else if (m_actionAfterInvoke == Action::FinishTransfer) {
514         if (!m_isSyncRequest) {
515             CurlRequestScheduler::singleton().callOnWorkerThread([protectedThis = makeRef(*this), finishedResultCode = m_finishedResultCode]() {
516                 protectedThis->didCompleteTransfer(finishedResultCode);
517             });
518         } else
519             didCompleteTransfer(m_finishedResultCode);
520     }
521 }
522
523 void CurlRequest::setRequestPaused(bool paused)
524 {
525     auto wasPaused = isPaused();
526
527     m_isPausedOfRequest = paused;
528
529     if (isPaused() == wasPaused)
530         return;
531
532     pausedStatusChanged();
533 }
534
535 void CurlRequest::setCallbackPaused(bool paused)
536 {
537     auto wasPaused = isPaused();
538
539     m_isPausedOfCallback = paused;
540
541     if (isPaused() == wasPaused)
542         return;
543
544     // In this case, PAUSE will be executed within didReceiveData(). Change pause state and return.
545     if (paused)
546         return;
547
548     pausedStatusChanged();
549 }
550
551 void CurlRequest::pausedStatusChanged()
552 {
553     if (isCompletedOrCancelled())
554         return;
555
556     if (!m_isSyncRequest && isMainThread()) {
557         CurlRequestScheduler::singleton().callOnWorkerThread([protectedThis = makeRef(*this), paused = isPaused()]() {
558             if (protectedThis->isCompletedOrCancelled())
559                 return;
560
561             auto error = protectedThis->m_curlHandle->pause(paused ? CURLPAUSE_ALL : CURLPAUSE_CONT);
562             if ((error != CURLE_OK) && !paused) {
563                 // Restarting the handle has failed so just cancel it.
564                 callOnMainThread([protectedThis = makeRef(protectedThis.get())]() {
565                     protectedThis->cancel();
566                 });
567             }
568         });
569     } else {
570         auto error = m_curlHandle->pause(isPaused() ? CURLPAUSE_ALL : CURLPAUSE_CONT);
571         if ((error != CURLE_OK) && !isPaused())
572             cancel();
573     }
574 }
575
576 void CurlRequest::enableDownloadToFile()
577 {
578     LockHolder locker(m_downloadMutex);
579     m_isEnabledDownloadToFile = true;
580 }
581
582 const String& CurlRequest::getDownloadedFilePath()
583 {
584     LockHolder locker(m_downloadMutex);
585     return m_downloadFilePath;
586 }
587
588 void CurlRequest::writeDataToDownloadFileIfEnabled(const SharedBuffer& buffer)
589 {
590     {
591         LockHolder locker(m_downloadMutex);
592
593         if (!m_isEnabledDownloadToFile)
594             return;
595
596         if (m_downloadFilePath.isEmpty())
597             m_downloadFilePath = FileSystem::openTemporaryFile("download", m_downloadFileHandle);
598     }
599
600     if (m_downloadFileHandle != FileSystem::invalidPlatformFileHandle)
601         FileSystem::writeToFile(m_downloadFileHandle, buffer.data(), buffer.size());
602 }
603
604 void CurlRequest::closeDownloadFile()
605 {
606     LockHolder locker(m_downloadMutex);
607
608     if (m_downloadFileHandle == FileSystem::invalidPlatformFileHandle)
609         return;
610
611     FileSystem::closeFile(m_downloadFileHandle);
612     m_downloadFileHandle = FileSystem::invalidPlatformFileHandle;
613 }
614
615 void CurlRequest::cleanupDownloadFile()
616 {
617     LockHolder locker(m_downloadMutex);
618
619     if (!m_downloadFilePath.isEmpty()) {
620         FileSystem::deleteFile(m_downloadFilePath);
621         m_downloadFilePath = String();
622     }
623 }
624
625 CURLcode CurlRequest::willSetupSslCtxCallback(CURL*, void* sslCtx, void* userData)
626 {
627     return static_cast<CurlRequest*>(userData)->willSetupSslCtx(sslCtx);
628 }
629
630 size_t CurlRequest::willSendDataCallback(char* ptr, size_t blockSize, size_t numberOfBlocks, void* userData)
631 {
632     return static_cast<CurlRequest*>(userData)->willSendData(ptr, blockSize, numberOfBlocks);
633 }
634
635 size_t CurlRequest::didReceiveHeaderCallback(char* ptr, size_t blockSize, size_t numberOfBlocks, void* userData)
636 {
637     return static_cast<CurlRequest*>(userData)->didReceiveHeader(String(ptr, blockSize * numberOfBlocks));
638 }
639
640 size_t CurlRequest::didReceiveDataCallback(char* ptr, size_t blockSize, size_t numberOfBlocks, void* userData)
641 {
642     return static_cast<CurlRequest*>(userData)->didReceiveData(SharedBuffer::create(ptr, blockSize * numberOfBlocks));
643 }
644
645 }
646
647 #endif