[Curl] Cookies are sometimes not set in download request.
[WebKit-https.git] / Source / WebCore / platform / network / curl / CurlDownload.cpp
1 /*
2  * Copyright (C) 2013 Apple Inc.  All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "config.h"
27 #include "CurlDownload.h"
28
29 #include "HTTPParsers.h"
30 #include "MainThreadTask.h"
31 #include "ResourceHandleManager.h"
32 #include "ResourceRequest.h"
33 #include <wtf/MainThread.h>
34 #include <wtf/text/CString.h>
35
36 using namespace WebCore;
37
38 template<> struct CrossThreadCopierBase<false, false, CurlDownload*> : public CrossThreadCopierPassThrough<CurlDownload*> {
39 };
40
41 namespace WebCore {
42
43 // CurlDownloadManager -------------------------------------------------------------------
44
45 CurlDownloadManager::CurlDownloadManager()
46 : m_threadId(0)
47 , m_curlMultiHandle(0)
48 , m_runThread(false)
49 {
50     curl_global_init(CURL_GLOBAL_ALL);
51     m_curlMultiHandle = curl_multi_init();
52 }
53
54 CurlDownloadManager::~CurlDownloadManager()
55 {
56     stopThread();
57     curl_multi_cleanup(m_curlMultiHandle);
58     curl_global_cleanup();
59 }
60
61 bool CurlDownloadManager::add(CURL* curlHandle)
62 {
63     MutexLocker locker(m_mutex);
64
65     m_pendingHandleList.append(curlHandle);
66     startThreadIfNeeded();
67
68     return true;
69 }
70
71 bool CurlDownloadManager::remove(CURL* curlHandle)
72 {
73     MutexLocker locker(m_mutex);
74
75     m_removedHandleList.append(curlHandle);
76
77     return true;
78 }
79
80 int CurlDownloadManager::getActiveDownloadCount() const
81 {
82     MutexLocker locker(m_mutex);
83     return m_activeHandleList.size();
84 }
85
86 int CurlDownloadManager::getPendingDownloadCount() const
87 {
88     MutexLocker locker(m_mutex);
89     return m_pendingHandleList.size();
90 }
91
92 void CurlDownloadManager::startThreadIfNeeded()
93 {
94     if (!m_runThread) {
95         if (m_threadId)
96             waitForThreadCompletion(m_threadId);
97         m_runThread = true;
98         m_threadId = createThread(downloadThread, this, "downloadThread");
99     }
100 }
101
102 void CurlDownloadManager::stopThread()
103 {
104     m_runThread = false;
105
106     if (m_threadId) {
107         waitForThreadCompletion(m_threadId);
108         m_threadId = 0;
109     }
110 }
111
112 void CurlDownloadManager::stopThreadIfIdle()
113 {
114     MutexLocker locker(m_mutex);
115
116     if (!getActiveDownloadCount() && !getPendingDownloadCount())
117         setRunThread(false);
118 }
119
120 void CurlDownloadManager::updateHandleList()
121 {
122     MutexLocker locker(m_mutex);
123
124     // Remove curl easy handles from multi list 
125     int size = m_removedHandleList.size();
126     for (int i = 0; i < size; i++) {
127         removeFromCurl(m_removedHandleList[0]);
128         m_removedHandleList.remove(0);
129     }
130
131     // Add pending curl easy handles to multi list 
132     size = m_pendingHandleList.size();
133     for (int i = 0; i < size; i++) {
134         addToCurl(m_pendingHandleList[0]);
135         m_pendingHandleList.remove(0);
136     }
137 }
138
139 bool CurlDownloadManager::addToCurl(CURL* curlHandle)
140 {
141     CURLMcode retval = curl_multi_add_handle(m_curlMultiHandle, curlHandle);
142     if (retval == CURLM_OK) {
143         m_activeHandleList.append(curlHandle);
144         return true;
145     }
146     return false;
147 }
148
149 bool CurlDownloadManager::removeFromCurl(CURL* curlHandle)
150 {
151     int handlePos = m_activeHandleList.find(curlHandle);
152
153     if (handlePos < 0)
154         return true;
155     
156     CURLMcode retval = curl_multi_remove_handle(m_curlMultiHandle, curlHandle);
157     if (retval == CURLM_OK) {
158         m_activeHandleList.remove(handlePos);
159         curl_easy_cleanup(curlHandle);
160         return true;
161     }
162     return false;
163 }
164
165 void CurlDownloadManager::downloadThread(void* data)
166 {
167     CurlDownloadManager* downloadManager = reinterpret_cast<CurlDownloadManager*>(data);
168
169     while (downloadManager->runThread()) {
170
171         downloadManager->updateHandleList();
172
173         // Retry 'select' if it was interrupted by a process signal.
174         int rc = 0;
175         do {
176             fd_set fdread;
177             fd_set fdwrite;
178             fd_set fdexcep;
179
180             int maxfd = 0;
181
182             const int selectTimeoutMS = 5;
183
184             struct timeval timeout;
185             timeout.tv_sec = 0;
186             timeout.tv_usec = selectTimeoutMS * 1000; // select waits microseconds
187
188             FD_ZERO(&fdread);
189             FD_ZERO(&fdwrite);
190             FD_ZERO(&fdexcep);
191             curl_multi_fdset(downloadManager->getMultiHandle(), &fdread, &fdwrite, &fdexcep, &maxfd);
192             // When the 3 file descriptors are empty, winsock will return -1
193             // and bail out, stopping the file download. So make sure we
194             // have valid file descriptors before calling select.
195             if (maxfd >= 0)
196                 rc = ::select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
197         } while (rc == -1 && errno == EINTR);
198
199         int activeDownloadCount = 0;
200         while (curl_multi_perform(downloadManager->getMultiHandle(), &activeDownloadCount) == CURLM_CALL_MULTI_PERFORM) { }
201
202         int messagesInQueue = 0;
203         CURLMsg* msg = curl_multi_info_read(downloadManager->getMultiHandle(), &messagesInQueue);
204
205         if (!msg)
206             continue;
207
208         CurlDownload* download = 0;
209         CURLcode err = curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &download);
210
211         if (msg->msg == CURLMSG_DONE) {
212             if (msg->data.result == CURLE_OK)
213                 callOnMainThread<CurlDownload*, CurlDownload*>(CurlDownload::downloadFinishedCallback, download);
214             else
215                 callOnMainThread<CurlDownload*, CurlDownload*>(CurlDownload::downloadFailedCallback, download);
216
217             downloadManager->removeFromCurl(msg->easy_handle);
218         }
219
220         downloadManager->stopThreadIfIdle();
221     }
222 }
223
224 // CurlDownload --------------------------------------------------------------------------
225
226 CurlDownloadManager CurlDownload::m_downloadManager;
227
228 CurlDownload::CurlDownload()
229 : m_curlHandle(0)
230 , m_customHeaders(0)
231 , m_url(0)
232 , m_tempHandle(invalidPlatformFileHandle)
233 , m_deletesFileUponFailure(false)
234 , m_listener(0)
235 {
236 }
237
238 CurlDownload::~CurlDownload()
239 {
240     MutexLocker locker(m_mutex);
241
242     if (m_url)
243         fastFree(m_url);
244
245     if (m_customHeaders)
246         curl_slist_free_all(m_customHeaders);
247
248     closeFile();
249     moveFileToDestination();
250 }
251
252 void CurlDownload::init(CurlDownloadListener* listener, const URL& url)
253 {
254     if (!listener)
255         return;
256
257     MutexLocker locker(m_mutex);
258
259     m_curlHandle = curl_easy_init();
260
261     String urlStr = url.string();
262     m_url = fastStrDup(urlStr.latin1().data());
263
264     curl_easy_setopt(m_curlHandle, CURLOPT_URL, m_url);
265     curl_easy_setopt(m_curlHandle, CURLOPT_PRIVATE, this);
266     curl_easy_setopt(m_curlHandle, CURLOPT_WRITEFUNCTION, writeCallback);
267     curl_easy_setopt(m_curlHandle, CURLOPT_WRITEDATA, this);
268     curl_easy_setopt(m_curlHandle, CURLOPT_HEADERFUNCTION, headerCallback);
269     curl_easy_setopt(m_curlHandle, CURLOPT_WRITEHEADER, this);
270     curl_easy_setopt(m_curlHandle, CURLOPT_FOLLOWLOCATION, 1);
271     curl_easy_setopt(m_curlHandle, CURLOPT_MAXREDIRS, 10);
272     curl_easy_setopt(m_curlHandle, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
273
274     const char* certPath = getenv("CURL_CA_BUNDLE_PATH");
275     if (certPath)
276         curl_easy_setopt(m_curlHandle, CURLOPT_CAINFO, certPath);
277
278     CURLSH* curlsh = ResourceHandleManager::sharedInstance()->getCurlShareHandle();
279     if (curlsh)
280         curl_easy_setopt(m_curlHandle, CURLOPT_SHARE, curlsh);
281
282     m_listener = listener;
283 }
284
285 void CurlDownload::init(CurlDownloadListener* listener, ResourceHandle*, const ResourceRequest& request, const ResourceResponse&)
286 {
287     if (!listener)
288         return;
289
290     MutexLocker locker(m_mutex);
291
292     URL url(ParsedURLString, request.url());
293
294     init(listener, url);
295
296     addHeaders(request);
297 }
298
299 bool CurlDownload::start()
300 {
301     return m_downloadManager.add(m_curlHandle);
302 }
303
304 bool CurlDownload::cancel()
305 {
306     return m_downloadManager.remove(m_curlHandle);
307 }
308
309 String CurlDownload::getTempPath() const
310 {
311     MutexLocker locker(m_mutex);
312     return m_tempPath;
313 }
314
315 String CurlDownload::getUrl() const
316 {
317     MutexLocker locker(m_mutex);
318     return String(m_url);
319 }
320
321 ResourceResponse CurlDownload::getResponse() const
322 {
323     MutexLocker locker(m_mutex);
324     return m_response;
325 }
326
327 void CurlDownload::closeFile()
328 {
329     MutexLocker locker(m_mutex);
330
331     if (m_tempHandle != invalidPlatformFileHandle) {
332         WebCore::closeFile(m_tempHandle);
333         m_tempHandle = invalidPlatformFileHandle;
334     }
335 }
336
337 void CurlDownload::moveFileToDestination()
338 {
339     if (m_destination.isEmpty())
340         return;
341
342     ::MoveFileEx(m_tempPath.charactersWithNullTermination().data(), m_destination.charactersWithNullTermination().data(), MOVEFILE_COPY_ALLOWED | MOVEFILE_REPLACE_EXISTING);
343 }
344
345 void CurlDownload::writeDataToFile(const char* data, int size)
346 {
347     if (m_tempPath.isEmpty())
348         m_tempPath = openTemporaryFile("download", m_tempHandle);
349
350     if (m_tempHandle != invalidPlatformFileHandle)
351         writeToFile(m_tempHandle, data, size);
352 }
353
354 void CurlDownload::addHeaders(const ResourceRequest& request)
355 {
356     if (request.httpHeaderFields().size() > 0) {
357         struct curl_slist* headers = 0;
358
359         HTTPHeaderMap customHeaders = request.httpHeaderFields();
360         HTTPHeaderMap::const_iterator end = customHeaders.end();
361         for (HTTPHeaderMap::const_iterator it = customHeaders.begin(); it != end; ++it) {
362             const String& value = it->value;
363             String headerString(it->key);
364             if (value.isEmpty())
365                 // Insert the ; to tell curl that this header has an empty value.
366                 headerString.append(";");
367             else {
368                 headerString.append(": ");
369                 headerString.append(value);
370             }
371             CString headerLatin1 = headerString.latin1();
372             headers = curl_slist_append(headers, headerLatin1.data());
373         }
374
375         if (headers) {
376             curl_easy_setopt(m_curlHandle, CURLOPT_HTTPHEADER, headers);
377             m_customHeaders = headers;
378         }
379     }
380 }
381
382 void CurlDownload::didReceiveHeader(const String& header)
383 {
384     MutexLocker locker(m_mutex);
385
386     if (header == "\r\n" || header == "\n") {
387
388         long httpCode = 0;
389         CURLcode err = curl_easy_getinfo(m_curlHandle, CURLINFO_RESPONSE_CODE, &httpCode);
390
391         if (httpCode >= 200 && httpCode < 300) {
392             const char* url = 0;
393             err = curl_easy_getinfo(m_curlHandle, CURLINFO_EFFECTIVE_URL, &url);
394             m_response.setURL(URL(ParsedURLString, url));
395
396             m_response.setMimeType(extractMIMETypeFromMediaType(m_response.httpHeaderField("Content-Type")));
397             m_response.setTextEncodingName(extractCharsetFromMediaType(m_response.httpHeaderField("Content-Type")));
398             m_response.setSuggestedFilename(filenameFromHTTPContentDisposition(m_response.httpHeaderField("Content-Disposition")));
399
400             callOnMainThread<CurlDownload*, CurlDownload*>(receivedResponseCallback, this);
401         }
402     } else {
403         int splitPos = header.find(":");
404         if (splitPos != -1)
405             m_response.setHTTPHeaderField(header.left(splitPos), header.substring(splitPos+1).stripWhiteSpace());
406     }
407 }
408
409 void CurlDownload::didReceiveData(void* data, int size)
410 {
411     MutexLocker locker(m_mutex);
412
413     callOnMainThread<CurlDownload*, CurlDownload*, int, int>(receivedDataCallback, this, size);
414
415     writeDataToFile(static_cast<const char*>(data), size);
416 }
417
418 void CurlDownload::didReceiveResponse()
419 {
420     if (m_listener)
421         m_listener->didReceiveResponse();
422 }
423
424 void CurlDownload::didReceiveDataOfLength(int size)
425 {
426     if (m_listener)
427         m_listener->didReceiveDataOfLength(size);
428 }
429
430 void CurlDownload::didFinish()
431 {
432     closeFile();
433     moveFileToDestination();
434
435     if (m_listener)
436         m_listener->didFinish();
437 }
438
439 void CurlDownload::didFail()
440 {
441     MutexLocker locker(m_mutex);
442
443     closeFile();
444
445     if (m_deletesFileUponFailure)
446         deleteFile(m_tempPath);
447
448     if (m_listener)
449         m_listener->didFail();
450 }
451
452 size_t CurlDownload::writeCallback(void* ptr, size_t size, size_t nmemb, void* data)
453 {
454     size_t totalSize = size * nmemb;
455     CurlDownload* download = reinterpret_cast<CurlDownload*>(data);
456
457     if (download)
458         download->didReceiveData(ptr, totalSize);
459
460     return totalSize;
461 }
462
463 size_t CurlDownload::headerCallback(char* ptr, size_t size, size_t nmemb, void* data)
464 {
465     size_t totalSize = size * nmemb;
466     CurlDownload* download = reinterpret_cast<CurlDownload*>(data);
467
468     String header = String::fromUTF8WithLatin1Fallback(static_cast<const char*>(ptr), totalSize);
469
470     if (download)
471         download->didReceiveHeader(header);
472
473     return totalSize;
474 }
475
476 void CurlDownload::downloadFinishedCallback(CurlDownload* download)
477 {
478     if (download)
479         download->didFinish();
480 }
481
482 void CurlDownload::downloadFailedCallback(CurlDownload* download)
483 {
484     if (download)
485         download->didFail();
486 }
487
488 void CurlDownload::receivedDataCallback(CurlDownload* download, int size)
489 {
490     if (download)
491         download->didReceiveDataOfLength(size);
492 }
493
494 void CurlDownload::receivedResponseCallback(CurlDownload* download)
495 {
496     if (download)
497         download->didReceiveResponse();
498 }
499
500 }