e40987e6383851e6c805abcc8fd91a36f2c5fa1c
[WebKit-https.git] / Source / WebCore / platform / network / curl / ResourceHandleManager.cpp
1 /*
2  * Copyright (C) 2004, 2006 Apple Inc.  All rights reserved.
3  * Copyright (C) 2006 Michael Emmel mike.emmel@gmail.com
4  * Copyright (C) 2007 Alp Toker <alp@atoker.com>
5  * Copyright (C) 2007 Holger Hans Peter Freyther
6  * Copyright (C) 2008 Collabora Ltd.
7  * Copyright (C) 2008 Nuanti Ltd.
8  * Copyright (C) 2009 Appcelerator Inc.
9  * Copyright (C) 2009 Brent Fulgham <bfulgham@webkit.org>
10  * Copyright (C) 2013 Peter Gal <galpeter@inf.u-szeged.hu>, University of Szeged
11  * Copyright (C) 2013 Alex Christensen <achristensen@webkit.org>
12  * Copyright (C) 2013 University of Szeged
13  * All rights reserved.
14  *
15  * Redistribution and use in source and binary forms, with or without
16  * modification, are permitted provided that the following conditions
17  * are met:
18  * 1. Redistributions of source code must retain the above copyright
19  *    notice, this list of conditions and the following disclaimer.
20  * 2. Redistributions in binary form must reproduce the above copyright
21  *    notice, this list of conditions and the following disclaimer in the
22  *    documentation and/or other materials provided with the distribution.
23  *
24  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
25  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
28  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
29  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
30  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
31  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
32  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35  */
36
37 #include "config.h"
38 #include "ResourceHandleManager.h"
39
40 #if USE(CURL)
41
42 #include "CredentialStorage.h"
43 #include "CurlCacheManager.h"
44 #include "DataURL.h"
45 #include "HTTPHeaderNames.h"
46 #include "HTTPParsers.h"
47 #include "MIMETypeRegistry.h"
48 #include "MultipartHandle.h"
49 #include "ResourceError.h"
50 #include "ResourceHandle.h"
51 #include "ResourceHandleInternal.h"
52 #include "SSLHandle.h"
53
54 #if OS(WINDOWS)
55 #include "WebCoreBundleWin.h"
56 #include <shlobj.h>
57 #include <shlwapi.h>
58 #else
59 #include <sys/param.h>
60 #define MAX_PATH MAXPATHLEN
61 #endif
62
63 #include <errno.h>
64 #include <stdio.h>
65 #if ENABLE(WEB_TIMING)
66 #include <wtf/CurrentTime.h>
67 #endif
68 #if USE(CF)
69 #include <wtf/RetainPtr.h>
70 #endif
71 #include <wtf/Threading.h>
72 #include <wtf/Vector.h>
73 #include <wtf/text/CString.h>
74
75
76 namespace WebCore {
77
78 const int selectTimeoutMS = 5;
79 const double pollTimeSeconds = 0.05;
80 const int maxRunningJobs = 5;
81
82 static const bool ignoreSSLErrors = getenv("WEBKIT_IGNORE_SSL_ERRORS");
83
84 static CString certificatePath()
85 {
86 #if USE(CF)
87     CFBundleRef webKitBundleRef = webKitBundle();
88     if (webKitBundleRef) {
89         RetainPtr<CFURLRef> certURLRef = adoptCF(CFBundleCopyResourceURL(webKitBundleRef, CFSTR("cacert"), CFSTR("pem"), CFSTR("certificates")));
90         if (certURLRef) {
91             char path[MAX_PATH];
92             CFURLGetFileSystemRepresentation(certURLRef.get(), false, reinterpret_cast<UInt8*>(path), MAX_PATH);
93             return path;
94         }
95     }
96 #endif
97     char* envPath = getenv("CURL_CA_BUNDLE_PATH");
98     if (envPath)
99        return envPath;
100
101     return CString();
102 }
103
104 static char* cookieJarPath()
105 {
106     char* cookieJarPath = getenv("CURL_COOKIE_JAR_PATH");
107     if (cookieJarPath)
108         return fastStrDup(cookieJarPath);
109
110 #if OS(WINDOWS)
111     char executablePath[MAX_PATH];
112     char appDataDirectory[MAX_PATH];
113     char cookieJarFullPath[MAX_PATH];
114     char cookieJarDirectory[MAX_PATH];
115
116     if (FAILED(::SHGetFolderPathA(0, CSIDL_LOCAL_APPDATA | CSIDL_FLAG_CREATE, 0, 0, appDataDirectory))
117         || FAILED(::GetModuleFileNameA(0, executablePath, MAX_PATH)))
118         return fastStrDup("cookies.dat");
119
120     ::PathRemoveExtensionA(executablePath);
121     LPSTR executableName = ::PathFindFileNameA(executablePath);
122     sprintf_s(cookieJarDirectory, MAX_PATH, "%s/%s", appDataDirectory, executableName);
123     sprintf_s(cookieJarFullPath, MAX_PATH, "%s/cookies.dat", cookieJarDirectory);
124
125     if (::SHCreateDirectoryExA(0, cookieJarDirectory, 0) != ERROR_SUCCESS
126         && ::GetLastError() != ERROR_FILE_EXISTS
127         && ::GetLastError() != ERROR_ALREADY_EXISTS)
128         return fastStrDup("cookies.dat");
129
130     return fastStrDup(cookieJarFullPath);
131 #else
132     return fastStrDup("cookies.dat");
133 #endif
134 }
135
136 static Lock* sharedResourceMutex(curl_lock_data data)
137 {
138     DEPRECATED_DEFINE_STATIC_LOCAL(Lock, cookieMutex, ());
139     DEPRECATED_DEFINE_STATIC_LOCAL(Lock, dnsMutex, ());
140     DEPRECATED_DEFINE_STATIC_LOCAL(Lock, shareMutex, ());
141
142     switch (data) {
143         case CURL_LOCK_DATA_COOKIE:
144             return &cookieMutex;
145         case CURL_LOCK_DATA_DNS:
146             return &dnsMutex;
147         case CURL_LOCK_DATA_SHARE:
148             return &shareMutex;
149         default:
150             ASSERT_NOT_REACHED();
151             return NULL;
152     }
153 }
154
155 #if ENABLE(WEB_TIMING)
156 static int milisecondsSinceRequest(double requestTime)
157 {
158     return static_cast<int>((monotonicallyIncreasingTime() - requestTime) * 1000.0);
159 }
160
161 static void calculateWebTimingInformations(ResourceHandleInternal* d)
162 {
163     double startTransfertTime = 0;
164     double preTransferTime = 0;
165     double dnslookupTime = 0;
166     double connectTime = 0;
167     double appConnectTime = 0;
168
169     curl_easy_getinfo(d->m_handle, CURLINFO_NAMELOOKUP_TIME, &dnslookupTime);
170     curl_easy_getinfo(d->m_handle, CURLINFO_CONNECT_TIME, &connectTime);
171     curl_easy_getinfo(d->m_handle, CURLINFO_APPCONNECT_TIME, &appConnectTime);
172     curl_easy_getinfo(d->m_handle, CURLINFO_STARTTRANSFER_TIME, &startTransfertTime);
173     curl_easy_getinfo(d->m_handle, CURLINFO_PRETRANSFER_TIME, &preTransferTime);
174
175     d->m_response.resourceLoadTiming().domainLookupStart = 0;
176     d->m_response.resourceLoadTiming().domainLookupEnd = static_cast<int>(dnslookupTime * 1000);
177
178     d->m_response.resourceLoadTiming().connectStart = static_cast<int>(dnslookupTime * 1000);
179     d->m_response.resourceLoadTiming().connectEnd = static_cast<int>(connectTime * 1000);
180
181     d->m_response.resourceLoadTiming().requestStart = static_cast<int>(connectTime *1000);
182     d->m_response.resourceLoadTiming().responseStart =static_cast<int>(preTransferTime * 1000);
183
184     if (appConnectTime)
185         d->m_response.resourceLoadTiming().secureConnectionStart = static_cast<int>(connectTime * 1000);
186 }
187 #endif
188
189 // libcurl does not implement its own thread synchronization primitives.
190 // these two functions provide mutexes for cookies, and for the global DNS
191 // cache.
192 static void curl_lock_callback(CURL* /* handle */, curl_lock_data data, curl_lock_access /* access */, void* /* userPtr */)
193 {
194     if (Lock* mutex = sharedResourceMutex(data))
195         mutex->lock();
196 }
197
198 static void curl_unlock_callback(CURL* /* handle */, curl_lock_data data, void* /* userPtr */)
199 {
200     if (Lock* mutex = sharedResourceMutex(data))
201         mutex->unlock();
202 }
203
204 inline static bool isHttpInfo(int statusCode)
205 {
206     return 100 <= statusCode && statusCode < 200;
207 }
208
209 inline static bool isHttpRedirect(int statusCode)
210 {
211     return 300 <= statusCode && statusCode < 400 && statusCode != 304;
212 }
213
214 inline static bool isHttpAuthentication(int statusCode)
215 {
216     return statusCode == 401;
217 }
218
219 inline static bool isHttpNotModified(int statusCode)
220 {
221     return statusCode == 304;
222 }
223
224 ResourceHandleManager::ResourceHandleManager()
225     : m_downloadTimer(*this, &ResourceHandleManager::downloadTimerCallback)
226     , m_cookieJarFileName(cookieJarPath())
227     , m_certificatePath (certificatePath())
228     , m_runningJobs(0)
229 #ifndef NDEBUG
230     , m_logFile(nullptr)
231 #endif
232 {
233     curl_global_init(CURL_GLOBAL_ALL);
234     m_curlMultiHandle = curl_multi_init();
235     m_curlShareHandle = curl_share_init();
236     curl_share_setopt(m_curlShareHandle, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE);
237     curl_share_setopt(m_curlShareHandle, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
238     curl_share_setopt(m_curlShareHandle, CURLSHOPT_LOCKFUNC, curl_lock_callback);
239     curl_share_setopt(m_curlShareHandle, CURLSHOPT_UNLOCKFUNC, curl_unlock_callback);
240
241     initCookieSession();
242
243 #ifndef NDEBUG
244     char* logFile = getenv("CURL_LOG_FILE");
245     if (logFile)
246         m_logFile = fopen(logFile, "a");
247 #endif
248 }
249
250 ResourceHandleManager::~ResourceHandleManager()
251 {
252     curl_multi_cleanup(m_curlMultiHandle);
253     curl_share_cleanup(m_curlShareHandle);
254     if (m_cookieJarFileName)
255         fastFree(m_cookieJarFileName);
256     curl_global_cleanup();
257
258 #ifndef NDEBUG
259     if (m_logFile)
260         fclose(m_logFile);
261 #endif
262 }
263
264 CURLSH* ResourceHandleManager::getCurlShareHandle() const
265 {
266     return m_curlShareHandle;
267 }
268
269 void ResourceHandleManager::setCookieJarFileName(const char* cookieJarFileName)
270 {
271     m_cookieJarFileName = fastStrDup(cookieJarFileName);
272 }
273
274 const char* ResourceHandleManager::getCookieJarFileName() const
275 {
276     return m_cookieJarFileName;
277 }
278
279 ResourceHandleManager* ResourceHandleManager::sharedInstance()
280 {
281     static ResourceHandleManager* sharedInstance = 0;
282     if (!sharedInstance)
283         sharedInstance = new ResourceHandleManager();
284     return sharedInstance;
285 }
286
287 static void handleLocalReceiveResponse (CURL* handle, ResourceHandle* job, ResourceHandleInternal* d)
288 {
289     // since the code in headerCallback will not have run for local files
290     // the code to set the URL and fire didReceiveResponse is never run,
291     // which means the ResourceLoader's response does not contain the URL.
292     // Run the code here for local files to resolve the issue.
293     // TODO: See if there is a better approach for handling this.
294      const char* hdr;
295      CURLcode err = curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &hdr);
296      ASSERT_UNUSED(err, CURLE_OK == err);
297      d->m_response.setURL(URL(ParsedURLString, hdr));
298      if (d->client())
299          d->client()->didReceiveResponse(job, d->m_response);
300      d->m_response.setResponseFired(true);
301 }
302
303
304 // called with data after all headers have been processed via headerCallback
305 static size_t writeCallback(void* ptr, size_t size, size_t nmemb, void* data)
306 {
307     ResourceHandle* job = static_cast<ResourceHandle*>(data);
308     ResourceHandleInternal* d = job->getInternal();
309     if (d->m_cancelled)
310         return 0;
311
312     // We should never be called when deferred loading is activated.
313     ASSERT(!d->m_defersLoading);
314
315     size_t totalSize = size * nmemb;
316
317     // this shouldn't be necessary but apparently is. CURL writes the data
318     // of html page even if it is a redirect that was handled internally
319     // can be observed e.g. on gmail.com
320     CURL* h = d->m_handle;
321     long httpCode = 0;
322     CURLcode err = curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &httpCode);
323     if (CURLE_OK == err && httpCode >= 300 && httpCode < 400)
324         return totalSize;
325
326     if (!d->m_response.responseFired()) {
327         handleLocalReceiveResponse(h, job, d);
328         if (d->m_cancelled)
329             return 0;
330     }
331
332     if (d->m_multipartHandle)
333         d->m_multipartHandle->contentReceived(static_cast<const char*>(ptr), totalSize);
334     else if (d->client()) {
335         d->client()->didReceiveData(job, static_cast<char*>(ptr), totalSize, 0);
336         CurlCacheManager::getInstance().didReceiveData(*job, static_cast<char*>(ptr), totalSize);
337     }
338
339     return totalSize;
340 }
341
342 static bool isAppendableHeader(const String &key)
343 {
344     static const char* appendableHeaders[] = {
345         "access-control-allow-headers",
346         "access-control-allow-methods",
347         "access-control-allow-origin",
348         "access-control-expose-headers",
349         "allow",
350         "cache-control",
351         "connection",
352         "content-encoding",
353         "content-language",
354         "if-match",
355         "if-none-match",
356         "keep-alive",
357         "pragma",
358         "proxy-authenticate",
359         "public",
360         "server",
361         "set-cookie",
362         "te",
363         "trailer",
364         "transfer-encoding",
365         "upgrade",
366         "user-agent",
367         "vary",
368         "via",
369         "warning",
370         "www-authenticate",
371         0
372     };
373
374     // Custom headers start with 'X-', and need no further checking.
375     if (key.startsWith("x-", /* caseSensitive */ false))
376         return true;
377
378     for (unsigned i = 0; appendableHeaders[i]; ++i)
379         if (equalIgnoringCase(key, appendableHeaders[i]))
380             return true;
381
382     return false;
383 }
384
385 static void removeLeadingAndTrailingQuotes(String& value)
386 {
387     unsigned length = value.length();
388     if (value.startsWith('"') && value.endsWith('"') && length > 1)
389         value = value.substring(1, length-2);
390 }
391
392 static bool getProtectionSpace(CURL* h, const ResourceResponse& response, ProtectionSpace& protectionSpace)
393 {
394     CURLcode err;
395
396     long port = 0;
397     err = curl_easy_getinfo(h, CURLINFO_PRIMARY_PORT, &port);
398     if (err != CURLE_OK)
399         return false;
400
401     long availableAuth = CURLAUTH_NONE;
402     err = curl_easy_getinfo(h, CURLINFO_HTTPAUTH_AVAIL, &availableAuth);
403     if (err != CURLE_OK)
404         return false;
405
406     const char* effectiveUrl = 0;
407     err = curl_easy_getinfo(h, CURLINFO_EFFECTIVE_URL, &effectiveUrl);
408     if (err != CURLE_OK)
409         return false;
410
411     URL url(ParsedURLString, effectiveUrl);
412
413     String host = url.host();
414     String protocol = url.protocol();
415
416     String realm;
417
418     const String authHeader = response.httpHeaderField(HTTPHeaderName::Authorization);
419     const String realmString = "realm=";
420     int realmPos = authHeader.find(realmString);
421     if (realmPos > 0) {
422         realm = authHeader.substring(realmPos + realmString.length());
423         realm = realm.left(realm.find(','));
424         removeLeadingAndTrailingQuotes(realm);
425     }
426
427     ProtectionSpaceServerType serverType = ProtectionSpaceServerHTTP;
428     if (protocol == "https")
429         serverType = ProtectionSpaceServerHTTPS;
430
431     ProtectionSpaceAuthenticationScheme authScheme = ProtectionSpaceAuthenticationSchemeUnknown;
432
433     if (availableAuth & CURLAUTH_BASIC)
434         authScheme = ProtectionSpaceAuthenticationSchemeHTTPBasic;
435     if (availableAuth & CURLAUTH_DIGEST)
436         authScheme = ProtectionSpaceAuthenticationSchemeHTTPDigest;
437     if (availableAuth & CURLAUTH_GSSNEGOTIATE)
438         authScheme = ProtectionSpaceAuthenticationSchemeNegotiate;
439     if (availableAuth & CURLAUTH_NTLM)
440         authScheme = ProtectionSpaceAuthenticationSchemeNTLM;
441
442     protectionSpace = ProtectionSpace(host, port, serverType, realm, authScheme);
443
444     return true;
445 }
446
447 /*
448  * This is being called for each HTTP header in the response. This includes '\r\n'
449  * for the last line of the header.
450  *
451  * We will add each HTTP Header to the ResourceResponse and on the termination
452  * of the header (\r\n) we will parse Content-Type and Content-Disposition and
453  * update the ResourceResponse and then send it away.
454  *
455  */
456 static size_t headerCallback(char* ptr, size_t size, size_t nmemb, void* data)
457 {
458     ResourceHandle* job = static_cast<ResourceHandle*>(data);
459     ResourceHandleInternal* d = job->getInternal();
460     if (d->m_cancelled)
461         return 0;
462
463     // We should never be called when deferred loading is activated.
464     ASSERT(!d->m_defersLoading);
465
466     size_t totalSize = size * nmemb;
467     ResourceHandleClient* client = d->client();
468
469     String header(static_cast<const char*>(ptr), totalSize);
470
471     /*
472      * a) We can finish and send the ResourceResponse
473      * b) We will add the current header to the HTTPHeaderMap of the ResourceResponse
474      *
475      * The HTTP standard requires to use \r\n but for compatibility it recommends to
476      * accept also \n.
477      */
478     if (header == String("\r\n") || header == String("\n")) {
479         CURL* h = d->m_handle;
480
481         long httpCode = 0;
482         curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &httpCode);
483
484         if (isHttpInfo(httpCode)) {
485             // Just return when receiving http info, e.g. HTTP/1.1 100 Continue.
486             // If not, the request might be cancelled, because the MIME type will be empty for this response.
487             return totalSize;
488         }
489
490         double contentLength = 0;
491         curl_easy_getinfo(h, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &contentLength);
492         d->m_response.setExpectedContentLength(static_cast<long long int>(contentLength));
493
494         const char* hdr;
495         curl_easy_getinfo(h, CURLINFO_EFFECTIVE_URL, &hdr);
496         d->m_response.setURL(URL(ParsedURLString, hdr));
497
498         d->m_response.setHTTPStatusCode(httpCode);
499         d->m_response.setMimeType(extractMIMETypeFromMediaType(d->m_response.httpHeaderField(HTTPHeaderName::ContentType)).lower());
500         d->m_response.setTextEncodingName(extractCharsetFromMediaType(d->m_response.httpHeaderField(HTTPHeaderName::ContentType)));
501
502         if (d->m_response.isMultipart()) {
503             String boundary;
504             bool parsed = MultipartHandle::extractBoundary(d->m_response.httpHeaderField(HTTPHeaderName::ContentType), boundary);
505             if (parsed)
506                 d->m_multipartHandle = std::make_unique<MultipartHandle>(job, boundary);
507         }
508
509         // HTTP redirection
510         if (isHttpRedirect(httpCode)) {
511             String location = d->m_response.httpHeaderField(HTTPHeaderName::Location);
512             if (!location.isEmpty()) {
513                 URL newURL = URL(job->firstRequest().url(), location);
514
515                 ResourceRequest redirectedRequest = job->firstRequest();
516                 redirectedRequest.setURL(newURL);
517                 if (client)
518                     client->willSendRequest(job, redirectedRequest, d->m_response);
519
520                 d->m_firstRequest.setURL(newURL);
521
522                 return totalSize;
523             }
524         } else if (isHttpAuthentication(httpCode)) {
525             ProtectionSpace protectionSpace;
526             if (getProtectionSpace(d->m_handle, d->m_response, protectionSpace)) {
527                 Credential credential;
528                 AuthenticationChallenge challenge(protectionSpace, credential, d->m_authFailureCount, d->m_response, ResourceError());
529                 challenge.setAuthenticationClient(job);
530                 job->didReceiveAuthenticationChallenge(challenge);
531                 d->m_authFailureCount++;
532                 return totalSize;
533             }
534         }
535
536         if (client) {
537             if (isHttpNotModified(httpCode)) {
538                 const String& url = job->firstRequest().url().string();
539                 if (CurlCacheManager::getInstance().getCachedResponse(url, d->m_response)) {
540                     if (d->m_addedCacheValidationHeaders) {
541                         d->m_response.setHTTPStatusCode(200);
542                         d->m_response.setHTTPStatusText("OK");
543                     }
544                 }
545             }
546             client->didReceiveResponse(job, d->m_response);
547             CurlCacheManager::getInstance().didReceiveResponse(*job, d->m_response);
548         }
549         d->m_response.setResponseFired(true);
550
551     } else {
552         int splitPos = header.find(":");
553         if (splitPos != -1) {
554             String key = header.left(splitPos).stripWhiteSpace();
555             String value = header.substring(splitPos + 1).stripWhiteSpace();
556
557             if (isAppendableHeader(key))
558                 d->m_response.addHTTPHeaderField(key, value);
559             else
560                 d->m_response.setHTTPHeaderField(key, value);
561         } else if (header.startsWith("HTTP", false)) {
562             // This is the first line of the response.
563             // Extract the http status text from this.
564             //
565             // If the FOLLOWLOCATION option is enabled for the curl handle then
566             // curl will follow the redirections internally. Thus this header callback
567             // will be called more than one time with the line starting "HTTP" for one job.
568             long httpCode = 0;
569             curl_easy_getinfo(d->m_handle, CURLINFO_RESPONSE_CODE, &httpCode);
570
571             String httpCodeString = String::number(httpCode);
572             int statusCodePos = header.find(httpCodeString);
573
574             if (statusCodePos != -1) {
575                 // The status text is after the status code.
576                 String status = header.substring(statusCodePos + httpCodeString.length());
577                 d->m_response.setHTTPStatusText(status.stripWhiteSpace());
578             }
579
580         }
581     }
582
583     return totalSize;
584 }
585
586 /* This is called to obtain HTTP POST or PUT data.
587    Iterate through FormData elements and upload files.
588    Carefully respect the given buffer size and fill the rest of the data at the next calls.
589 */
590 size_t readCallback(void* ptr, size_t size, size_t nmemb, void* data)
591 {
592     ResourceHandle* job = static_cast<ResourceHandle*>(data);
593     ResourceHandleInternal* d = job->getInternal();
594
595     if (d->m_cancelled)
596         return 0;
597
598     // We should never be called when deferred loading is activated.
599     ASSERT(!d->m_defersLoading);
600
601     if (!size || !nmemb)
602         return 0;
603
604     if (!d->m_formDataStream.hasMoreElements())
605         return 0;
606
607     size_t sent = d->m_formDataStream.read(ptr, size, nmemb);
608
609     // Something went wrong so cancel the job.
610     if (!sent)
611         job->cancel();
612
613     return sent;
614 }
615
616 void ResourceHandleManager::downloadTimerCallback()
617 {
618     startScheduledJobs();
619
620     fd_set fdread;
621     fd_set fdwrite;
622     fd_set fdexcep;
623     int maxfd = 0;
624
625     struct timeval timeout;
626     timeout.tv_sec = 0;
627     timeout.tv_usec = selectTimeoutMS * 1000;       // select waits microseconds
628
629     // Retry 'select' if it was interrupted by a process signal.
630     int rc = 0;
631     do {
632         FD_ZERO(&fdread);
633         FD_ZERO(&fdwrite);
634         FD_ZERO(&fdexcep);
635         curl_multi_fdset(m_curlMultiHandle, &fdread, &fdwrite, &fdexcep, &maxfd);
636         // When the 3 file descriptors are empty, winsock will return -1
637         // and bail out, stopping the file download. So make sure we
638         // have valid file descriptors before calling select.
639         if (maxfd >= 0)
640             rc = ::select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
641     } while (rc == -1 && errno == EINTR);
642
643     if (-1 == rc) {
644 #ifndef NDEBUG
645         perror("bad: select() returned -1: ");
646 #endif
647         return;
648     }
649
650     int runningHandles = 0;
651     while (curl_multi_perform(m_curlMultiHandle, &runningHandles) == CURLM_CALL_MULTI_PERFORM) { }
652
653     // check the curl messages indicating completed transfers
654     // and free their resources
655     while (true) {
656         int messagesInQueue;
657         CURLMsg* msg = curl_multi_info_read(m_curlMultiHandle, &messagesInQueue);
658         if (!msg)
659             break;
660
661         // find the node which has same d->m_handle as completed transfer
662         CURL* handle = msg->easy_handle;
663         ASSERT(handle);
664         ResourceHandle* job = 0;
665         CURLcode err = curl_easy_getinfo(handle, CURLINFO_PRIVATE, &job);
666         ASSERT_UNUSED(err, CURLE_OK == err);
667         ASSERT(job);
668         if (!job)
669             continue;
670         ResourceHandleInternal* d = job->getInternal();
671         ASSERT(d->m_handle == handle);
672
673         if (d->m_cancelled) {
674             removeFromCurl(job);
675             continue;
676         }
677
678         if (CURLMSG_DONE != msg->msg)
679             continue;
680
681
682         if (CURLE_OK == msg->data.result) {
683 #if ENABLE(WEB_TIMING)
684             calculateWebTimingInformations(d);
685 #endif
686             if (!d->m_response.responseFired()) {
687                 handleLocalReceiveResponse(d->m_handle, job, d);
688                 if (d->m_cancelled) {
689                     removeFromCurl(job);
690                     continue;
691                 }
692             }
693
694             if (d->m_multipartHandle)
695                 d->m_multipartHandle->contentEnded();
696
697             if (d->client()) {
698                 d->client()->didFinishLoading(job, 0);
699                 CurlCacheManager::getInstance().didFinishLoading(*job);
700             }
701         } else {
702             char* url = 0;
703             curl_easy_getinfo(d->m_handle, CURLINFO_EFFECTIVE_URL, &url);
704 #ifndef NDEBUG
705             fprintf(stderr, "Curl ERROR for url='%s', error: '%s'\n", url, curl_easy_strerror(msg->data.result));
706 #endif
707             if (d->client()) {
708                 ResourceError resourceError(String(), msg->data.result, String(url), String(curl_easy_strerror(msg->data.result)));
709                 resourceError.setSSLErrors(d->m_sslErrors);
710                 d->client()->didFail(job, resourceError);
711                 CurlCacheManager::getInstance().didFail(*job);
712             }
713         }
714
715         removeFromCurl(job);
716     }
717
718     bool started = startScheduledJobs(); // new jobs might have been added in the meantime
719
720     if (!m_downloadTimer.isActive() && (started || (runningHandles > 0)))
721         m_downloadTimer.startOneShot(pollTimeSeconds);
722 }
723
724 void ResourceHandleManager::setProxyInfo(const String& host,
725                                          unsigned long port,
726                                          ProxyType type,
727                                          const String& username,
728                                          const String& password)
729 {
730     m_proxyType = type;
731
732     if (!host.length()) {
733         m_proxy = emptyString();
734     } else {
735         String userPass;
736         if (username.length() || password.length())
737             userPass = username + ":" + password + "@";
738
739         m_proxy = String("http://") + userPass + host + ":" + String::number(port);
740     }
741 }
742
743 void ResourceHandleManager::removeFromCurl(ResourceHandle* job)
744 {
745     ResourceHandleInternal* d = job->getInternal();
746     ASSERT(d->m_handle);
747     if (!d->m_handle)
748         return;
749     m_runningJobs--;
750     curl_multi_remove_handle(m_curlMultiHandle, d->m_handle);
751     curl_easy_cleanup(d->m_handle);
752     d->m_handle = 0;
753     job->deref();
754 }
755
756 static inline size_t getFormElementsCount(ResourceHandle* job)
757 {
758     RefPtr<FormData> formData = job->firstRequest().httpBody();
759
760     if (!formData)
761         return 0;
762
763     // Resolve the blob elements so the formData can correctly report it's size.
764     formData = formData->resolveBlobReferences();
765     job->firstRequest().setHTTPBody(formData);
766
767     return formData->elements().size();
768 }
769
770 static void setupFormData(ResourceHandle* job, CURLoption sizeOption, struct curl_slist** headers)
771 {
772     ResourceHandleInternal* d = job->getInternal();
773     Vector<FormDataElement> elements = job->firstRequest().httpBody()->elements();
774     size_t numElements = elements.size();
775
776     // The size of a curl_off_t could be different in WebKit and in cURL depending on
777     // compilation flags of both.
778     static int expectedSizeOfCurlOffT = 0;
779     if (!expectedSizeOfCurlOffT) {
780         curl_version_info_data *infoData = curl_version_info(CURLVERSION_NOW);
781         if (infoData->features & CURL_VERSION_LARGEFILE)
782             expectedSizeOfCurlOffT = sizeof(long long);
783         else
784             expectedSizeOfCurlOffT = sizeof(int);
785     }
786
787     static const long long maxCurlOffT = (1LL << (expectedSizeOfCurlOffT * 8 - 1)) - 1;
788     // Obtain the total size of the form data
789     curl_off_t size = 0;
790     bool chunkedTransfer = false;
791     for (size_t i = 0; i < numElements; i++) {
792         FormDataElement element = elements[i];
793         if (element.m_type == FormDataElement::Type::EncodedFile) {
794             long long fileSizeResult;
795             if (getFileSize(element.m_filename, fileSizeResult)) {
796                 if (fileSizeResult > maxCurlOffT) {
797                     // File size is too big for specifying it to cURL
798                     chunkedTransfer = true;
799                     break;
800                 }
801                 size += fileSizeResult;
802             } else {
803                 chunkedTransfer = true;
804                 break;
805             }
806         } else
807             size += elements[i].m_data.size();
808     }
809
810     // cURL guesses that we want chunked encoding as long as we specify the header
811     if (chunkedTransfer)
812         *headers = curl_slist_append(*headers, "Transfer-Encoding: chunked");
813     else {
814         if (sizeof(long long) == expectedSizeOfCurlOffT)
815             curl_easy_setopt(d->m_handle, sizeOption, (long long)size);
816         else
817             curl_easy_setopt(d->m_handle, sizeOption, (int)size);
818     }
819
820     curl_easy_setopt(d->m_handle, CURLOPT_READFUNCTION, readCallback);
821     curl_easy_setopt(d->m_handle, CURLOPT_READDATA, job);
822 }
823
824 void ResourceHandleManager::setupPUT(ResourceHandle* job, struct curl_slist** headers)
825 {
826     ResourceHandleInternal* d = job->getInternal();
827     curl_easy_setopt(d->m_handle, CURLOPT_UPLOAD, TRUE);
828     curl_easy_setopt(d->m_handle, CURLOPT_INFILESIZE, 0);
829
830     // Disable the Expect: 100 continue header
831     *headers = curl_slist_append(*headers, "Expect:");
832
833     size_t numElements = getFormElementsCount(job);
834     if (!numElements)
835         return;
836
837     setupFormData(job, CURLOPT_INFILESIZE_LARGE, headers);
838 }
839
840 void ResourceHandleManager::setupPOST(ResourceHandle* job, struct curl_slist** headers)
841 {
842     ResourceHandleInternal* d = job->getInternal();
843     curl_easy_setopt(d->m_handle, CURLOPT_POST, TRUE);
844     curl_easy_setopt(d->m_handle, CURLOPT_POSTFIELDSIZE, 0);
845
846     size_t numElements = getFormElementsCount(job);
847     if (!numElements)
848         return;
849
850     // Do not stream for simple POST data
851     if (numElements == 1) {
852         job->firstRequest().httpBody()->flatten(d->m_postBytes);
853         if (d->m_postBytes.size()) {
854             curl_easy_setopt(d->m_handle, CURLOPT_POSTFIELDSIZE, d->m_postBytes.size());
855             curl_easy_setopt(d->m_handle, CURLOPT_POSTFIELDS, d->m_postBytes.data());
856         }
857         return;
858     }
859
860     setupFormData(job, CURLOPT_POSTFIELDSIZE_LARGE, headers);
861 }
862
863 void ResourceHandleManager::add(ResourceHandle* job)
864 {
865     // we can be called from within curl, so to avoid re-entrancy issues
866     // schedule this job to be added the next time we enter curl download loop
867     job->ref();
868     m_resourceHandleList.append(job);
869     if (!m_downloadTimer.isActive())
870         m_downloadTimer.startOneShot(pollTimeSeconds);
871 }
872
873 bool ResourceHandleManager::removeScheduledJob(ResourceHandle* job)
874 {
875     int size = m_resourceHandleList.size();
876     for (int i = 0; i < size; i++) {
877         if (job == m_resourceHandleList[i]) {
878             m_resourceHandleList.remove(i);
879             job->deref();
880             return true;
881         }
882     }
883     return false;
884 }
885
886 bool ResourceHandleManager::startScheduledJobs()
887 {
888     // TODO: Create a separate stack of jobs for each domain.
889
890     bool started = false;
891     while (!m_resourceHandleList.isEmpty() && m_runningJobs < maxRunningJobs) {
892         ResourceHandle* job = m_resourceHandleList[0];
893         m_resourceHandleList.remove(0);
894         startJob(job);
895         started = true;
896     }
897     return started;
898 }
899
900 void ResourceHandleManager::dispatchSynchronousJob(ResourceHandle* job)
901 {
902     URL kurl = job->firstRequest().url();
903
904     if (kurl.protocolIsData()) {
905         handleDataURL(job);
906         return;
907     }
908
909     ResourceHandleInternal* handle = job->getInternal();
910
911     // If defersLoading is true and we call curl_easy_perform
912     // on a paused handle, libcURL would do the transfert anyway
913     // and we would assert so force defersLoading to be false.
914     handle->m_defersLoading = false;
915
916     initializeHandle(job);
917
918     // curl_easy_perform blocks until the transfert is finished.
919     CURLcode ret =  curl_easy_perform(handle->m_handle);
920
921     if (ret != CURLE_OK) {
922         ResourceError error(String(handle->m_url), ret, String(handle->m_url), String(curl_easy_strerror(ret)));
923         error.setSSLErrors(handle->m_sslErrors);
924         handle->client()->didFail(job, error);
925     } else {
926         if (handle->client())
927             handle->client()->didReceiveResponse(job, handle->m_response);
928     }
929
930 #if ENABLE(WEB_TIMING)
931     calculateWebTimingInformations(handle);
932 #endif
933
934     curl_easy_cleanup(handle->m_handle);
935 }
936
937 void ResourceHandleManager::startJob(ResourceHandle* job)
938 {
939     URL url = job->firstRequest().url();
940
941     if (url.protocolIsData()) {
942         handleDataURL(job);
943         job->deref();
944         return;
945     }
946
947     initializeHandle(job);
948
949     m_runningJobs++;
950     CURLMcode ret = curl_multi_add_handle(m_curlMultiHandle, job->getInternal()->m_handle);
951     // don't call perform, because events must be async
952     // timeout will occur and do curl_multi_perform
953     if (ret && ret != CURLM_CALL_MULTI_PERFORM) {
954 #ifndef NDEBUG
955         fprintf(stderr, "Error %d starting job %s\n", ret, encodeWithURLEscapeSequences(job->firstRequest().url().string()).latin1().data());
956 #endif
957         job->cancel();
958         return;
959     }
960 }
961
962 void ResourceHandleManager::applyAuthenticationToRequest(ResourceHandle* handle, ResourceRequest& request)
963 {
964     // m_user/m_pass are credentials given manually, for instance, by the arguments passed to XMLHttpRequest.open().
965     ResourceHandleInternal* d = handle->getInternal();
966
967     if (handle->shouldUseCredentialStorage()) {
968         if (d->m_user.isEmpty() && d->m_pass.isEmpty()) {
969             // <rdar://problem/7174050> - For URLs that match the paths of those previously challenged for HTTP Basic authentication, 
970             // try and reuse the credential preemptively, as allowed by RFC 2617.
971             d->m_initialCredential = CredentialStorage::defaultCredentialStorage().get(request.url());
972         } else {
973             // If there is already a protection space known for the URL, update stored credentials
974             // before sending a request. This makes it possible to implement logout by sending an
975             // XMLHttpRequest with known incorrect credentials, and aborting it immediately (so that
976             // an authentication dialog doesn't pop up).
977             CredentialStorage::defaultCredentialStorage().set(Credential(d->m_user, d->m_pass, CredentialPersistenceNone), request.url());
978         }
979     }
980
981     String user = d->m_user;
982     String password = d->m_pass;
983
984     if (!d->m_initialCredential.isEmpty()) {
985         user = d->m_initialCredential.user();
986         password = d->m_initialCredential.password();
987         curl_easy_setopt(d->m_handle, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
988     }
989
990     // It seems we need to set CURLOPT_USERPWD even if username and password is empty.
991     // Otherwise cURL will not automatically continue with a new request after a 401 response.
992
993     // curl CURLOPT_USERPWD expects username:password
994     String userpass = user + ":" + password;
995     curl_easy_setopt(d->m_handle, CURLOPT_USERPWD, userpass.utf8().data());
996 }
997
998 void ResourceHandleManager::initializeHandle(ResourceHandle* job)
999 {
1000     static const int allowedProtocols = CURLPROTO_FILE | CURLPROTO_FTP | CURLPROTO_FTPS | CURLPROTO_HTTP | CURLPROTO_HTTPS;
1001     URL url = job->firstRequest().url();
1002
1003     // Remove any fragment part, otherwise curl will send it as part of the request.
1004     url.removeFragmentIdentifier();
1005
1006     ResourceHandleInternal* d = job->getInternal();
1007     String urlString = url.string();
1008
1009     if (url.isLocalFile()) {
1010         // Remove any query part sent to a local file.
1011         if (!url.query().isEmpty()) {
1012             // By setting the query to a null string it'll be removed.
1013             url.setQuery(String());
1014             urlString = url.string();
1015         }
1016         // Determine the MIME type based on the path.
1017         d->m_response.setMimeType(MIMETypeRegistry::getMIMETypeForPath(url));
1018     }
1019
1020     d->m_handle = curl_easy_init();
1021
1022     if (d->m_defersLoading) {
1023         CURLcode error = curl_easy_pause(d->m_handle, CURLPAUSE_ALL);
1024         // If we did not pause the handle, we would ASSERT in the
1025         // header callback. So just assert here.
1026         ASSERT_UNUSED(error, error == CURLE_OK);
1027     }
1028 #ifndef NDEBUG
1029     if (getenv("DEBUG_CURL"))
1030         curl_easy_setopt(d->m_handle, CURLOPT_VERBOSE, 1);
1031     if (m_logFile)
1032         curl_easy_setopt(d->m_handle, CURLOPT_STDERR, m_logFile);
1033 #endif
1034     curl_easy_setopt(d->m_handle, CURLOPT_SSL_VERIFYPEER, 1L);
1035     curl_easy_setopt(d->m_handle, CURLOPT_SSL_VERIFYHOST, 2L);
1036     curl_easy_setopt(d->m_handle, CURLOPT_PRIVATE, job);
1037     curl_easy_setopt(d->m_handle, CURLOPT_ERRORBUFFER, m_curlErrorBuffer);
1038     curl_easy_setopt(d->m_handle, CURLOPT_WRITEFUNCTION, writeCallback);
1039     curl_easy_setopt(d->m_handle, CURLOPT_WRITEDATA, job);
1040     curl_easy_setopt(d->m_handle, CURLOPT_HEADERFUNCTION, headerCallback);
1041     curl_easy_setopt(d->m_handle, CURLOPT_WRITEHEADER, job);
1042     curl_easy_setopt(d->m_handle, CURLOPT_AUTOREFERER, 1);
1043     curl_easy_setopt(d->m_handle, CURLOPT_FOLLOWLOCATION, 1);
1044     curl_easy_setopt(d->m_handle, CURLOPT_MAXREDIRS, 10);
1045     curl_easy_setopt(d->m_handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
1046     curl_easy_setopt(d->m_handle, CURLOPT_SHARE, m_curlShareHandle);
1047     curl_easy_setopt(d->m_handle, CURLOPT_DNS_CACHE_TIMEOUT, 60 * 5); // 5 minutes
1048     curl_easy_setopt(d->m_handle, CURLOPT_PROTOCOLS, allowedProtocols);
1049     curl_easy_setopt(d->m_handle, CURLOPT_REDIR_PROTOCOLS, allowedProtocols);
1050     setSSLClientCertificate(job);
1051
1052     if (ignoreSSLErrors)
1053         curl_easy_setopt(d->m_handle, CURLOPT_SSL_VERIFYPEER, false);
1054     else
1055         setSSLVerifyOptions(job);
1056
1057     if (!m_certificatePath.isNull())
1058        curl_easy_setopt(d->m_handle, CURLOPT_CAINFO, m_certificatePath.data());
1059
1060     // enable gzip and deflate through Accept-Encoding:
1061     curl_easy_setopt(d->m_handle, CURLOPT_ENCODING, "");
1062
1063     // url must remain valid through the request
1064     ASSERT(!d->m_url);
1065
1066     // url is in ASCII so latin1() will only convert it to char* without character translation.
1067     d->m_url = fastStrDup(urlString.latin1().data());
1068     curl_easy_setopt(d->m_handle, CURLOPT_URL, d->m_url);
1069
1070     if (m_cookieJarFileName)
1071         curl_easy_setopt(d->m_handle, CURLOPT_COOKIEJAR, m_cookieJarFileName);
1072
1073     struct curl_slist* headers = 0;
1074     if (job->firstRequest().httpHeaderFields().size() > 0) {
1075         HTTPHeaderMap customHeaders = job->firstRequest().httpHeaderFields();
1076
1077         bool hasCacheHeaders = customHeaders.contains(HTTPHeaderName::IfModifiedSince) || customHeaders.contains(HTTPHeaderName::IfNoneMatch);
1078         if (!hasCacheHeaders && CurlCacheManager::getInstance().isCached(url)) {
1079             CurlCacheManager::getInstance().addCacheEntryClient(url, job);
1080             HTTPHeaderMap& requestHeaders = CurlCacheManager::getInstance().requestHeaders(url);
1081
1082             // append additional cache information
1083             HTTPHeaderMap::const_iterator it = requestHeaders.begin();
1084             HTTPHeaderMap::const_iterator end = requestHeaders.end();
1085             while (it != end) {
1086                 customHeaders.set(it->key, it->value);
1087                 ++it;
1088             }
1089             d->m_addedCacheValidationHeaders = true;
1090         }
1091
1092         HTTPHeaderMap::const_iterator end = customHeaders.end();
1093         for (HTTPHeaderMap::const_iterator it = customHeaders.begin(); it != end; ++it) {
1094             String key = it->key;
1095             String value = it->value;
1096             String headerString(key);
1097             if (value.isEmpty())
1098                 // Insert the ; to tell curl that this header has an empty value.
1099                 headerString.append(";");
1100             else {
1101                 headerString.append(": ");
1102                 headerString.append(value);
1103             }
1104             CString headerLatin1 = headerString.latin1();
1105             headers = curl_slist_append(headers, headerLatin1.data());
1106         }
1107     }
1108
1109     String method = job->firstRequest().httpMethod();
1110     if ("GET" == method)
1111         curl_easy_setopt(d->m_handle, CURLOPT_HTTPGET, TRUE);
1112     else if ("POST" == method)
1113         setupPOST(job, &headers);
1114     else if ("PUT" == method)
1115         setupPUT(job, &headers);
1116     else if ("HEAD" == method)
1117         curl_easy_setopt(d->m_handle, CURLOPT_NOBODY, TRUE);
1118     else {
1119         curl_easy_setopt(d->m_handle, CURLOPT_CUSTOMREQUEST, method.ascii().data());
1120         setupPUT(job, &headers);
1121     }
1122
1123     if (headers) {
1124         curl_easy_setopt(d->m_handle, CURLOPT_HTTPHEADER, headers);
1125         d->m_customHeaders = headers;
1126     }
1127
1128     applyAuthenticationToRequest(job, job->firstRequest());
1129
1130     // Set proxy options if we have them.
1131     if (m_proxy.length()) {
1132         curl_easy_setopt(d->m_handle, CURLOPT_PROXY, m_proxy.utf8().data());
1133         curl_easy_setopt(d->m_handle, CURLOPT_PROXYTYPE, m_proxyType);
1134     }
1135 }
1136
1137 void ResourceHandleManager::initCookieSession()
1138 {
1139     // Curl saves both persistent cookies, and session cookies to the cookie file.
1140     // The session cookies should be deleted before starting a new session.
1141
1142     CURL* curl = curl_easy_init();
1143
1144     if (!curl)
1145         return;
1146
1147     curl_easy_setopt(curl, CURLOPT_SHARE, m_curlShareHandle);
1148
1149     if (m_cookieJarFileName) {
1150         curl_easy_setopt(curl, CURLOPT_COOKIEFILE, m_cookieJarFileName);
1151         curl_easy_setopt(curl, CURLOPT_COOKIEJAR, m_cookieJarFileName);
1152     }
1153
1154     curl_easy_setopt(curl, CURLOPT_COOKIESESSION, 1);
1155
1156     curl_easy_cleanup(curl);
1157 }
1158
1159 void ResourceHandleManager::cancel(ResourceHandle* job)
1160 {
1161     if (removeScheduledJob(job))
1162         return;
1163
1164     ResourceHandleInternal* d = job->getInternal();
1165     d->m_cancelled = true;
1166     if (!m_downloadTimer.isActive())
1167         m_downloadTimer.startOneShot(pollTimeSeconds);
1168 }
1169
1170 } // namespace WebCore
1171
1172 #endif