cd977bb3c07fa1a7e6da8bc562ae553367da7727
[WebKit-https.git] / Source / WebCore / platform / network / curl / ResourceHandleManager.cpp
1 /*
2  * Copyright (C) 2004, 2006 Apple Inc.  All rights reserved.
3  * Copyright (C) 2006 Michael Emmel mike.emmel@gmail.com
4  * Copyright (C) 2007 Alp Toker <alp@atoker.com>
5  * Copyright (C) 2007 Holger Hans Peter Freyther
6  * Copyright (C) 2008 Collabora Ltd.
7  * Copyright (C) 2008 Nuanti Ltd.
8  * Copyright (C) 2009 Appcelerator Inc.
9  * Copyright (C) 2009 Brent Fulgham <bfulgham@webkit.org>
10  * Copyright (C) 2013 Peter Gal <galpeter@inf.u-szeged.hu>, University of Szeged
11  * Copyright (C) 2013 Alex Christensen <achristensen@webkit.org>
12  * Copyright (C) 2013 University of Szeged
13  * All rights reserved.
14  *
15  * Redistribution and use in source and binary forms, with or without
16  * modification, are permitted provided that the following conditions
17  * are met:
18  * 1. Redistributions of source code must retain the above copyright
19  *    notice, this list of conditions and the following disclaimer.
20  * 2. Redistributions in binary form must reproduce the above copyright
21  *    notice, this list of conditions and the following disclaimer in the
22  *    documentation and/or other materials provided with the distribution.
23  *
24  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
25  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
28  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
29  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
30  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
31  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
32  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35  */
36
37 #include "config.h"
38 #include "ResourceHandleManager.h"
39
40 #if USE(CURL)
41
42 #include "CredentialStorage.h"
43 #include "CurlCacheManager.h"
44 #include "DataURL.h"
45 #include "HTTPHeaderNames.h"
46 #include "HTTPParsers.h"
47 #include "MIMETypeRegistry.h"
48 #include "MultipartHandle.h"
49 #include "ResourceError.h"
50 #include "ResourceHandle.h"
51 #include "ResourceHandleInternal.h"
52 #include "SSLHandle.h"
53
54 #if OS(WINDOWS)
55 #include "WebCoreBundleWin.h"
56 #include <shlobj.h>
57 #include <shlwapi.h>
58 #else
59 #include <sys/param.h>
60 #define MAX_PATH MAXPATHLEN
61 #endif
62
63 #include <errno.h>
64 #include <stdio.h>
65 #if ENABLE(WEB_TIMING)
66 #include <wtf/CurrentTime.h>
67 #endif
68 #if USE(CF)
69 #include <wtf/RetainPtr.h>
70 #endif
71 #include <wtf/Lock.h>
72 #include <wtf/Threading.h>
73 #include <wtf/Vector.h>
74 #include <wtf/text/CString.h>
75
76
77 namespace WebCore {
78
79 const int selectTimeoutMS = 5;
80 const double pollTimeSeconds = 0.05;
81 const int maxRunningJobs = 128;
82
83 static const bool ignoreSSLErrors = getenv("WEBKIT_IGNORE_SSL_ERRORS");
84
85 static CString certificatePath()
86 {
87 #if USE(CF)
88     CFBundleRef webKitBundleRef = webKitBundle();
89     if (webKitBundleRef) {
90         RetainPtr<CFURLRef> certURLRef = adoptCF(CFBundleCopyResourceURL(webKitBundleRef, CFSTR("cacert"), CFSTR("pem"), CFSTR("certificates")));
91         if (certURLRef) {
92             char path[MAX_PATH];
93             CFURLGetFileSystemRepresentation(certURLRef.get(), false, reinterpret_cast<UInt8*>(path), MAX_PATH);
94             return path;
95         }
96     }
97 #endif
98     char* envPath = getenv("CURL_CA_BUNDLE_PATH");
99     if (envPath)
100        return envPath;
101
102     return CString();
103 }
104
105 static char* cookieJarPath()
106 {
107     char* cookieJarPath = getenv("CURL_COOKIE_JAR_PATH");
108     if (cookieJarPath)
109         return fastStrDup(cookieJarPath);
110
111 #if OS(WINDOWS)
112     char executablePath[MAX_PATH];
113     char appDataDirectory[MAX_PATH];
114     char cookieJarFullPath[MAX_PATH];
115     char cookieJarDirectory[MAX_PATH];
116
117     if (FAILED(::SHGetFolderPathA(0, CSIDL_LOCAL_APPDATA | CSIDL_FLAG_CREATE, 0, 0, appDataDirectory))
118         || FAILED(::GetModuleFileNameA(0, executablePath, MAX_PATH)))
119         return fastStrDup("cookies.dat");
120
121     ::PathRemoveExtensionA(executablePath);
122     LPSTR executableName = ::PathFindFileNameA(executablePath);
123     sprintf_s(cookieJarDirectory, MAX_PATH, "%s/%s", appDataDirectory, executableName);
124     sprintf_s(cookieJarFullPath, MAX_PATH, "%s/cookies.dat", cookieJarDirectory);
125
126     if (::SHCreateDirectoryExA(0, cookieJarDirectory, 0) != ERROR_SUCCESS
127         && ::GetLastError() != ERROR_FILE_EXISTS
128         && ::GetLastError() != ERROR_ALREADY_EXISTS)
129         return fastStrDup("cookies.dat");
130
131     return fastStrDup(cookieJarFullPath);
132 #else
133     return fastStrDup("cookies.dat");
134 #endif
135 }
136
137 static Lock* sharedResourceMutex(curl_lock_data data)
138 {
139     DEPRECATED_DEFINE_STATIC_LOCAL(Lock, cookieMutex, ());
140     DEPRECATED_DEFINE_STATIC_LOCAL(Lock, dnsMutex, ());
141     DEPRECATED_DEFINE_STATIC_LOCAL(Lock, shareMutex, ());
142
143     switch (data) {
144         case CURL_LOCK_DATA_COOKIE:
145             return &cookieMutex;
146         case CURL_LOCK_DATA_DNS:
147             return &dnsMutex;
148         case CURL_LOCK_DATA_SHARE:
149             return &shareMutex;
150         default:
151             ASSERT_NOT_REACHED();
152             return NULL;
153     }
154 }
155
156 #if ENABLE(WEB_TIMING)
157 static int milisecondsSinceRequest(double requestTime)
158 {
159     return static_cast<int>((monotonicallyIncreasingTime() - requestTime) * 1000.0);
160 }
161
162 static void calculateWebTimingInformations(ResourceHandleInternal* d)
163 {
164     double startTransfertTime = 0;
165     double preTransferTime = 0;
166     double dnslookupTime = 0;
167     double connectTime = 0;
168     double appConnectTime = 0;
169
170     curl_easy_getinfo(d->m_handle, CURLINFO_NAMELOOKUP_TIME, &dnslookupTime);
171     curl_easy_getinfo(d->m_handle, CURLINFO_CONNECT_TIME, &connectTime);
172     curl_easy_getinfo(d->m_handle, CURLINFO_APPCONNECT_TIME, &appConnectTime);
173     curl_easy_getinfo(d->m_handle, CURLINFO_STARTTRANSFER_TIME, &startTransfertTime);
174     curl_easy_getinfo(d->m_handle, CURLINFO_PRETRANSFER_TIME, &preTransferTime);
175
176     d->m_response.resourceLoadTiming().domainLookupStart = 0;
177     d->m_response.resourceLoadTiming().domainLookupEnd = static_cast<int>(dnslookupTime * 1000);
178
179     d->m_response.resourceLoadTiming().connectStart = static_cast<int>(dnslookupTime * 1000);
180     d->m_response.resourceLoadTiming().connectEnd = static_cast<int>(connectTime * 1000);
181
182     d->m_response.resourceLoadTiming().requestStart = static_cast<int>(connectTime *1000);
183     d->m_response.resourceLoadTiming().responseStart =static_cast<int>(preTransferTime * 1000);
184
185     if (appConnectTime)
186         d->m_response.resourceLoadTiming().secureConnectionStart = static_cast<int>(connectTime * 1000);
187 }
188 #endif
189
190 // libcurl does not implement its own thread synchronization primitives.
191 // these two functions provide mutexes for cookies, and for the global DNS
192 // cache.
193 static void curl_lock_callback(CURL* /* handle */, curl_lock_data data, curl_lock_access /* access */, void* /* userPtr */)
194 {
195     if (Lock* mutex = sharedResourceMutex(data))
196         mutex->lock();
197 }
198
199 static void curl_unlock_callback(CURL* /* handle */, curl_lock_data data, void* /* userPtr */)
200 {
201     if (Lock* mutex = sharedResourceMutex(data))
202         mutex->unlock();
203 }
204
205 inline static bool isHttpInfo(int statusCode)
206 {
207     return 100 <= statusCode && statusCode < 200;
208 }
209
210 inline static bool isHttpRedirect(int statusCode)
211 {
212     return 300 <= statusCode && statusCode < 400 && statusCode != 304;
213 }
214
215 inline static bool isHttpAuthentication(int statusCode)
216 {
217     return statusCode == 401;
218 }
219
220 inline static bool isHttpNotModified(int statusCode)
221 {
222     return statusCode == 304;
223 }
224
225 ResourceHandleManager::ResourceHandleManager()
226     : m_downloadTimer(*this, &ResourceHandleManager::downloadTimerCallback)
227     , m_cookieJarFileName(cookieJarPath())
228     , m_certificatePath (certificatePath())
229     , m_runningJobs(0)
230 #ifndef NDEBUG
231     , m_logFile(nullptr)
232 #endif
233 {
234     curl_global_init(CURL_GLOBAL_ALL);
235     m_curlMultiHandle = curl_multi_init();
236     m_curlShareHandle = curl_share_init();
237     curl_share_setopt(m_curlShareHandle, CURLSHOPT_SHARE, CURL_LOCK_DATA_COOKIE);
238     curl_share_setopt(m_curlShareHandle, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS);
239     curl_share_setopt(m_curlShareHandle, CURLSHOPT_LOCKFUNC, curl_lock_callback);
240     curl_share_setopt(m_curlShareHandle, CURLSHOPT_UNLOCKFUNC, curl_unlock_callback);
241
242     initCookieSession();
243
244 #ifndef NDEBUG
245     char* logFile = getenv("CURL_LOG_FILE");
246     if (logFile)
247         m_logFile = fopen(logFile, "a");
248 #endif
249 }
250
251 ResourceHandleManager::~ResourceHandleManager()
252 {
253     curl_multi_cleanup(m_curlMultiHandle);
254     curl_share_cleanup(m_curlShareHandle);
255     if (m_cookieJarFileName)
256         fastFree(m_cookieJarFileName);
257     curl_global_cleanup();
258
259 #ifndef NDEBUG
260     if (m_logFile)
261         fclose(m_logFile);
262 #endif
263 }
264
265 CURLSH* ResourceHandleManager::getCurlShareHandle() const
266 {
267     return m_curlShareHandle;
268 }
269
270 void ResourceHandleManager::setCookieJarFileName(const char* cookieJarFileName)
271 {
272     m_cookieJarFileName = fastStrDup(cookieJarFileName);
273 }
274
275 const char* ResourceHandleManager::getCookieJarFileName() const
276 {
277     return m_cookieJarFileName;
278 }
279
280 ResourceHandleManager* ResourceHandleManager::sharedInstance()
281 {
282     static ResourceHandleManager* sharedInstance = 0;
283     if (!sharedInstance)
284         sharedInstance = new ResourceHandleManager();
285     return sharedInstance;
286 }
287
288 static void handleLocalReceiveResponse (CURL* handle, ResourceHandle* job, ResourceHandleInternal* d)
289 {
290     // since the code in headerCallback will not have run for local files
291     // the code to set the URL and fire didReceiveResponse is never run,
292     // which means the ResourceLoader's response does not contain the URL.
293     // Run the code here for local files to resolve the issue.
294     // TODO: See if there is a better approach for handling this.
295      const char* hdr;
296      CURLcode err = curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &hdr);
297      ASSERT_UNUSED(err, CURLE_OK == err);
298      d->m_response.setURL(URL(ParsedURLString, hdr));
299      if (d->client())
300          d->client()->didReceiveResponse(job, d->m_response);
301      d->m_response.setResponseFired(true);
302 }
303
304
305 // called with data after all headers have been processed via headerCallback
306 static size_t writeCallback(void* ptr, size_t size, size_t nmemb, void* data)
307 {
308     ResourceHandle* job = static_cast<ResourceHandle*>(data);
309     ResourceHandleInternal* d = job->getInternal();
310     if (d->m_cancelled)
311         return 0;
312
313     // We should never be called when deferred loading is activated.
314     ASSERT(!d->m_defersLoading);
315
316     size_t totalSize = size * nmemb;
317
318     // this shouldn't be necessary but apparently is. CURL writes the data
319     // of html page even if it is a redirect that was handled internally
320     // can be observed e.g. on gmail.com
321     CURL* h = d->m_handle;
322     long httpCode = 0;
323     CURLcode err = curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &httpCode);
324     if (CURLE_OK == err && httpCode >= 300 && httpCode < 400)
325         return totalSize;
326
327     if (!d->m_response.responseFired()) {
328         handleLocalReceiveResponse(h, job, d);
329         if (d->m_cancelled)
330             return 0;
331     }
332
333     if (d->m_multipartHandle)
334         d->m_multipartHandle->contentReceived(static_cast<const char*>(ptr), totalSize);
335     else if (d->client()) {
336         d->client()->didReceiveData(job, static_cast<char*>(ptr), totalSize, 0);
337         CurlCacheManager::getInstance().didReceiveData(*job, static_cast<char*>(ptr), totalSize);
338     }
339
340     return totalSize;
341 }
342
343 static bool isAppendableHeader(const String &key)
344 {
345     static const char* appendableHeaders[] = {
346         "access-control-allow-headers",
347         "access-control-allow-methods",
348         "access-control-allow-origin",
349         "access-control-expose-headers",
350         "allow",
351         "cache-control",
352         "connection",
353         "content-encoding",
354         "content-language",
355         "if-match",
356         "if-none-match",
357         "keep-alive",
358         "pragma",
359         "proxy-authenticate",
360         "public",
361         "server",
362         "set-cookie",
363         "te",
364         "trailer",
365         "transfer-encoding",
366         "upgrade",
367         "user-agent",
368         "vary",
369         "via",
370         "warning",
371         "www-authenticate",
372         0
373     };
374
375     // Custom headers start with 'X-', and need no further checking.
376     if (key.startsWith("x-", /* caseSensitive */ false))
377         return true;
378
379     for (unsigned i = 0; appendableHeaders[i]; ++i)
380         if (equalIgnoringCase(key, appendableHeaders[i]))
381             return true;
382
383     return false;
384 }
385
386 static void removeLeadingAndTrailingQuotes(String& value)
387 {
388     unsigned length = value.length();
389     if (value.startsWith('"') && value.endsWith('"') && length > 1)
390         value = value.substring(1, length-2);
391 }
392
393 static bool getProtectionSpace(CURL* h, const ResourceResponse& response, ProtectionSpace& protectionSpace)
394 {
395     CURLcode err;
396
397     long port = 0;
398     err = curl_easy_getinfo(h, CURLINFO_PRIMARY_PORT, &port);
399     if (err != CURLE_OK)
400         return false;
401
402     long availableAuth = CURLAUTH_NONE;
403     err = curl_easy_getinfo(h, CURLINFO_HTTPAUTH_AVAIL, &availableAuth);
404     if (err != CURLE_OK)
405         return false;
406
407     const char* effectiveUrl = 0;
408     err = curl_easy_getinfo(h, CURLINFO_EFFECTIVE_URL, &effectiveUrl);
409     if (err != CURLE_OK)
410         return false;
411
412     URL url(ParsedURLString, effectiveUrl);
413
414     String host = url.host();
415     String protocol = url.protocol();
416
417     String realm;
418
419     const String authHeader = response.httpHeaderField(HTTPHeaderName::Authorization);
420     const String realmString = "realm=";
421     int realmPos = authHeader.find(realmString);
422     if (realmPos > 0) {
423         realm = authHeader.substring(realmPos + realmString.length());
424         realm = realm.left(realm.find(','));
425         removeLeadingAndTrailingQuotes(realm);
426     }
427
428     ProtectionSpaceServerType serverType = ProtectionSpaceServerHTTP;
429     if (protocol == "https")
430         serverType = ProtectionSpaceServerHTTPS;
431
432     ProtectionSpaceAuthenticationScheme authScheme = ProtectionSpaceAuthenticationSchemeUnknown;
433
434     if (availableAuth & CURLAUTH_BASIC)
435         authScheme = ProtectionSpaceAuthenticationSchemeHTTPBasic;
436     if (availableAuth & CURLAUTH_DIGEST)
437         authScheme = ProtectionSpaceAuthenticationSchemeHTTPDigest;
438     if (availableAuth & CURLAUTH_GSSNEGOTIATE)
439         authScheme = ProtectionSpaceAuthenticationSchemeNegotiate;
440     if (availableAuth & CURLAUTH_NTLM)
441         authScheme = ProtectionSpaceAuthenticationSchemeNTLM;
442
443     protectionSpace = ProtectionSpace(host, port, serverType, realm, authScheme);
444
445     return true;
446 }
447
448 /*
449  * This is being called for each HTTP header in the response. This includes '\r\n'
450  * for the last line of the header.
451  *
452  * We will add each HTTP Header to the ResourceResponse and on the termination
453  * of the header (\r\n) we will parse Content-Type and Content-Disposition and
454  * update the ResourceResponse and then send it away.
455  *
456  */
457 static size_t headerCallback(char* ptr, size_t size, size_t nmemb, void* data)
458 {
459     ResourceHandle* job = static_cast<ResourceHandle*>(data);
460     ResourceHandleInternal* d = job->getInternal();
461     if (d->m_cancelled)
462         return 0;
463
464     // We should never be called when deferred loading is activated.
465     ASSERT(!d->m_defersLoading);
466
467     size_t totalSize = size * nmemb;
468     ResourceHandleClient* client = d->client();
469
470     String header(static_cast<const char*>(ptr), totalSize);
471
472     /*
473      * a) We can finish and send the ResourceResponse
474      * b) We will add the current header to the HTTPHeaderMap of the ResourceResponse
475      *
476      * The HTTP standard requires to use \r\n but for compatibility it recommends to
477      * accept also \n.
478      */
479     if (header == String("\r\n") || header == String("\n")) {
480         CURL* h = d->m_handle;
481
482         long httpCode = 0;
483         curl_easy_getinfo(h, CURLINFO_RESPONSE_CODE, &httpCode);
484
485         if (isHttpInfo(httpCode)) {
486             // Just return when receiving http info, e.g. HTTP/1.1 100 Continue.
487             // If not, the request might be cancelled, because the MIME type will be empty for this response.
488             return totalSize;
489         }
490
491         double contentLength = 0;
492         curl_easy_getinfo(h, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &contentLength);
493         d->m_response.setExpectedContentLength(static_cast<long long int>(contentLength));
494
495         const char* hdr;
496         curl_easy_getinfo(h, CURLINFO_EFFECTIVE_URL, &hdr);
497         d->m_response.setURL(URL(ParsedURLString, hdr));
498
499         d->m_response.setHTTPStatusCode(httpCode);
500         d->m_response.setMimeType(extractMIMETypeFromMediaType(d->m_response.httpHeaderField(HTTPHeaderName::ContentType)).lower());
501         d->m_response.setTextEncodingName(extractCharsetFromMediaType(d->m_response.httpHeaderField(HTTPHeaderName::ContentType)));
502
503         if (d->m_response.isMultipart()) {
504             String boundary;
505             bool parsed = MultipartHandle::extractBoundary(d->m_response.httpHeaderField(HTTPHeaderName::ContentType), boundary);
506             if (parsed)
507                 d->m_multipartHandle = std::make_unique<MultipartHandle>(job, boundary);
508         }
509
510         // HTTP redirection
511         if (isHttpRedirect(httpCode)) {
512             String location = d->m_response.httpHeaderField(HTTPHeaderName::Location);
513             if (!location.isEmpty()) {
514                 URL newURL = URL(job->firstRequest().url(), location);
515
516                 ResourceRequest redirectedRequest = job->firstRequest();
517                 redirectedRequest.setURL(newURL);
518                 if (client)
519                     client->willSendRequest(job, redirectedRequest, d->m_response);
520
521                 d->m_firstRequest.setURL(newURL);
522
523                 return totalSize;
524             }
525         } else if (isHttpAuthentication(httpCode)) {
526             ProtectionSpace protectionSpace;
527             if (getProtectionSpace(d->m_handle, d->m_response, protectionSpace)) {
528                 Credential credential;
529                 AuthenticationChallenge challenge(protectionSpace, credential, d->m_authFailureCount, d->m_response, ResourceError());
530                 challenge.setAuthenticationClient(job);
531                 job->didReceiveAuthenticationChallenge(challenge);
532                 d->m_authFailureCount++;
533                 return totalSize;
534             }
535         }
536
537         if (client) {
538             if (isHttpNotModified(httpCode)) {
539                 const String& url = job->firstRequest().url().string();
540                 if (CurlCacheManager::getInstance().getCachedResponse(url, d->m_response)) {
541                     if (d->m_addedCacheValidationHeaders) {
542                         d->m_response.setHTTPStatusCode(200);
543                         d->m_response.setHTTPStatusText("OK");
544                     }
545                 }
546             }
547             client->didReceiveResponse(job, d->m_response);
548             CurlCacheManager::getInstance().didReceiveResponse(*job, d->m_response);
549         }
550         d->m_response.setResponseFired(true);
551
552     } else {
553         int splitPos = header.find(":");
554         if (splitPos != -1) {
555             String key = header.left(splitPos).stripWhiteSpace();
556             String value = header.substring(splitPos + 1).stripWhiteSpace();
557
558             if (isAppendableHeader(key))
559                 d->m_response.addHTTPHeaderField(key, value);
560             else
561                 d->m_response.setHTTPHeaderField(key, value);
562         } else if (header.startsWith("HTTP", false)) {
563             // This is the first line of the response.
564             // Extract the http status text from this.
565             //
566             // If the FOLLOWLOCATION option is enabled for the curl handle then
567             // curl will follow the redirections internally. Thus this header callback
568             // will be called more than one time with the line starting "HTTP" for one job.
569             long httpCode = 0;
570             curl_easy_getinfo(d->m_handle, CURLINFO_RESPONSE_CODE, &httpCode);
571
572             String httpCodeString = String::number(httpCode);
573             int statusCodePos = header.find(httpCodeString);
574
575             if (statusCodePos != -1) {
576                 // The status text is after the status code.
577                 String status = header.substring(statusCodePos + httpCodeString.length());
578                 d->m_response.setHTTPStatusText(status.stripWhiteSpace());
579             }
580
581         }
582     }
583
584     return totalSize;
585 }
586
587 /* This is called to obtain HTTP POST or PUT data.
588    Iterate through FormData elements and upload files.
589    Carefully respect the given buffer size and fill the rest of the data at the next calls.
590 */
591 size_t readCallback(void* ptr, size_t size, size_t nmemb, void* data)
592 {
593     ResourceHandle* job = static_cast<ResourceHandle*>(data);
594     ResourceHandleInternal* d = job->getInternal();
595
596     if (d->m_cancelled)
597         return 0;
598
599     // We should never be called when deferred loading is activated.
600     ASSERT(!d->m_defersLoading);
601
602     if (!size || !nmemb)
603         return 0;
604
605     if (!d->m_formDataStream.hasMoreElements())
606         return 0;
607
608     size_t sent = d->m_formDataStream.read(ptr, size, nmemb);
609
610     // Something went wrong so cancel the job.
611     if (!sent)
612         job->cancel();
613
614     return sent;
615 }
616
617 void ResourceHandleManager::downloadTimerCallback()
618 {
619     startScheduledJobs();
620
621     fd_set fdread;
622     fd_set fdwrite;
623     fd_set fdexcep;
624     int maxfd = 0;
625
626     struct timeval timeout;
627     timeout.tv_sec = 0;
628     timeout.tv_usec = selectTimeoutMS * 1000;       // select waits microseconds
629
630     // Retry 'select' if it was interrupted by a process signal.
631     int rc = 0;
632     do {
633         FD_ZERO(&fdread);
634         FD_ZERO(&fdwrite);
635         FD_ZERO(&fdexcep);
636         curl_multi_fdset(m_curlMultiHandle, &fdread, &fdwrite, &fdexcep, &maxfd);
637         // When the 3 file descriptors are empty, winsock will return -1
638         // and bail out, stopping the file download. So make sure we
639         // have valid file descriptors before calling select.
640         if (maxfd >= 0)
641             rc = ::select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
642     } while (rc == -1 && errno == EINTR);
643
644     if (-1 == rc) {
645 #ifndef NDEBUG
646         perror("bad: select() returned -1: ");
647 #endif
648         return;
649     }
650
651     int runningHandles = 0;
652     while (curl_multi_perform(m_curlMultiHandle, &runningHandles) == CURLM_CALL_MULTI_PERFORM) { }
653
654     // check the curl messages indicating completed transfers
655     // and free their resources
656     while (true) {
657         int messagesInQueue;
658         CURLMsg* msg = curl_multi_info_read(m_curlMultiHandle, &messagesInQueue);
659         if (!msg)
660             break;
661
662         // find the node which has same d->m_handle as completed transfer
663         CURL* handle = msg->easy_handle;
664         ASSERT(handle);
665         ResourceHandle* job = 0;
666         CURLcode err = curl_easy_getinfo(handle, CURLINFO_PRIVATE, &job);
667         ASSERT_UNUSED(err, CURLE_OK == err);
668         ASSERT(job);
669         if (!job)
670             continue;
671         ResourceHandleInternal* d = job->getInternal();
672         ASSERT(d->m_handle == handle);
673
674         if (d->m_cancelled) {
675             removeFromCurl(job);
676             continue;
677         }
678
679         if (CURLMSG_DONE != msg->msg)
680             continue;
681
682
683         if (CURLE_OK == msg->data.result) {
684 #if ENABLE(WEB_TIMING)
685             calculateWebTimingInformations(d);
686 #endif
687             if (!d->m_response.responseFired()) {
688                 handleLocalReceiveResponse(d->m_handle, job, d);
689                 if (d->m_cancelled) {
690                     removeFromCurl(job);
691                     continue;
692                 }
693             }
694
695             if (d->m_multipartHandle)
696                 d->m_multipartHandle->contentEnded();
697
698             if (d->client()) {
699                 d->client()->didFinishLoading(job, 0);
700                 CurlCacheManager::getInstance().didFinishLoading(*job);
701             }
702         } else {
703             char* url = 0;
704             curl_easy_getinfo(d->m_handle, CURLINFO_EFFECTIVE_URL, &url);
705 #ifndef NDEBUG
706             fprintf(stderr, "Curl ERROR for url='%s', error: '%s'\n", url, curl_easy_strerror(msg->data.result));
707 #endif
708             if (d->client()) {
709                 ResourceError resourceError(String(), msg->data.result, URL(ParsedURLString, String(url)), String(curl_easy_strerror(msg->data.result)));
710                 resourceError.setSSLErrors(d->m_sslErrors);
711                 d->client()->didFail(job, resourceError);
712                 CurlCacheManager::getInstance().didFail(*job);
713             }
714         }
715
716         removeFromCurl(job);
717     }
718
719     bool started = startScheduledJobs(); // new jobs might have been added in the meantime
720
721     if (!m_downloadTimer.isActive() && (started || (runningHandles > 0)))
722         m_downloadTimer.startOneShot(pollTimeSeconds);
723 }
724
725 void ResourceHandleManager::setProxyInfo(const String& host,
726                                          unsigned long port,
727                                          ProxyType type,
728                                          const String& username,
729                                          const String& password)
730 {
731     m_proxyType = type;
732
733     if (!host.length()) {
734         m_proxy = emptyString();
735     } else {
736         String userPass;
737         if (username.length() || password.length())
738             userPass = username + ":" + password + "@";
739
740         m_proxy = String("http://") + userPass + host + ":" + String::number(port);
741     }
742 }
743
744 void ResourceHandleManager::removeFromCurl(ResourceHandle* job)
745 {
746     ResourceHandleInternal* d = job->getInternal();
747     ASSERT(d->m_handle);
748     if (!d->m_handle)
749         return;
750     m_runningJobs--;
751     curl_multi_remove_handle(m_curlMultiHandle, d->m_handle);
752     curl_easy_cleanup(d->m_handle);
753     d->m_handle = 0;
754     job->deref();
755 }
756
757 static inline size_t getFormElementsCount(ResourceHandle* job)
758 {
759     RefPtr<FormData> formData = job->firstRequest().httpBody();
760
761     if (!formData)
762         return 0;
763
764     // Resolve the blob elements so the formData can correctly report it's size.
765     formData = formData->resolveBlobReferences();
766     job->firstRequest().setHTTPBody(formData);
767
768     return formData->elements().size();
769 }
770
771 static void setupFormData(ResourceHandle* job, CURLoption sizeOption, struct curl_slist** headers)
772 {
773     ResourceHandleInternal* d = job->getInternal();
774     Vector<FormDataElement> elements = job->firstRequest().httpBody()->elements();
775     size_t numElements = elements.size();
776
777     // The size of a curl_off_t could be different in WebKit and in cURL depending on
778     // compilation flags of both.
779     static int expectedSizeOfCurlOffT = 0;
780     if (!expectedSizeOfCurlOffT) {
781         curl_version_info_data *infoData = curl_version_info(CURLVERSION_NOW);
782         if (infoData->features & CURL_VERSION_LARGEFILE)
783             expectedSizeOfCurlOffT = sizeof(long long);
784         else
785             expectedSizeOfCurlOffT = sizeof(int);
786     }
787
788     static const long long maxCurlOffT = (1LL << (expectedSizeOfCurlOffT * 8 - 1)) - 1;
789     // Obtain the total size of the form data
790     curl_off_t size = 0;
791     bool chunkedTransfer = false;
792     for (size_t i = 0; i < numElements; i++) {
793         FormDataElement element = elements[i];
794         if (element.m_type == FormDataElement::Type::EncodedFile) {
795             long long fileSizeResult;
796             if (getFileSize(element.m_filename, fileSizeResult)) {
797                 if (fileSizeResult > maxCurlOffT) {
798                     // File size is too big for specifying it to cURL
799                     chunkedTransfer = true;
800                     break;
801                 }
802                 size += fileSizeResult;
803             } else {
804                 chunkedTransfer = true;
805                 break;
806             }
807         } else
808             size += elements[i].m_data.size();
809     }
810
811     // cURL guesses that we want chunked encoding as long as we specify the header
812     if (chunkedTransfer)
813         *headers = curl_slist_append(*headers, "Transfer-Encoding: chunked");
814     else {
815         if (sizeof(long long) == expectedSizeOfCurlOffT)
816             curl_easy_setopt(d->m_handle, sizeOption, (long long)size);
817         else
818             curl_easy_setopt(d->m_handle, sizeOption, (int)size);
819     }
820
821     curl_easy_setopt(d->m_handle, CURLOPT_READFUNCTION, readCallback);
822     curl_easy_setopt(d->m_handle, CURLOPT_READDATA, job);
823 }
824
825 void ResourceHandleManager::setupPUT(ResourceHandle* job, struct curl_slist** headers)
826 {
827     ResourceHandleInternal* d = job->getInternal();
828     curl_easy_setopt(d->m_handle, CURLOPT_UPLOAD, TRUE);
829     curl_easy_setopt(d->m_handle, CURLOPT_INFILESIZE, 0);
830
831     // Disable the Expect: 100 continue header
832     *headers = curl_slist_append(*headers, "Expect:");
833
834     size_t numElements = getFormElementsCount(job);
835     if (!numElements)
836         return;
837
838     setupFormData(job, CURLOPT_INFILESIZE_LARGE, headers);
839 }
840
841 void ResourceHandleManager::setupPOST(ResourceHandle* job, struct curl_slist** headers)
842 {
843     ResourceHandleInternal* d = job->getInternal();
844     curl_easy_setopt(d->m_handle, CURLOPT_POST, TRUE);
845     curl_easy_setopt(d->m_handle, CURLOPT_POSTFIELDSIZE, 0);
846
847     size_t numElements = getFormElementsCount(job);
848     if (!numElements)
849         return;
850
851     // Do not stream for simple POST data
852     if (numElements == 1) {
853         job->firstRequest().httpBody()->flatten(d->m_postBytes);
854         if (d->m_postBytes.size()) {
855             curl_easy_setopt(d->m_handle, CURLOPT_POSTFIELDSIZE, d->m_postBytes.size());
856             curl_easy_setopt(d->m_handle, CURLOPT_POSTFIELDS, d->m_postBytes.data());
857         }
858         return;
859     }
860
861     setupFormData(job, CURLOPT_POSTFIELDSIZE_LARGE, headers);
862 }
863
864 void ResourceHandleManager::add(ResourceHandle* job)
865 {
866     // we can be called from within curl, so to avoid re-entrancy issues
867     // schedule this job to be added the next time we enter curl download loop
868     job->ref();
869     m_resourceHandleList.append(job);
870     if (!m_downloadTimer.isActive())
871         m_downloadTimer.startOneShot(pollTimeSeconds);
872 }
873
874 bool ResourceHandleManager::removeScheduledJob(ResourceHandle* job)
875 {
876     int size = m_resourceHandleList.size();
877     for (int i = 0; i < size; i++) {
878         if (job == m_resourceHandleList[i]) {
879             m_resourceHandleList.remove(i);
880             job->deref();
881             return true;
882         }
883     }
884     return false;
885 }
886
887 bool ResourceHandleManager::startScheduledJobs()
888 {
889     // TODO: Create a separate stack of jobs for each domain.
890
891     bool started = false;
892     while (!m_resourceHandleList.isEmpty() && m_runningJobs < maxRunningJobs) {
893         ResourceHandle* job = m_resourceHandleList[0];
894         m_resourceHandleList.remove(0);
895         startJob(job);
896         started = true;
897     }
898     return started;
899 }
900
901 void ResourceHandleManager::dispatchSynchronousJob(ResourceHandle* job)
902 {
903     URL kurl = job->firstRequest().url();
904
905     if (kurl.protocolIsData()) {
906         handleDataURL(job);
907         return;
908     }
909
910     ResourceHandleInternal* handle = job->getInternal();
911
912     // If defersLoading is true and we call curl_easy_perform
913     // on a paused handle, libcURL would do the transfert anyway
914     // and we would assert so force defersLoading to be false.
915     handle->m_defersLoading = false;
916
917     initializeHandle(job);
918
919     // curl_easy_perform blocks until the transfert is finished.
920     CURLcode ret =  curl_easy_perform(handle->m_handle);
921
922     if (ret != CURLE_OK) {
923         ResourceError error(String(handle->m_url), ret, kurl, String(curl_easy_strerror(ret)));
924         error.setSSLErrors(handle->m_sslErrors);
925         handle->client()->didFail(job, error);
926     } else {
927         if (handle->client())
928             handle->client()->didReceiveResponse(job, handle->m_response);
929     }
930
931 #if ENABLE(WEB_TIMING)
932     calculateWebTimingInformations(handle);
933 #endif
934
935     curl_easy_cleanup(handle->m_handle);
936 }
937
938 void ResourceHandleManager::startJob(ResourceHandle* job)
939 {
940     URL url = job->firstRequest().url();
941
942     if (url.protocolIsData()) {
943         handleDataURL(job);
944         job->deref();
945         return;
946     }
947
948     initializeHandle(job);
949
950     m_runningJobs++;
951     CURLMcode ret = curl_multi_add_handle(m_curlMultiHandle, job->getInternal()->m_handle);
952     // don't call perform, because events must be async
953     // timeout will occur and do curl_multi_perform
954     if (ret && ret != CURLM_CALL_MULTI_PERFORM) {
955 #ifndef NDEBUG
956         fprintf(stderr, "Error %d starting job %s\n", ret, encodeWithURLEscapeSequences(job->firstRequest().url().string()).latin1().data());
957 #endif
958         job->cancel();
959         return;
960     }
961 }
962
963 void ResourceHandleManager::applyAuthenticationToRequest(ResourceHandle* handle, ResourceRequest& request)
964 {
965     // m_user/m_pass are credentials given manually, for instance, by the arguments passed to XMLHttpRequest.open().
966     ResourceHandleInternal* d = handle->getInternal();
967
968     if (handle->shouldUseCredentialStorage()) {
969         if (d->m_user.isEmpty() && d->m_pass.isEmpty()) {
970             // <rdar://problem/7174050> - For URLs that match the paths of those previously challenged for HTTP Basic authentication, 
971             // try and reuse the credential preemptively, as allowed by RFC 2617.
972             d->m_initialCredential = CredentialStorage::defaultCredentialStorage().get(request.url());
973         } else {
974             // If there is already a protection space known for the URL, update stored credentials
975             // before sending a request. This makes it possible to implement logout by sending an
976             // XMLHttpRequest with known incorrect credentials, and aborting it immediately (so that
977             // an authentication dialog doesn't pop up).
978             CredentialStorage::defaultCredentialStorage().set(Credential(d->m_user, d->m_pass, CredentialPersistenceNone), request.url());
979         }
980     }
981
982     String user = d->m_user;
983     String password = d->m_pass;
984
985     if (!d->m_initialCredential.isEmpty()) {
986         user = d->m_initialCredential.user();
987         password = d->m_initialCredential.password();
988         curl_easy_setopt(d->m_handle, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
989     }
990
991     // It seems we need to set CURLOPT_USERPWD even if username and password is empty.
992     // Otherwise cURL will not automatically continue with a new request after a 401 response.
993
994     // curl CURLOPT_USERPWD expects username:password
995     String userpass = user + ":" + password;
996     curl_easy_setopt(d->m_handle, CURLOPT_USERPWD, userpass.utf8().data());
997 }
998
999 void ResourceHandleManager::initializeHandle(ResourceHandle* job)
1000 {
1001     static const int allowedProtocols = CURLPROTO_FILE | CURLPROTO_FTP | CURLPROTO_FTPS | CURLPROTO_HTTP | CURLPROTO_HTTPS;
1002     URL url = job->firstRequest().url();
1003
1004     // Remove any fragment part, otherwise curl will send it as part of the request.
1005     url.removeFragmentIdentifier();
1006
1007     ResourceHandleInternal* d = job->getInternal();
1008     String urlString = url.string();
1009
1010     if (url.isLocalFile()) {
1011         // Remove any query part sent to a local file.
1012         if (!url.query().isEmpty()) {
1013             // By setting the query to a null string it'll be removed.
1014             url.setQuery(String());
1015             urlString = url.string();
1016         }
1017         // Determine the MIME type based on the path.
1018         d->m_response.setMimeType(MIMETypeRegistry::getMIMETypeForPath(url));
1019     }
1020
1021     d->m_handle = curl_easy_init();
1022
1023     if (d->m_defersLoading) {
1024         CURLcode error = curl_easy_pause(d->m_handle, CURLPAUSE_ALL);
1025         // If we did not pause the handle, we would ASSERT in the
1026         // header callback. So just assert here.
1027         ASSERT_UNUSED(error, error == CURLE_OK);
1028     }
1029 #ifndef NDEBUG
1030     if (getenv("DEBUG_CURL"))
1031         curl_easy_setopt(d->m_handle, CURLOPT_VERBOSE, 1);
1032     if (m_logFile)
1033         curl_easy_setopt(d->m_handle, CURLOPT_STDERR, m_logFile);
1034 #endif
1035     curl_easy_setopt(d->m_handle, CURLOPT_SSL_VERIFYPEER, 1L);
1036     curl_easy_setopt(d->m_handle, CURLOPT_SSL_VERIFYHOST, 2L);
1037     curl_easy_setopt(d->m_handle, CURLOPT_PRIVATE, job);
1038     curl_easy_setopt(d->m_handle, CURLOPT_ERRORBUFFER, m_curlErrorBuffer);
1039     curl_easy_setopt(d->m_handle, CURLOPT_WRITEFUNCTION, writeCallback);
1040     curl_easy_setopt(d->m_handle, CURLOPT_WRITEDATA, job);
1041     curl_easy_setopt(d->m_handle, CURLOPT_HEADERFUNCTION, headerCallback);
1042     curl_easy_setopt(d->m_handle, CURLOPT_WRITEHEADER, job);
1043     curl_easy_setopt(d->m_handle, CURLOPT_AUTOREFERER, 1);
1044     curl_easy_setopt(d->m_handle, CURLOPT_FOLLOWLOCATION, 1);
1045     curl_easy_setopt(d->m_handle, CURLOPT_MAXREDIRS, 10);
1046     curl_easy_setopt(d->m_handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
1047     curl_easy_setopt(d->m_handle, CURLOPT_SHARE, m_curlShareHandle);
1048     curl_easy_setopt(d->m_handle, CURLOPT_DNS_CACHE_TIMEOUT, 60 * 5); // 5 minutes
1049     curl_easy_setopt(d->m_handle, CURLOPT_PROTOCOLS, allowedProtocols);
1050     curl_easy_setopt(d->m_handle, CURLOPT_REDIR_PROTOCOLS, allowedProtocols);
1051     setSSLClientCertificate(job);
1052
1053     if (ignoreSSLErrors)
1054         curl_easy_setopt(d->m_handle, CURLOPT_SSL_VERIFYPEER, false);
1055     else
1056         setSSLVerifyOptions(job);
1057
1058     if (!m_certificatePath.isNull())
1059        curl_easy_setopt(d->m_handle, CURLOPT_CAINFO, m_certificatePath.data());
1060
1061     // enable gzip and deflate through Accept-Encoding:
1062     curl_easy_setopt(d->m_handle, CURLOPT_ENCODING, "");
1063
1064     // url must remain valid through the request
1065     ASSERT(!d->m_url);
1066
1067     // url is in ASCII so latin1() will only convert it to char* without character translation.
1068     d->m_url = fastStrDup(urlString.latin1().data());
1069     curl_easy_setopt(d->m_handle, CURLOPT_URL, d->m_url);
1070
1071     if (m_cookieJarFileName)
1072         curl_easy_setopt(d->m_handle, CURLOPT_COOKIEJAR, m_cookieJarFileName);
1073
1074     struct curl_slist* headers = 0;
1075     if (job->firstRequest().httpHeaderFields().size() > 0) {
1076         HTTPHeaderMap customHeaders = job->firstRequest().httpHeaderFields();
1077
1078         bool hasCacheHeaders = customHeaders.contains(HTTPHeaderName::IfModifiedSince) || customHeaders.contains(HTTPHeaderName::IfNoneMatch);
1079         if (!hasCacheHeaders && CurlCacheManager::getInstance().isCached(url)) {
1080             CurlCacheManager::getInstance().addCacheEntryClient(url, job);
1081             HTTPHeaderMap& requestHeaders = CurlCacheManager::getInstance().requestHeaders(url);
1082
1083             // append additional cache information
1084             HTTPHeaderMap::const_iterator it = requestHeaders.begin();
1085             HTTPHeaderMap::const_iterator end = requestHeaders.end();
1086             while (it != end) {
1087                 customHeaders.set(it->key, it->value);
1088                 ++it;
1089             }
1090             d->m_addedCacheValidationHeaders = true;
1091         }
1092
1093         HTTPHeaderMap::const_iterator end = customHeaders.end();
1094         for (HTTPHeaderMap::const_iterator it = customHeaders.begin(); it != end; ++it) {
1095             String key = it->key;
1096             String value = it->value;
1097             String headerString(key);
1098             if (value.isEmpty())
1099                 // Insert the ; to tell curl that this header has an empty value.
1100                 headerString.append(";");
1101             else {
1102                 headerString.append(": ");
1103                 headerString.append(value);
1104             }
1105             CString headerLatin1 = headerString.latin1();
1106             headers = curl_slist_append(headers, headerLatin1.data());
1107         }
1108     }
1109
1110     String method = job->firstRequest().httpMethod();
1111     if ("GET" == method)
1112         curl_easy_setopt(d->m_handle, CURLOPT_HTTPGET, TRUE);
1113     else if ("POST" == method)
1114         setupPOST(job, &headers);
1115     else if ("PUT" == method)
1116         setupPUT(job, &headers);
1117     else if ("HEAD" == method)
1118         curl_easy_setopt(d->m_handle, CURLOPT_NOBODY, TRUE);
1119     else {
1120         curl_easy_setopt(d->m_handle, CURLOPT_CUSTOMREQUEST, method.ascii().data());
1121         setupPUT(job, &headers);
1122     }
1123
1124     if (headers) {
1125         curl_easy_setopt(d->m_handle, CURLOPT_HTTPHEADER, headers);
1126         d->m_customHeaders = headers;
1127     }
1128
1129     applyAuthenticationToRequest(job, job->firstRequest());
1130
1131     // Set proxy options if we have them.
1132     if (m_proxy.length()) {
1133         curl_easy_setopt(d->m_handle, CURLOPT_PROXY, m_proxy.utf8().data());
1134         curl_easy_setopt(d->m_handle, CURLOPT_PROXYTYPE, m_proxyType);
1135     }
1136 }
1137
1138 void ResourceHandleManager::initCookieSession()
1139 {
1140     // Curl saves both persistent cookies, and session cookies to the cookie file.
1141     // The session cookies should be deleted before starting a new session.
1142
1143     CURL* curl = curl_easy_init();
1144
1145     if (!curl)
1146         return;
1147
1148     curl_easy_setopt(curl, CURLOPT_SHARE, m_curlShareHandle);
1149
1150     if (m_cookieJarFileName) {
1151         curl_easy_setopt(curl, CURLOPT_COOKIEFILE, m_cookieJarFileName);
1152         curl_easy_setopt(curl, CURLOPT_COOKIEJAR, m_cookieJarFileName);
1153     }
1154
1155     curl_easy_setopt(curl, CURLOPT_COOKIESESSION, 1);
1156
1157     curl_easy_cleanup(curl);
1158 }
1159
1160 void ResourceHandleManager::cancel(ResourceHandle* job)
1161 {
1162     if (removeScheduledJob(job))
1163         return;
1164
1165     ResourceHandleInternal* d = job->getInternal();
1166     d->m_cancelled = true;
1167     if (!m_downloadTimer.isActive())
1168         m_downloadTimer.startOneShot(pollTimeSeconds);
1169 }
1170
1171 } // namespace WebCore
1172
1173 #endif