[Curl] Use SQLite database in cookie jar implementation for Curl port
[WebKit-https.git] / Source / WebCore / platform / network / curl / ResourceHandleCurlDelegate.cpp
1 /*
2  * Copyright (C) 2004, 2006 Apple Inc.  All rights reserved.
3  * Copyright (C) 2005, 2006 Michael Emmel mike.emmel@gmail.com
4  * Copyright (C) 2018 Sony Interactive Entertainment Inc.
5  * All rights reserved.
6  * Copyright (C) 2017 NAVER Corp. All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
18  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
20  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
21  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
22  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
23  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
24  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
25  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28  */
29
30 #include "config.h"
31 #include "ResourceHandleCurlDelegate.h"
32
33 #if USE(CURL)
34
35 #include "AuthenticationChallenge.h"
36 #include "CookieJarCurl.h"
37 #include "CredentialStorage.h"
38 #include "CurlCacheManager.h"
39 #include "CurlRequest.h"
40 #include "HTTPParsers.h"
41 #include "NetworkStorageSession.h"
42 #include "ResourceHandleInternal.h"
43 #include "SharedBuffer.h"
44 #include "TextEncoding.h"
45 #include <wtf/CompletionHandler.h>
46 #include <wtf/text/Base64.h>
47
48 namespace WebCore {
49
50 ResourceHandleCurlDelegate::ResourceHandleCurlDelegate(ResourceHandle* handle)
51     : m_handle(handle)
52     , m_firstRequest(handle->firstRequest().isolatedCopy())
53     , m_currentRequest(handle->firstRequest().isolatedCopy())
54     , m_shouldUseCredentialStorage(handle->shouldUseCredentialStorage())
55     , m_user(handle->getInternal()->m_user.isolatedCopy())
56     , m_pass(handle->getInternal()->m_pass.isolatedCopy())
57     , m_initialCredential(handle->getInternal()->m_initialCredential)
58     , m_defersLoading(handle->getInternal()->m_defersLoading)
59 {
60
61 }
62
63 ResourceHandleCurlDelegate::~ResourceHandleCurlDelegate()
64 {
65     if (m_curlRequest)
66         m_curlRequest->setClient(nullptr);
67 }
68
69 bool ResourceHandleCurlDelegate::hasHandle() const
70 {
71     return !!m_handle;
72 }
73
74 void ResourceHandleCurlDelegate::releaseHandle()
75 {
76     m_handle = nullptr;
77 }
78
79 void ResourceHandleCurlDelegate::start()
80 {
81     ASSERT(isMainThread());
82
83     auto credential = getCredential(m_currentRequest, false);
84
85     m_curlRequest = createCurlRequest(m_currentRequest);
86     m_curlRequest->setUserPass(credential.first, credential.second);
87     m_curlRequest->start();
88 }
89
90 void ResourceHandleCurlDelegate::cancel()
91 {
92     ASSERT(isMainThread());
93
94     releaseHandle();
95
96     if (!m_curlRequest)
97         m_curlRequest->cancel();
98 }
99
100 void ResourceHandleCurlDelegate::setDefersLoading(bool defers)
101 {
102     ASSERT(isMainThread());
103
104     if (defers == m_defersLoading)
105         return;
106
107     m_defersLoading = defers;
108
109     if (!m_curlRequest)
110         return;
111
112     if (m_defersLoading)
113         m_curlRequest->suspend();
114     else
115         m_curlRequest->resume();
116 }
117
118 void ResourceHandleCurlDelegate::setAuthentication(const String& user, const String& password)
119 {
120     ASSERT(isMainThread());
121
122     if (!m_curlRequest)
123         return;
124
125     bool isSyncRequest = m_curlRequest->isSyncRequest();
126     m_curlRequest->cancel();
127     m_curlRequest->setClient(nullptr);
128
129     m_curlRequest = createCurlRequest(m_currentRequest);
130     m_curlRequest->setUserPass(user, password);
131     m_curlRequest->start(isSyncRequest);
132 }
133
134 void ResourceHandleCurlDelegate::dispatchSynchronousJob()
135 {
136     if (m_currentRequest.url().protocolIsData()) {
137         handleDataURL();
138         return;
139     }
140
141     // If defersLoading is true and we call curl_easy_perform
142     // on a paused handle, libcURL would do the transfert anyway
143     // and we would assert so force defersLoading to be false.
144     m_defersLoading = false;
145
146     m_curlRequest = createCurlRequest(m_currentRequest);
147     m_curlRequest->start(true);
148 }
149
150 Ref<CurlRequest> ResourceHandleCurlDelegate::createCurlRequest(ResourceRequest& request)
151 {
152     ASSERT(isMainThread());
153
154     // CurlCache : append additional cache information
155     m_addedCacheValidationHeaders = false;
156
157     bool hasCacheHeaders = request.httpHeaderFields().contains(HTTPHeaderName::IfModifiedSince) || request.httpHeaderFields().contains(HTTPHeaderName::IfNoneMatch);
158     if (!hasCacheHeaders) {
159         auto& cache = CurlCacheManager::singleton();
160         URL cacheUrl = request.url();
161         cacheUrl.removeFragmentIdentifier();
162
163         if (cache.isCached(cacheUrl)) {
164             cache.addCacheEntryClient(cacheUrl, m_handle);
165
166             for (const auto& entry : cache.requestHeaders(cacheUrl))
167                 request.addHTTPHeaderField(entry.key, entry.value);
168
169             m_addedCacheValidationHeaders = true;
170         }
171     }
172
173     CurlRequest::ShouldSuspend shouldSuspend = m_defersLoading ? CurlRequest::ShouldSuspend::Yes : CurlRequest::ShouldSuspend::No;
174     return CurlRequest::create(request, this, shouldSuspend, CurlRequest::EnableMultipart::Yes);
175 }
176
177 bool ResourceHandleCurlDelegate::cancelledOrClientless()
178 {
179     if (!m_handle)
180         return true;
181
182     return !m_handle->client();
183 }
184
185 void ResourceHandleCurlDelegate::curlDidSendData(unsigned long long bytesSent, unsigned long long totalBytesToBeSent)
186 {
187     ASSERT(isMainThread());
188
189     if (cancelledOrClientless())
190         return;
191
192     m_handle->client()->didSendData(m_handle, bytesSent, totalBytesToBeSent);
193 }
194
195 static void handleCookieHeaders(const CurlResponse& response)
196 {
197     static const auto setCookieHeader = "set-cookie: ";
198
199     auto& defaultStorageSession = NetworkStorageSession::defaultStorageSession();
200     const CookieJarCurl& cookieJar = defaultStorageSession.cookieStorage();
201     for (auto header : response.headers) {
202         if (header.startsWithIgnoringASCIICase(setCookieHeader)) {
203             String setCookieString = header.right(header.length() - strlen(setCookieHeader));
204             cookieJar.setCookiesFromHTTPResponse(defaultStorageSession, response.url, setCookieString);
205         }
206     }
207 }
208
209 void ResourceHandleCurlDelegate::curlDidReceiveResponse(const CurlResponse& receivedResponse)
210 {
211     ASSERT(isMainThread());
212     ASSERT(!m_defersLoading);
213
214     if (cancelledOrClientless())
215         return;
216
217     m_handle->getInternal()->m_response = ResourceResponse(receivedResponse);
218
219     if (m_curlRequest)
220         m_handle->getInternal()->m_response.setDeprecatedNetworkLoadMetrics(m_curlRequest->getNetworkLoadMetrics());
221
222     handleCookieHeaders(receivedResponse);
223
224     if (response().shouldRedirect()) {
225         willSendRequest();
226         return;
227     }
228
229     if (response().isUnauthorized()) {
230         AuthenticationChallenge challenge(receivedResponse, m_authFailureCount, response(), m_handle);
231         m_handle->didReceiveAuthenticationChallenge(challenge);
232         m_authFailureCount++;
233         return;
234     }
235
236     if (m_handle->client()) {
237         if (response().isNotModified()) {
238             URL cacheUrl = m_currentRequest.url();
239             cacheUrl.removeFragmentIdentifier();
240
241             if (CurlCacheManager::singleton().getCachedResponse(cacheUrl, response())) {
242                 if (m_addedCacheValidationHeaders) {
243                     response().setHTTPStatusCode(200);
244                     response().setHTTPStatusText("OK");
245                 }
246             }
247         }
248
249         CurlCacheManager::singleton().didReceiveResponse(*m_handle, response());
250
251         auto protectedThis = makeRef(*m_handle);
252         m_handle->didReceiveResponse(ResourceResponse(response()), [this, protectedThis = makeRef(*this)] {
253             continueAfterDidReceiveResponse();
254         });
255     }
256 }
257
258 void ResourceHandleCurlDelegate::curlDidReceiveBuffer(Ref<SharedBuffer>&& buffer)
259 {
260     ASSERT(isMainThread());
261
262     if (cancelledOrClientless())
263         return;
264
265     CurlCacheManager::singleton().didReceiveData(*m_handle, buffer->data(), buffer->size());
266     m_handle->client()->didReceiveBuffer(m_handle, WTFMove(buffer), buffer->size());
267 }
268
269 void ResourceHandleCurlDelegate::curlDidComplete()
270 {
271     ASSERT(isMainThread());
272
273     if (cancelledOrClientless())
274         return;
275
276     if (m_curlRequest)
277         m_handle->getInternal()->m_response.setDeprecatedNetworkLoadMetrics(m_curlRequest->getNetworkLoadMetrics());
278
279     if (m_handle->client()) {
280         CurlCacheManager::singleton().didFinishLoading(*m_handle);
281         m_handle->client()->didFinishLoading(m_handle);
282     }
283 }
284
285 void ResourceHandleCurlDelegate::curlDidFailWithError(const ResourceError& resourceError)
286 {
287     ASSERT(isMainThread());
288
289     if (cancelledOrClientless())
290         return;
291
292     CurlCacheManager::singleton().didFail(*m_handle);
293     m_handle->client()->didFail(m_handle, resourceError);
294 }
295
296 void ResourceHandleCurlDelegate::continueDidReceiveResponse()
297 {
298     ASSERT(isMainThread());
299
300     continueAfterDidReceiveResponse();
301 }
302
303 void ResourceHandleCurlDelegate::platformContinueSynchronousDidReceiveResponse()
304 {
305     ASSERT(isMainThread());
306
307     continueAfterDidReceiveResponse();
308 }
309
310 void ResourceHandleCurlDelegate::continueAfterDidReceiveResponse()
311 {
312     ASSERT(isMainThread());
313
314     // continueDidReceiveResponse might cancel the load.
315     if (cancelledOrClientless() || !m_curlRequest)
316         return;
317
318     m_curlRequest->completeDidReceiveResponse();
319 }
320
321 bool ResourceHandleCurlDelegate::shouldRedirectAsGET(const ResourceRequest& request, bool crossOrigin)
322 {
323     if (request.httpMethod() == "GET" || request.httpMethod() == "HEAD")
324         return false;
325
326     if (!request.url().protocolIsInHTTPFamily())
327         return true;
328
329     if (response().isSeeOther())
330         return true;
331
332     if ((response().isMovedPermanently() || response().isFound()) && (request.httpMethod() == "POST"))
333         return true;
334
335     if (crossOrigin && (request.httpMethod() == "DELETE"))
336         return true;
337
338     return false;
339 }
340
341 void ResourceHandleCurlDelegate::willSendRequest()
342 {
343     ASSERT(isMainThread());
344
345     static const int maxRedirects = 20;
346
347     if (m_redirectCount++ > maxRedirects) {
348         m_handle->client()->didFail(m_handle, ResourceError::httpError(CURLE_TOO_MANY_REDIRECTS, response().url()));
349         return;
350     }
351
352     String location = response().httpHeaderField(HTTPHeaderName::Location);
353     URL newURL = URL(m_handle->getInternal()->m_response.url(), location);
354     bool crossOrigin = !protocolHostAndPortAreEqual(m_firstRequest.url(), newURL);
355
356     ResourceRequest newRequest = m_firstRequest;
357     newRequest.setURL(newURL);
358
359     if (shouldRedirectAsGET(newRequest, crossOrigin)) {
360         newRequest.setHTTPMethod("GET");
361         newRequest.setHTTPBody(nullptr);
362         newRequest.clearHTTPContentType();
363     }
364
365     // Should not set Referer after a redirect from a secure resource to non-secure one.
366     if (!newURL.protocolIs("https") && protocolIs(newRequest.httpReferrer(), "https") && m_handle->context()->shouldClearReferrerOnHTTPSToHTTPRedirect())
367         newRequest.clearHTTPReferrer();
368
369     m_user = newURL.user();
370     m_pass = newURL.pass();
371     newRequest.removeCredentials();
372
373     if (crossOrigin) {
374         // If the network layer carries over authentication headers from the original request
375         // in a cross-origin redirect, we want to clear those headers here. 
376         newRequest.clearHTTPAuthorization();
377         newRequest.clearHTTPOrigin();
378     }
379
380     ResourceResponse responseCopy = response();
381     m_handle->client()->willSendRequestAsync(m_handle, WTFMove(newRequest), WTFMove(responseCopy), [this, protectedThis = makeRef(*this)] (ResourceRequest&& request) {
382         continueWillSendRequest(WTFMove(request));
383     });
384 }
385
386 void ResourceHandleCurlDelegate::continueWillSendRequest(ResourceRequest&& request)
387 {
388     ASSERT(isMainThread());
389
390     continueAfterWillSendRequest(WTFMove(request));
391 }
392
393 void ResourceHandleCurlDelegate::continueAfterWillSendRequest(ResourceRequest&& request)
394 {
395     ASSERT(isMainThread());
396
397     // willSendRequest might cancel the load.
398     if (cancelledOrClientless() || !m_curlRequest)
399         return;
400
401     m_currentRequest = WTFMove(request);
402
403     bool isSyncRequest = m_curlRequest->isSyncRequest();
404     m_curlRequest->cancel();
405     m_curlRequest->setClient(nullptr);
406
407     m_curlRequest = createCurlRequest(m_currentRequest);
408
409     if (protocolHostAndPortAreEqual(m_currentRequest.url(), response().url())) {
410         auto credential = getCredential(m_currentRequest, true);
411         m_curlRequest->setUserPass(credential.first, credential.second);
412     }
413
414     m_curlRequest->start(isSyncRequest);
415 }
416
417 ResourceResponse& ResourceHandleCurlDelegate::response()
418 {
419     return m_handle->getInternal()->m_response;
420 }
421
422 void ResourceHandleCurlDelegate::handleDataURL()
423 {
424     ASSERT(m_firstRequest.url().protocolIsData());
425     String url = m_firstRequest.url().string();
426
427     ASSERT(m_handle->client());
428
429     auto index = url.find(',');
430     if (index == notFound) {
431         m_handle->client()->cannotShowURL(m_handle);
432         return;
433     }
434
435     String mediaType = url.substring(5, index - 5);
436     String data = url.substring(index + 1);
437     auto originalSize = data.length();
438
439     bool base64 = mediaType.endsWithIgnoringASCIICase(";base64");
440     if (base64)
441         mediaType = mediaType.left(mediaType.length() - 7);
442
443     if (mediaType.isEmpty())
444         mediaType = "text/plain";
445
446     String mimeType = extractMIMETypeFromMediaType(mediaType);
447     String charset = extractCharsetFromMediaType(mediaType);
448
449     if (charset.isEmpty())
450         charset = "US-ASCII";
451
452     ResourceResponse response;
453     response.setMimeType(mimeType);
454     response.setTextEncodingName(charset);
455     response.setURL(m_firstRequest.url());
456
457     if (base64) {
458         data = decodeURLEscapeSequences(data);
459         m_handle->didReceiveResponse(WTFMove(response), [this, protectedThis = makeRef(*this)] {
460             continueAfterDidReceiveResponse();
461         });
462
463         // didReceiveResponse might cause the client to be deleted.
464         if (m_handle->client()) {
465             Vector<char> out;
466             if (base64Decode(data, out, Base64IgnoreSpacesAndNewLines) && out.size() > 0)
467                 m_handle->client()->didReceiveBuffer(m_handle, SharedBuffer::create(out.data(), out.size()), originalSize);
468         }
469     } else {
470         TextEncoding encoding(charset);
471         data = decodeURLEscapeSequences(data, encoding);
472         m_handle->didReceiveResponse(WTFMove(response), [this, protectedThis = makeRef(*this)] {
473             continueAfterDidReceiveResponse();
474         });
475
476         // didReceiveResponse might cause the client to be deleted.
477         if (m_handle->client()) {
478             auto encodedData = encoding.encode(data, UnencodableHandling::URLEncodedEntities);
479             if (encodedData.size())
480                 m_handle->client()->didReceiveBuffer(m_handle, SharedBuffer::create(WTFMove(encodedData)), originalSize);
481         }
482     }
483
484     if (m_handle->client())
485         m_handle->client()->didFinishLoading(m_handle);
486 }
487
488 std::pair<String, String> ResourceHandleCurlDelegate::getCredential(ResourceRequest& request, bool redirect)
489 {
490     // m_user/m_pass are credentials given manually, for instance, by the arguments passed to XMLHttpRequest.open().
491     String partition = request.cachePartition();
492
493     if (m_shouldUseCredentialStorage) {
494         if (m_user.isEmpty() && m_pass.isEmpty()) {
495             // <rdar://problem/7174050> - For URLs that match the paths of those previously challenged for HTTP Basic authentication, 
496             // try and reuse the credential preemptively, as allowed by RFC 2617.
497             m_initialCredential = CredentialStorage::defaultCredentialStorage().get(partition, request.url());
498         } else if (!redirect) {
499             // If there is already a protection space known for the URL, update stored credentials
500             // before sending a request. This makes it possible to implement logout by sending an
501             // XMLHttpRequest with known incorrect credentials, and aborting it immediately (so that
502             // an authentication dialog doesn't pop up).
503             CredentialStorage::defaultCredentialStorage().set(partition, Credential(m_user, m_pass, CredentialPersistenceNone), request.url());
504         }
505     }
506
507     String user = m_user;
508     String password = m_pass;
509
510     if (!m_initialCredential.isEmpty()) {
511         user = m_initialCredential.user();
512         password = m_initialCredential.password();
513     }
514
515     if (user.isEmpty() && password.isEmpty())
516         return std::pair<String, String>("", "");
517
518     return std::pair<String, String>(user, password);
519 }
520
521 } // namespace WebCore
522
523 #endif