[WTF] Add makeUnique<T>, which ensures T is fast-allocated, makeUnique / makeUniqueWi...
[WebKit-https.git] / Source / WebCore / platform / network / curl / ResourceHandleCurl.cpp
index ecdbf11..707d0a6 100644 (file)
 
 #if USE(CURL)
 
+#include "CookieJar.h"
+#include "CookieJarCurl.h"
 #include "CredentialStorage.h"
+#include "CurlCacheManager.h"
 #include "CurlContext.h"
-#include "FileSystem.h"
+#include "CurlRequest.h"
+#include "HTTPParsers.h"
 #include "Logging.h"
+#include "NetworkStorageSession.h"
 #include "ResourceHandleInternal.h"
+#include "SameSiteInfo.h"
+#include "SharedBuffer.h"
 #include "SynchronousLoaderClient.h"
+#include "TextEncoding.h"
+#include <wtf/CompletionHandler.h>
+#include <wtf/FileSystem.h>
+#include <wtf/text/Base64.h>
 
 namespace WebCore {
 
 ResourceHandleInternal::~ResourceHandleInternal()
 {
-    if (m_delegate)
-        m_delegate->releaseHandle();
+    if (m_curlRequest)
+        m_curlRequest->invalidateClient();
 }
 
 ResourceHandle::~ResourceHandle() = default;
@@ -64,20 +75,97 @@ bool ResourceHandle::start()
         return false;
 
     // Only allow the POST and GET methods for non-HTTP requests.
-    const ResourceRequest& request = firstRequest();
+    auto request = firstRequest();
     if (!request.url().protocolIsInHTTPFamily() && request.httpMethod() != "GET" && request.httpMethod() != "POST") {
         scheduleFailure(InvalidURLFailure); // Error must not be reported immediately
         return true;
     }
 
-    d->m_delegate = adoptRef(new ResourceHandleCurlDelegate(this));
-    return d->m_delegate->start();
+    d->m_startTime = MonotonicTime::now();
+
+    d->m_curlRequest = createCurlRequest(WTFMove(request));
+
+    if (auto credential = getCredential(d->m_firstRequest, false)) {
+        d->m_curlRequest->setUserPass(credential->user(), credential->password());
+        d->m_curlRequest->setAuthenticationScheme(ProtectionSpaceAuthenticationSchemeHTTPBasic);
+    }
+
+    d->m_curlRequest->setStartTime(d->m_startTime);
+    d->m_curlRequest->start();
+
+    return true;
 }
 
 void ResourceHandle::cancel()
 {
-    if (d->m_delegate)
-        d->m_delegate->cancel();
+    ASSERT(isMainThread());
+
+    d->m_cancelled = true;
+
+    if (d->m_curlRequest)
+        d->m_curlRequest->cancel();
+}
+
+bool ResourceHandle::cancelledOrClientless()
+{
+    if (d->m_cancelled)
+        return true;
+
+    return !client();
+}
+
+void ResourceHandle::addCacheValidationHeaders(ResourceRequest& request)
+{
+    ASSERT(isMainThread());
+
+    d->m_addedCacheValidationHeaders = false;
+
+    auto hasCacheHeaders = request.httpHeaderFields().contains(HTTPHeaderName::IfModifiedSince) || request.httpHeaderFields().contains(HTTPHeaderName::IfNoneMatch);
+    if (hasCacheHeaders)
+        return;
+
+    auto& cache = CurlCacheManager::singleton();
+    URL cacheUrl = request.url();
+    cacheUrl.removeFragmentIdentifier();
+
+    if (cache.isCached(cacheUrl)) {
+        cache.addCacheEntryClient(cacheUrl, this);
+
+        for (const auto& entry : cache.requestHeaders(cacheUrl))
+            request.addHTTPHeaderField(entry.key, entry.value);
+
+        d->m_addedCacheValidationHeaders = true;
+    }
+}
+
+Ref<CurlRequest> ResourceHandle::createCurlRequest(ResourceRequest&& request, RequestStatus status)
+{
+    ASSERT(isMainThread());
+
+    if (status == RequestStatus::NewRequest) {
+        addCacheValidationHeaders(request);
+
+        auto& storageSession = *d->m_context->storageSession();
+        auto& cookieJar = storageSession.cookieStorage();
+        auto includeSecureCookies = request.url().protocolIs("https") ? IncludeSecureCookies::Yes : IncludeSecureCookies::No;
+        String cookieHeaderField = cookieJar.cookieRequestHeaderFieldValue(storageSession, request.firstPartyForCookies(), SameSiteInfo::create(request), request.url(), WTF::nullopt, WTF::nullopt, includeSecureCookies).first;
+        if (!cookieHeaderField.isEmpty())
+            request.addHTTPHeaderField(HTTPHeaderName::Cookie, cookieHeaderField);
+    }
+
+    CurlRequest::ShouldSuspend shouldSuspend = d->m_defersLoading ? CurlRequest::ShouldSuspend::Yes : CurlRequest::ShouldSuspend::No;
+    // FIXME: Use a correct sessionID.
+    auto curlRequest = CurlRequest::create(request, *delegate(), PAL::SessionID::emptySessionID(), shouldSuspend, CurlRequest::EnableMultipart::Yes, CurlRequest::CaptureNetworkLoadMetrics::Basic, d->m_messageQueue);
+    
+    return curlRequest;
+}
+
+CurlResourceHandleDelegate* ResourceHandle::delegate()
+{
+    if (!d->m_delegate)
+        d->m_delegate = makeUnique<CurlResourceHandleDelegate>(*this);
+
+    return d->m_delegate.get();
 }
 
 #if OS(WINDOWS)
@@ -86,14 +174,14 @@ void ResourceHandle::setHostAllowsAnyHTTPSCertificate(const String& host)
 {
     ASSERT(isMainThread());
 
-    CurlContext::singleton().sslHandle().setHostAllowsAnyHTTPSCertificate(host);
+    CurlContext::singleton().sslHandle().allowAnyHTTPSCertificatesForHost(host);
 }
 
 void ResourceHandle::setClientCertificateInfo(const String& host, const String& certificate, const String& key)
 {
     ASSERT(isMainThread());
 
-    if (fileExists(certificate))
+    if (FileSystem::fileExists(certificate))
         CurlContext::singleton().sslHandle().setClientCertificateInfo(host, certificate, key);
     else
         LOG(Network, "Invalid client certificate file: %s!\n", certificate.latin1().data());
@@ -113,8 +201,18 @@ void ResourceHandle::platformSetDefersLoading(bool defers)
 {
     ASSERT(isMainThread());
 
-    if (d->m_delegate)
-        d->m_delegate->setDefersLoading(defers);
+    if (defers == d->m_defersLoading)
+        return;
+
+    d->m_defersLoading = defers;
+
+    if (!d->m_curlRequest)
+        return;
+
+    if (d->m_defersLoading)
+        d->m_curlRequest->suspend();
+    else
+        d->m_curlRequest->resume();
 }
 
 bool ResourceHandle::shouldUseCredentialStorage()
@@ -134,10 +232,9 @@ void ResourceHandle::didReceiveAuthenticationChallenge(const AuthenticationChall
         URL urlToStore;
         if (challenge.failureResponse().httpStatusCode() == 401)
             urlToStore = challenge.failureResponse().url();
-        CredentialStorage::defaultCredentialStorage().set(partition, credential, challenge.protectionSpace(), urlToStore);
-        
-        if (d->m_delegate)
-            d->m_delegate->setAuthentication(credential.user(), credential.password());
+        d->m_context->storageSession()->credentialStorage().set(partition, credential, challenge.protectionSpace(), urlToStore);
+
+        restartRequestWithCredential(challenge.protectionSpace(), credential);
 
         d->m_user = String();
         d->m_pass = String();
@@ -150,20 +247,19 @@ void ResourceHandle::didReceiveAuthenticationChallenge(const AuthenticationChall
             // The stored credential wasn't accepted, stop using it.
             // There is a race condition here, since a different credential might have already been stored by another ResourceHandle,
             // but the observable effect should be very minor, if any.
-            CredentialStorage::defaultCredentialStorage().remove(partition, challenge.protectionSpace());
+            d->m_context->storageSession()->credentialStorage().remove(partition, challenge.protectionSpace());
         }
 
         if (!challenge.previousFailureCount()) {
-            Credential credential = CredentialStorage::defaultCredentialStorage().get(partition, challenge.protectionSpace());
+            Credential credential = d->m_context->storageSession()->credentialStorage().get(partition, challenge.protectionSpace());
             if (!credential.isEmpty() && credential != d->m_initialCredential) {
                 ASSERT(credential.persistence() == CredentialPersistenceNone);
                 if (challenge.failureResponse().httpStatusCode() == 401) {
                     // Store the credential back, possibly adding it as a default for this directory.
-                    CredentialStorage::defaultCredentialStorage().set(partition, credential, challenge.protectionSpace(), challenge.failureResponse().url());
+                    d->m_context->storageSession()->credentialStorage().set(partition, credential, challenge.protectionSpace(), challenge.failureResponse().url());
                 }
 
-                if (d->m_delegate)
-                    d->m_delegate->setAuthentication(credential.user(), credential.password());
+                restartRequestWithCredential(challenge.protectionSpace(), credential);
                 return;
             }
         }
@@ -194,12 +290,11 @@ void ResourceHandle::receivedCredential(const AuthenticationChallenge& challenge
     if (shouldUseCredentialStorage()) {
         if (challenge.failureResponse().httpStatusCode() == 401) {
             URL urlToStore = challenge.failureResponse().url();
-            CredentialStorage::defaultCredentialStorage().set(partition, credential, challenge.protectionSpace(), urlToStore);
+            d->m_context->storageSession()->credentialStorage().set(partition, credential, challenge.protectionSpace(), urlToStore);
         }
     }
 
-    if (d->m_delegate)
-        d->m_delegate->setAuthentication(credential.user(), credential.password());
+    restartRequestWithCredential(challenge.protectionSpace(), credential);
 
     clearAuthentication();
 }
@@ -213,8 +308,9 @@ void ResourceHandle::receivedRequestToContinueWithoutCredential(const Authentica
 
     clearAuthentication();
 
-    auto protectedThis = makeRef(*this);
-    didReceiveResponse(ResourceResponse(d->m_response));
+    didReceiveResponse(ResourceResponse(delegate()->response()), [this, protectedThis = makeRef(*this)] {
+        continueAfterDidReceiveResponse();
+    });
 }
 
 void ResourceHandle::receivedCancellation(const AuthenticationChallenge& challenge)
@@ -240,44 +336,265 @@ void ResourceHandle::receivedChallengeRejection(const AuthenticationChallenge&)
     ASSERT_NOT_REACHED();
 }
 
-void ResourceHandle::platformLoadResourceSynchronously(NetworkingContext* context, const ResourceRequest& request, StoredCredentialsPolicy, ResourceError& error, ResourceResponse& response, Vector<char>& data)
+Optional<Credential> ResourceHandle::getCredential(const ResourceRequest& request, bool redirect)
+{
+    // m_user/m_pass are credentials given manually, for instance, by the arguments passed to XMLHttpRequest.open().
+    Credential credential { d->m_user, d->m_pass, CredentialPersistenceNone };
+
+    if (shouldUseCredentialStorage()) {
+        String partition = request.cachePartition();
+
+        if (credential.isEmpty()) {
+            // <rdar://problem/7174050> - For URLs that match the paths of those previously challenged for HTTP Basic authentication, 
+            // try and reuse the credential preemptively, as allowed by RFC 2617.
+            d->m_initialCredential = d->m_context->storageSession()->credentialStorage().get(partition, request.url());
+        } else if (!redirect) {
+            // If there is already a protection space known for the URL, update stored credentials
+            // before sending a request. This makes it possible to implement logout by sending an
+            // XMLHttpRequest with known incorrect credentials, and aborting it immediately (so that
+            // an authentication dialog doesn't pop up).
+            d->m_context->storageSession()->credentialStorage().set(partition, credential, request.url());
+        }
+    }
+
+    if (!d->m_initialCredential.isEmpty())
+        return d->m_initialCredential;
+
+    return WTF::nullopt;
+}
+
+void ResourceHandle::restartRequestWithCredential(const ProtectionSpace& protectionSpace, const Credential& credential)
+{
+    ASSERT(isMainThread());
+
+    if (!d->m_curlRequest)
+        return;
+    
+    auto previousRequest = d->m_curlRequest->resourceRequest();
+    d->m_curlRequest->cancel();
+
+    d->m_curlRequest = createCurlRequest(WTFMove(previousRequest), RequestStatus::ReusedRequest);
+    d->m_curlRequest->setAuthenticationScheme(protectionSpace.authenticationScheme());
+    d->m_curlRequest->setUserPass(credential.user(), credential.password());
+    d->m_curlRequest->setStartTime(d->m_startTime);
+    d->m_curlRequest->start();
+}
+
+void ResourceHandle::platformLoadResourceSynchronously(NetworkingContext* context, const ResourceRequest& request, StoredCredentialsPolicy storedCredentialsPolicy, ResourceError& error, ResourceResponse& response, Vector<char>& data)
 {
     ASSERT(isMainThread());
+    ASSERT(!request.isEmpty());
 
     SynchronousLoaderClient client;
-    RefPtr<ResourceHandle> handle = adoptRef(new ResourceHandle(context, request, &client, false, false));
+    client.setAllowStoredCredentials(storedCredentialsPolicy == StoredCredentialsPolicy::Use);
+
+    bool defersLoading = false;
+    bool shouldContentSniff = true;
+    bool shouldContentEncodingSniff = true;
+    RefPtr<ResourceHandle> handle = adoptRef(new ResourceHandle(context, request, &client, defersLoading, shouldContentSniff, shouldContentEncodingSniff));
+    handle->d->m_messageQueue = &client.messageQueue();
+    handle->d->m_startTime = MonotonicTime::now();
+
+    if (request.url().protocolIsData()) {
+        handle->handleDataURL();
+        return;
+    }
+
+    auto requestCopy = handle->firstRequest();
+    handle->d->m_curlRequest = handle->createCurlRequest(WTFMove(requestCopy));
 
-    handle->d->m_delegate = adoptRef(new ResourceHandleCurlDelegate(handle.get()));
-    handle->d->m_delegate->dispatchSynchronousJob();
+    if (auto credential = handle->getCredential(handle->d->m_firstRequest, false)) {
+        handle->d->m_curlRequest->setUserPass(credential->user(), credential->password());
+        handle->d->m_curlRequest->setAuthenticationScheme(ProtectionSpaceAuthenticationSchemeHTTPBasic);
+    }
+
+    handle->d->m_curlRequest->setStartTime(handle->d->m_startTime);
+    handle->d->m_curlRequest->start();
+
+    do {
+        if (auto task = client.messageQueue().waitForMessage())
+            (*task)();
+    } while (!client.messageQueue().killed() && !handle->cancelledOrClientless());
 
     error = client.error();
     data.swap(client.mutableData());
     response = client.response();
 }
 
-void ResourceHandle::continueDidReceiveResponse()
+void ResourceHandle::platformContinueSynchronousDidReceiveResponse()
 {
     ASSERT(isMainThread());
 
-    if (d->m_delegate)
-        d->m_delegate->continueDidReceiveResponse();
+    continueAfterDidReceiveResponse();
 }
 
-void ResourceHandle::platformContinueSynchronousDidReceiveResponse()
+void ResourceHandle::continueAfterDidReceiveResponse()
+{
+    ASSERT(isMainThread());
+
+    // continueDidReceiveResponse might cancel the load.
+    if (cancelledOrClientless() || !d->m_curlRequest)
+        return;
+
+    d->m_curlRequest->completeDidReceiveResponse();
+}
+
+bool ResourceHandle::shouldRedirectAsGET(const ResourceRequest& request, bool crossOrigin)
+{
+    if (request.httpMethod() == "GET" || request.httpMethod() == "HEAD")
+        return false;
+
+    if (!request.url().protocolIsInHTTPFamily())
+        return true;
+
+    if (delegate()->response().isSeeOther())
+        return true;
+
+    if ((delegate()->response().isMovedPermanently() || delegate()->response().isFound()) && (request.httpMethod() == "POST"))
+        return true;
+
+    if (crossOrigin && (request.httpMethod() == "DELETE"))
+        return true;
+
+    return false;
+}
+
+void ResourceHandle::willSendRequest()
 {
     ASSERT(isMainThread());
 
-    if (d->m_delegate)
-        d->m_delegate->platformContinueSynchronousDidReceiveResponse();
+    static const int maxRedirects = 20;
+
+    if (d->m_redirectCount++ > maxRedirects) {
+        client()->didFail(this, ResourceError::httpError(CURLE_TOO_MANY_REDIRECTS, delegate()->response().url()));
+        return;
+    }
+
+    String location = delegate()->response().httpHeaderField(HTTPHeaderName::Location);
+    URL newURL = URL(delegate()->response().url(), location);
+    bool crossOrigin = !protocolHostAndPortAreEqual(d->m_firstRequest.url(), newURL);
+
+    ResourceRequest newRequest = d->m_firstRequest;
+    newRequest.setURL(newURL);
+
+    if (shouldRedirectAsGET(newRequest, crossOrigin)) {
+        newRequest.setHTTPMethod("GET");
+        newRequest.setHTTPBody(nullptr);
+        newRequest.clearHTTPContentType();
+    }
+
+    // Should not set Referer after a redirect from a secure resource to non-secure one.
+    if (!newURL.protocolIs("https") && protocolIs(newRequest.httpReferrer(), "https") && context()->shouldClearReferrerOnHTTPSToHTTPRedirect())
+        newRequest.clearHTTPReferrer();
+
+    d->m_user = newURL.user();
+    d->m_pass = newURL.pass();
+    newRequest.removeCredentials();
+
+    if (crossOrigin) {
+        // If the network layer carries over authentication headers from the original request
+        // in a cross-origin redirect, we want to clear those headers here. 
+        newRequest.clearHTTPAuthorization();
+        newRequest.clearHTTPOrigin();
+        d->m_startTime = WTF::MonotonicTime::now();
+    }
+
+    ResourceResponse responseCopy = delegate()->response();
+    client()->willSendRequestAsync(this, WTFMove(newRequest), WTFMove(responseCopy), [this, protectedThis = makeRef(*this)] (ResourceRequest&& request) {
+        continueAfterWillSendRequest(WTFMove(request));
+    });
 }
 
-void ResourceHandle::continueWillSendRequest(ResourceRequest&& request)
+void ResourceHandle::continueAfterWillSendRequest(ResourceRequest&& request)
 {
     ASSERT(isMainThread());
-    ASSERT(!client() || client()->usesAsyncCallbacks());
 
-    if (d->m_delegate)
-        d->m_delegate->continueWillSendRequest(WTFMove(request));
+    // willSendRequest might cancel the load.
+    if (cancelledOrClientless() || !d->m_curlRequest)
+        return;
+
+    if (request.isNull()) {
+        cancel();
+        return;
+    }
+
+    auto shouldForwardCredential = protocolHostAndPortAreEqual(request.url(), delegate()->response().url());
+    auto credential = getCredential(request, true);
+
+    d->m_curlRequest->cancel();
+    d->m_curlRequest = createCurlRequest(WTFMove(request));
+
+    if (shouldForwardCredential && credential)
+        d->m_curlRequest->setUserPass(credential->user(), credential->password());
+
+    d->m_curlRequest->setStartTime(d->m_startTime);
+    d->m_curlRequest->start();
+}
+
+void ResourceHandle::handleDataURL()
+{
+    ASSERT(d->m_firstRequest.url().protocolIsData());
+    String url = d->m_firstRequest.url().string();
+
+    ASSERT(client());
+
+    auto index = url.find(',');
+    if (index == notFound) {
+        client()->cannotShowURL(this);
+        return;
+    }
+
+    String mediaType = url.substring(5, index - 5);
+    String data = url.substring(index + 1);
+    auto originalSize = data.length();
+
+    bool base64 = mediaType.endsWithIgnoringASCIICase(";base64");
+    if (base64)
+        mediaType = mediaType.left(mediaType.length() - 7);
+
+    if (mediaType.isEmpty())
+        mediaType = "text/plain"_s;
+
+    String mimeType = extractMIMETypeFromMediaType(mediaType);
+    String charset = extractCharsetFromMediaType(mediaType);
+
+    if (charset.isEmpty())
+        charset = "US-ASCII"_s;
+
+    ResourceResponse response;
+    response.setMimeType(mimeType);
+    response.setTextEncodingName(charset);
+    response.setURL(d->m_firstRequest.url());
+
+    if (base64) {
+        data = decodeURLEscapeSequences(data);
+        didReceiveResponse(WTFMove(response), [this, protectedThis = makeRef(*this)] {
+            continueAfterDidReceiveResponse();
+        });
+
+        // didReceiveResponse might cause the client to be deleted.
+        if (client()) {
+            Vector<char> out;
+            if (base64Decode(data, out, Base64IgnoreSpacesAndNewLines) && out.size() > 0)
+                client()->didReceiveBuffer(this, SharedBuffer::create(out.data(), out.size()), originalSize);
+        }
+    } else {
+        TextEncoding encoding(charset);
+        data = decodeURLEscapeSequences(data, encoding);
+        didReceiveResponse(WTFMove(response), [this, protectedThis = makeRef(*this)] {
+            continueAfterDidReceiveResponse();
+        });
+
+        // didReceiveResponse might cause the client to be deleted.
+        if (client()) {
+            auto encodedData = encoding.encode(data, UnencodableHandling::URLEncodedEntities);
+            if (encodedData.size())
+                client()->didReceiveBuffer(this, SharedBuffer::create(WTFMove(encodedData)), originalSize);
+        }
+    }
+
+    if (client())
+        client()->didFinishLoading(this);
 }
 
 } // namespace WebCore