Make sure cookies get flushed to disk before exiting or suspending the network process
[WebKit-https.git] / Source / WebKit / NetworkProcess / NetworkProcess.cpp
1 /*
2  * Copyright (C) 2012-2018 Apple Inc. All rights reserved.
3  * Copyright (C) 2018 Sony Interactive Entertainment Inc.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24  * THE POSSIBILITY OF SUCH DAMAGE.
25  */
26
27 #include "config.h"
28 #include "NetworkProcess.h"
29
30 #include "ArgumentCoders.h"
31 #include "Attachment.h"
32 #include "AuthenticationManager.h"
33 #include "ChildProcessMessages.h"
34 #include "DataReference.h"
35 #include "DownloadProxyMessages.h"
36 #if ENABLE(LEGACY_CUSTOM_PROTOCOL_MANAGER)
37 #include "LegacyCustomProtocolManager.h"
38 #endif
39 #include "Logging.h"
40 #include "NetworkBlobRegistry.h"
41 #include "NetworkConnectionToWebProcess.h"
42 #include "NetworkContentRuleListManagerMessages.h"
43 #include "NetworkProcessCreationParameters.h"
44 #include "NetworkProcessPlatformStrategies.h"
45 #include "NetworkProcessProxyMessages.h"
46 #include "NetworkResourceLoader.h"
47 #include "NetworkSession.h"
48 #include "PreconnectTask.h"
49 #include "RemoteNetworkingContext.h"
50 #include "SessionTracker.h"
51 #include "StatisticsData.h"
52 #include "WebCookieManager.h"
53 #include "WebCoreArgumentCoders.h"
54 #include "WebPageProxyMessages.h"
55 #include "WebProcessPoolMessages.h"
56 #include "WebsiteData.h"
57 #include "WebsiteDataFetchOption.h"
58 #include "WebsiteDataStore.h"
59 #include "WebsiteDataStoreParameters.h"
60 #include "WebsiteDataType.h"
61 #include <WebCore/DNS.h>
62 #include <WebCore/DeprecatedGlobalSettings.h>
63 #include <WebCore/DiagnosticLoggingClient.h>
64 #include <WebCore/LogInitialization.h>
65 #include <WebCore/MIMETypeRegistry.h>
66 #include <WebCore/NetworkStateNotifier.h>
67 #include <WebCore/NetworkStorageSession.h>
68 #include <WebCore/ResourceRequest.h>
69 #include <WebCore/RuntimeApplicationChecks.h>
70 #include <WebCore/SchemeRegistry.h>
71 #include <WebCore/SecurityOriginData.h>
72 #include <WebCore/SecurityOriginHash.h>
73 #include <WebCore/Settings.h>
74 #include <WebCore/URLParser.h>
75 #include <pal/SessionID.h>
76 #include <wtf/CallbackAggregator.h>
77 #include <wtf/OptionSet.h>
78 #include <wtf/ProcessPrivilege.h>
79 #include <wtf/RunLoop.h>
80 #include <wtf/text/AtomicString.h>
81 #include <wtf/text/CString.h>
82
83 #if ENABLE(SEC_ITEM_SHIM)
84 #include "SecItemShim.h"
85 #endif
86
87 #include "NetworkCache.h"
88 #include "NetworkCacheCoders.h"
89
90 #if ENABLE(NETWORK_CAPTURE)
91 #include "NetworkCaptureManager.h"
92 #endif
93
94 #if PLATFORM(COCOA)
95 #include "NetworkSessionCocoa.h"
96 #endif
97
98 using namespace WebCore;
99
100 namespace WebKit {
101
102 NetworkProcess& NetworkProcess::singleton()
103 {
104     static NeverDestroyed<NetworkProcess> networkProcess;
105     return networkProcess;
106 }
107
108 NetworkProcess::NetworkProcess()
109     : m_hasSetCacheModel(false)
110     , m_cacheModel(CacheModelDocumentViewer)
111     , m_diskCacheIsDisabledForTesting(false)
112     , m_canHandleHTTPSServerTrustEvaluation(true)
113 #if PLATFORM(COCOA)
114     , m_clearCacheDispatchGroup(0)
115 #endif
116 {
117     NetworkProcessPlatformStrategies::initialize();
118
119     addSupplement<AuthenticationManager>();
120     addSupplement<WebCookieManager>();
121 #if ENABLE(LEGACY_CUSTOM_PROTOCOL_MANAGER)
122     addSupplement<LegacyCustomProtocolManager>();
123 #endif
124
125     NetworkStateNotifier::singleton().addListener([this](bool isOnLine) {
126         auto webProcessConnections = m_webProcessConnections;
127         for (auto& webProcessConnection : webProcessConnections)
128             webProcessConnection->setOnLineState(isOnLine);
129     });
130 }
131
132 NetworkProcess::~NetworkProcess()
133 {
134     for (auto& callbacks : m_cacheStorageParametersCallbacks.values()) {
135         for (auto& callback : callbacks)
136             callback(String { }, 0);
137     }
138 }
139
140 AuthenticationManager& NetworkProcess::authenticationManager()
141 {
142     return *supplement<AuthenticationManager>();
143 }
144
145 DownloadManager& NetworkProcess::downloadManager()
146 {
147     static NeverDestroyed<DownloadManager> downloadManager(*this);
148     return downloadManager;
149 }
150
151 void NetworkProcess::removeNetworkConnectionToWebProcess(NetworkConnectionToWebProcess* connection)
152 {
153     size_t vectorIndex = m_webProcessConnections.find(connection);
154     ASSERT(vectorIndex != notFound);
155
156     m_webProcessConnections.remove(vectorIndex);
157 }
158
159 bool NetworkProcess::shouldTerminate()
160 {
161     // Network process keeps session cookies and credentials, so it should never terminate (as long as UI process connection is alive).
162     return false;
163 }
164
165 void NetworkProcess::didReceiveMessage(IPC::Connection& connection, IPC::Decoder& decoder)
166 {
167     if (messageReceiverMap().dispatchMessage(connection, decoder))
168         return;
169
170     if (decoder.messageReceiverName() == Messages::ChildProcess::messageReceiverName()) {
171         ChildProcess::didReceiveMessage(connection, decoder);
172         return;
173     }
174
175 #if ENABLE(CONTENT_EXTENSIONS)
176     if (decoder.messageReceiverName() == Messages::NetworkContentRuleListManager::messageReceiverName()) {
177         m_NetworkContentRuleListManager.didReceiveMessage(connection, decoder);
178         return;
179     }
180 #endif
181
182     didReceiveNetworkProcessMessage(connection, decoder);
183 }
184
185 void NetworkProcess::didReceiveSyncMessage(IPC::Connection& connection, IPC::Decoder& decoder, std::unique_ptr<IPC::Encoder>& replyEncoder)
186 {
187     if (messageReceiverMap().dispatchSyncMessage(connection, decoder, replyEncoder))
188         return;
189
190     didReceiveSyncNetworkProcessMessage(connection, decoder, replyEncoder);
191 }
192
193 void NetworkProcess::didClose(IPC::Connection&)
194 {
195     ASSERT(RunLoop::isMain());
196
197     // Make sure we flush all cookies to disk before exiting.
198     platformSyncAllCookies([this] {
199         stopRunLoop();
200     });
201 }
202
203 void NetworkProcess::didCreateDownload()
204 {
205     disableTermination();
206 }
207
208 void NetworkProcess::didDestroyDownload()
209 {
210     enableTermination();
211 }
212
213 IPC::Connection* NetworkProcess::downloadProxyConnection()
214 {
215     return parentProcessConnection();
216 }
217
218 AuthenticationManager& NetworkProcess::downloadsAuthenticationManager()
219 {
220     return authenticationManager();
221 }
222
223 void NetworkProcess::lowMemoryHandler(Critical critical)
224 {
225     if (m_suppressMemoryPressureHandler)
226         return;
227
228     WTF::releaseFastMallocFreeMemory();
229 }
230
231 void NetworkProcess::initializeNetworkProcess(NetworkProcessCreationParameters&& parameters)
232 {
233 #if HAVE(SEC_KEY_PROXY)
234     WTF::setProcessPrivileges({ ProcessPrivilege::CanAccessRawCookies });
235 #else
236     WTF::setProcessPrivileges({ ProcessPrivilege::CanAccessRawCookies, ProcessPrivilege::CanAccessCredentials });
237 #endif
238     WebCore::NetworkStorageSession::permitProcessToUseCookieAPI(true);
239     WebCore::setPresentingApplicationPID(parameters.presentingApplicationPID);
240     platformInitializeNetworkProcess(parameters);
241
242     WTF::Thread::setCurrentThreadIsUserInitiated();
243     AtomicString::init();
244
245     m_suppressMemoryPressureHandler = parameters.shouldSuppressMemoryPressureHandler;
246     m_loadThrottleLatency = parameters.loadThrottleLatency;
247     if (!m_suppressMemoryPressureHandler) {
248         auto& memoryPressureHandler = MemoryPressureHandler::singleton();
249         memoryPressureHandler.setLowMemoryHandler([this] (Critical critical, Synchronous) {
250             lowMemoryHandler(critical);
251         });
252         memoryPressureHandler.install();
253     }
254
255 #if ENABLE(NETWORK_CAPTURE)
256     NetworkCapture::Manager::singleton().initialize(
257         parameters.recordReplayMode,
258         parameters.recordReplayCacheLocation);
259 #endif
260
261     m_diskCacheIsDisabledForTesting = parameters.shouldUseTestingNetworkSession;
262
263     m_diskCacheSizeOverride = parameters.diskCacheSizeOverride;
264     setCacheModel(static_cast<uint32_t>(parameters.cacheModel));
265
266     setCanHandleHTTPSServerTrustEvaluation(parameters.canHandleHTTPSServerTrustEvaluation);
267
268     // FIXME: instead of handling this here, a message should be sent later (scales to multiple sessions)
269     if (parameters.privateBrowsingEnabled)
270         RemoteNetworkingContext::ensureWebsiteDataStoreSession(WebsiteDataStoreParameters::legacyPrivateSessionParameters());
271
272     if (parameters.shouldUseTestingNetworkSession)
273         NetworkStorageSession::switchToNewTestingSession();
274
275 #if HAVE(CFNETWORK_STORAGE_PARTITIONING) && !RELEASE_LOG_DISABLED
276     m_logCookieInformation = parameters.logCookieInformation;
277 #endif
278
279     SessionTracker::setSession(PAL::SessionID::defaultSessionID(), NetworkSession::create(WTFMove(parameters.defaultSessionParameters)));
280
281     auto* defaultSession = SessionTracker::networkSession(PAL::SessionID::defaultSessionID());
282     for (const auto& cookie : parameters.defaultSessionPendingCookies)
283         defaultSession->networkStorageSession().setCookie(cookie);
284
285     for (auto& supplement : m_supplements.values())
286         supplement->initialize(parameters);
287
288     for (auto& scheme : parameters.urlSchemesRegisteredAsSecure)
289         registerURLSchemeAsSecure(scheme);
290
291     for (auto& scheme : parameters.urlSchemesRegisteredAsBypassingContentSecurityPolicy)
292         registerURLSchemeAsBypassingContentSecurityPolicy(scheme);
293
294     for (auto& scheme : parameters.urlSchemesRegisteredAsLocal)
295         registerURLSchemeAsLocal(scheme);
296
297     for (auto& scheme : parameters.urlSchemesRegisteredAsNoAccess)
298         registerURLSchemeAsNoAccess(scheme);
299
300     for (auto& scheme : parameters.urlSchemesRegisteredAsDisplayIsolated)
301         registerURLSchemeAsDisplayIsolated(scheme);
302
303     for (auto& scheme : parameters.urlSchemesRegisteredAsCORSEnabled)
304         registerURLSchemeAsCORSEnabled(scheme);
305
306     for (auto& scheme : parameters.urlSchemesRegisteredAsCanDisplayOnlyIfCanRequest)
307         registerURLSchemeAsCanDisplayOnlyIfCanRequest(scheme);
308
309     RELEASE_LOG(Process, "%p - NetworkProcess::initializeNetworkProcess: Presenting process = %d", this, WebCore::presentingApplicationPID());
310 }
311
312 void NetworkProcess::initializeConnection(IPC::Connection* connection)
313 {
314     ChildProcess::initializeConnection(connection);
315
316     for (auto& supplement : m_supplements.values())
317         supplement->initializeConnection(connection);
318 }
319
320 void NetworkProcess::createNetworkConnectionToWebProcess()
321 {
322 #if USE(UNIX_DOMAIN_SOCKETS)
323     IPC::Connection::SocketPair socketPair = IPC::Connection::createPlatformConnection();
324
325     auto connection = NetworkConnectionToWebProcess::create(socketPair.server);
326     m_webProcessConnections.append(WTFMove(connection));
327
328     IPC::Attachment clientSocket(socketPair.client);
329     parentProcessConnection()->send(Messages::NetworkProcessProxy::DidCreateNetworkConnectionToWebProcess(clientSocket), 0);
330 #elif OS(DARWIN)
331     // Create the listening port.
332     mach_port_t listeningPort = MACH_PORT_NULL;
333     auto kr = mach_port_allocate(mach_task_self(), MACH_PORT_RIGHT_RECEIVE, &listeningPort);
334     if (kr != KERN_SUCCESS) {
335         RELEASE_LOG_ERROR(Process, "NetworkProcess::createNetworkConnectionToWebProcess: Could not allocate mach port, error %x", kr);
336         CRASH();
337     }
338     if (!MACH_PORT_VALID(listeningPort)) {
339         RELEASE_LOG_ERROR(Process, "NetworkProcess::createNetworkConnectionToWebProcess: Could not allocate mach port, returned port was invalid");
340         CRASH();
341     }
342
343     // Create a listening connection.
344     auto connection = NetworkConnectionToWebProcess::create(IPC::Connection::Identifier(listeningPort));
345     m_webProcessConnections.append(WTFMove(connection));
346
347     IPC::Attachment clientPort(listeningPort, MACH_MSG_TYPE_MAKE_SEND);
348     parentProcessConnection()->send(Messages::NetworkProcessProxy::DidCreateNetworkConnectionToWebProcess(clientPort), 0);
349 #elif OS(WINDOWS)
350     IPC::Connection::Identifier serverIdentifier, clientIdentifier;
351     if (!IPC::Connection::createServerAndClientIdentifiers(serverIdentifier, clientIdentifier)) {
352         LOG_ERROR("Failed to create server and client identifiers");
353         CRASH();
354     }
355
356     auto connection = NetworkConnectionToWebProcess::create(serverIdentifier);
357     m_webProcessConnections.append(WTFMove(connection));
358
359     IPC::Attachment clientSocket(clientIdentifier);
360     parentProcessConnection()->send(Messages::NetworkProcessProxy::DidCreateNetworkConnectionToWebProcess(clientSocket), 0);
361 #else
362     notImplemented();
363 #endif
364
365     if (!m_webProcessConnections.isEmpty())
366         m_webProcessConnections.last()->setOnLineState(NetworkStateNotifier::singleton().onLine());
367 }
368
369 void NetworkProcess::clearCachedCredentials()
370 {
371     NetworkStorageSession::defaultStorageSession().credentialStorage().clearCredentials();
372     if (auto* networkSession = SessionTracker::networkSession(PAL::SessionID::defaultSessionID()))
373         networkSession->clearCredentials();
374     else
375         ASSERT_NOT_REACHED();
376 }
377
378 void NetworkProcess::addWebsiteDataStore(WebsiteDataStoreParameters&& parameters)
379 {
380     RemoteNetworkingContext::ensureWebsiteDataStoreSession(WTFMove(parameters));
381 }
382
383 void NetworkProcess::destroySession(PAL::SessionID sessionID)
384 {
385     SessionTracker::destroySession(sessionID);
386     m_sessionsControlledByAutomation.remove(sessionID);
387     CacheStorage::Engine::destroyEngine(sessionID);
388 }
389
390 void NetworkProcess::grantSandboxExtensionsToStorageProcessForBlobs(const Vector<String>& filenames, Function<void ()>&& completionHandler)
391 {
392     static uint64_t lastRequestID;
393
394     uint64_t requestID = ++lastRequestID;
395     m_sandboxExtensionForBlobsCompletionHandlers.set(requestID, WTFMove(completionHandler));
396     parentProcessConnection()->send(Messages::NetworkProcessProxy::GrantSandboxExtensionsToStorageProcessForBlobs(requestID, filenames), 0);
397 }
398
399 void NetworkProcess::didGrantSandboxExtensionsToStorageProcessForBlobs(uint64_t requestID)
400 {
401     if (auto handler = m_sandboxExtensionForBlobsCompletionHandlers.take(requestID))
402         handler();
403 }
404
405 void NetworkProcess::writeBlobToFilePath(const WebCore::URL& url, const String& path, SandboxExtension::Handle&& handleForWriting, uint64_t requestID)
406 {
407     auto extension = SandboxExtension::create(WTFMove(handleForWriting));
408     if (!extension) {
409         parentProcessConnection()->send(Messages::NetworkProcessProxy::DidWriteBlobToFilePath(false, requestID), 0);
410         return;
411     }
412
413     extension->consume();
414     NetworkBlobRegistry::singleton().writeBlobToFilePath(url, path, [this, extension = WTFMove(extension), requestID] (bool success) {
415         extension->revoke();
416         parentProcessConnection()->send(Messages::NetworkProcessProxy::DidWriteBlobToFilePath(success, requestID), 0);
417     });
418 }
419
420 #if HAVE(CFNETWORK_STORAGE_PARTITIONING)
421 void NetworkProcess::updatePrevalentDomainsToBlockCookiesFor(PAL::SessionID sessionID, const Vector<String>& domainsToBlock, bool shouldClearFirst, uint64_t contextId)
422 {
423     if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID))
424         networkStorageSession->setPrevalentDomainsToBlockCookiesFor(domainsToBlock, shouldClearFirst);
425     parentProcessConnection()->send(Messages::NetworkProcessProxy::DidUpdateBlockCookies(contextId), 0);
426 }
427
428 void NetworkProcess::hasStorageAccessForFrame(PAL::SessionID sessionID, const String& resourceDomain, const String& firstPartyDomain, uint64_t frameID, uint64_t pageID, uint64_t contextId)
429 {
430     if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID))
431         parentProcessConnection()->send(Messages::NetworkProcessProxy::StorageAccessRequestResult(networkStorageSession->hasStorageAccess(resourceDomain, firstPartyDomain, frameID, pageID), contextId), 0);
432     else
433         ASSERT_NOT_REACHED();
434 }
435
436 void NetworkProcess::getAllStorageAccessEntries(PAL::SessionID sessionID, uint64_t contextId)
437 {
438     if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID))
439         parentProcessConnection()->send(Messages::NetworkProcessProxy::AllStorageAccessEntriesResult(networkStorageSession->getAllStorageAccessEntries(), contextId), 0);
440     else
441         ASSERT_NOT_REACHED();
442 }
443
444 void NetworkProcess::grantStorageAccess(PAL::SessionID sessionID, const String& resourceDomain, const String& firstPartyDomain, std::optional<uint64_t> frameID, uint64_t pageID, uint64_t contextId)
445 {
446     bool isStorageGranted = false;
447     if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID)) {
448         networkStorageSession->grantStorageAccess(resourceDomain, firstPartyDomain, frameID, pageID);
449         ASSERT(networkStorageSession->hasStorageAccess(resourceDomain, firstPartyDomain, frameID, pageID));
450         isStorageGranted = true;
451     } else
452         ASSERT_NOT_REACHED();
453
454     parentProcessConnection()->send(Messages::NetworkProcessProxy::StorageAccessRequestResult(isStorageGranted, contextId), 0);
455 }
456
457 void NetworkProcess::removeAllStorageAccess(PAL::SessionID sessionID, uint64_t contextId)
458 {
459     if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID))
460         networkStorageSession->removeAllStorageAccess();
461     else
462         ASSERT_NOT_REACHED();
463     parentProcessConnection()->send(Messages::NetworkProcessProxy::DidRemoveAllStorageAccess(contextId), 0);
464 }
465
466 void NetworkProcess::removePrevalentDomains(PAL::SessionID sessionID, const Vector<String>& domains)
467 {
468     if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID))
469         networkStorageSession->removePrevalentDomains(domains);
470 }
471 #endif
472
473 bool NetworkProcess::sessionIsControlledByAutomation(PAL::SessionID sessionID) const
474 {
475     return m_sessionsControlledByAutomation.contains(sessionID);
476 }
477
478 void NetworkProcess::setSessionIsControlledByAutomation(PAL::SessionID sessionID, bool controlled)
479 {
480     if (controlled)
481         m_sessionsControlledByAutomation.add(sessionID);
482     else
483         m_sessionsControlledByAutomation.remove(sessionID);
484 }
485
486 static void fetchDiskCacheEntries(PAL::SessionID sessionID, OptionSet<WebsiteDataFetchOption> fetchOptions, Function<void (Vector<WebsiteData::Entry>)>&& completionHandler)
487 {
488     if (auto* cache = NetworkProcess::singleton().cache()) {
489         HashMap<SecurityOriginData, uint64_t> originsAndSizes;
490         cache->traverse([fetchOptions, completionHandler = WTFMove(completionHandler), originsAndSizes = WTFMove(originsAndSizes)](auto* traversalEntry) mutable {
491             if (!traversalEntry) {
492                 Vector<WebsiteData::Entry> entries;
493
494                 for (auto& originAndSize : originsAndSizes)
495                     entries.append(WebsiteData::Entry { originAndSize.key, WebsiteDataType::DiskCache, originAndSize.value });
496
497                 RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler), entries = WTFMove(entries)] {
498                     completionHandler(entries);
499                 });
500
501                 return;
502             }
503
504             auto url = traversalEntry->entry.response().url();
505             auto result = originsAndSizes.add({url.protocol().toString(), url.host().toString(), url.port()}, 0);
506
507             if (fetchOptions.contains(WebsiteDataFetchOption::ComputeSizes))
508                 result.iterator->value += traversalEntry->entry.sourceStorageRecord().header.size() + traversalEntry->recordInfo.bodySize;
509         });
510
511         return;
512     }
513
514     RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler)] {
515         completionHandler({ });
516     });
517 }
518
519 void NetworkProcess::fetchWebsiteData(PAL::SessionID sessionID, OptionSet<WebsiteDataType> websiteDataTypes, OptionSet<WebsiteDataFetchOption> fetchOptions, uint64_t callbackID)
520 {
521     struct CallbackAggregator final : public RefCounted<CallbackAggregator> {
522         explicit CallbackAggregator(Function<void (WebsiteData)>&& completionHandler)
523             : m_completionHandler(WTFMove(completionHandler))
524         {
525         }
526
527         ~CallbackAggregator()
528         {
529             ASSERT(RunLoop::isMain());
530
531             RunLoop::main().dispatch([completionHandler = WTFMove(m_completionHandler), websiteData = WTFMove(m_websiteData)] {
532                 completionHandler(websiteData);
533             });
534         }
535
536         Function<void (WebsiteData)> m_completionHandler;
537         WebsiteData m_websiteData;
538     };
539
540     auto callbackAggregator = adoptRef(*new CallbackAggregator([this, callbackID] (WebsiteData websiteData) {
541         parentProcessConnection()->send(Messages::NetworkProcessProxy::DidFetchWebsiteData(callbackID, websiteData), 0);
542     }));
543
544     if (websiteDataTypes.contains(WebsiteDataType::Cookies)) {
545         if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID))
546             networkStorageSession->getHostnamesWithCookies(callbackAggregator->m_websiteData.hostNamesWithCookies);
547     }
548
549     if (websiteDataTypes.contains(WebsiteDataType::Credentials)) {
550         if (NetworkStorageSession::storageSession(sessionID))
551             callbackAggregator->m_websiteData.originsWithCredentials = NetworkStorageSession::storageSession(sessionID)->credentialStorage().originsWithCredentials();
552     }
553
554     if (websiteDataTypes.contains(WebsiteDataType::DOMCache)) {
555         CacheStorage::Engine::fetchEntries(sessionID, fetchOptions.contains(WebsiteDataFetchOption::ComputeSizes), [callbackAggregator = callbackAggregator.copyRef()](auto entries) mutable {
556             callbackAggregator->m_websiteData.entries.appendVector(entries);
557         });
558     }
559
560     if (websiteDataTypes.contains(WebsiteDataType::DiskCache)) {
561         fetchDiskCacheEntries(sessionID, fetchOptions, [callbackAggregator = WTFMove(callbackAggregator)](auto entries) mutable {
562             callbackAggregator->m_websiteData.entries.appendVector(entries);
563         });
564     }
565
566 #if PLATFORM(COCOA)
567     if (websiteDataTypes.contains(WebsiteDataType::HSTSCache)) {
568         if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID))
569             getHostNamesWithHSTSCache(*networkStorageSession, callbackAggregator->m_websiteData.hostNamesWithHSTSCache);
570     }
571 #endif
572 }
573
574 void NetworkProcess::deleteWebsiteData(PAL::SessionID sessionID, OptionSet<WebsiteDataType> websiteDataTypes, WallTime modifiedSince, uint64_t callbackID)
575 {
576 #if PLATFORM(COCOA)
577     if (websiteDataTypes.contains(WebsiteDataType::HSTSCache)) {
578         if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID))
579             clearHSTSCache(*networkStorageSession, modifiedSince);
580     }
581 #endif
582
583     if (websiteDataTypes.contains(WebsiteDataType::Cookies)) {
584         if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID))
585             networkStorageSession->deleteAllCookiesModifiedSince(modifiedSince);
586     }
587
588     if (websiteDataTypes.contains(WebsiteDataType::Credentials)) {
589         if (NetworkStorageSession::storageSession(sessionID))
590             NetworkStorageSession::storageSession(sessionID)->credentialStorage().clearCredentials();
591     }
592
593     auto clearTasksHandler = WTF::CallbackAggregator::create([this, callbackID] {
594         parentProcessConnection()->send(Messages::NetworkProcessProxy::DidDeleteWebsiteData(callbackID), 0);
595     });
596
597     if (websiteDataTypes.contains(WebsiteDataType::DOMCache))
598         CacheStorage::Engine::clearAllCaches(sessionID, [clearTasksHandler = clearTasksHandler.copyRef()] { });
599
600     if (websiteDataTypes.contains(WebsiteDataType::DiskCache) && !sessionID.isEphemeral())
601         clearDiskCache(modifiedSince, [clearTasksHandler = WTFMove(clearTasksHandler)] { });
602 }
603
604 static void clearDiskCacheEntries(const Vector<SecurityOriginData>& origins, Function<void ()>&& completionHandler)
605 {
606     if (auto* cache = NetworkProcess::singleton().cache()) {
607         HashSet<RefPtr<SecurityOrigin>> originsToDelete;
608         for (auto& origin : origins)
609             originsToDelete.add(origin.securityOrigin());
610
611         Vector<NetworkCache::Key> cacheKeysToDelete;
612         cache->traverse([cache, completionHandler = WTFMove(completionHandler), originsToDelete = WTFMove(originsToDelete), cacheKeysToDelete = WTFMove(cacheKeysToDelete)](auto* traversalEntry) mutable {
613             if (traversalEntry) {
614                 if (originsToDelete.contains(SecurityOrigin::create(traversalEntry->entry.response().url())))
615                     cacheKeysToDelete.append(traversalEntry->entry.key());
616                 return;
617             }
618
619             cache->remove(cacheKeysToDelete, WTFMove(completionHandler));
620             return;
621         });
622
623         return;
624     }
625
626     RunLoop::main().dispatch(WTFMove(completionHandler));
627 }
628
629 void NetworkProcess::deleteWebsiteDataForOrigins(PAL::SessionID sessionID, OptionSet<WebsiteDataType> websiteDataTypes, const Vector<SecurityOriginData>& originDatas, const Vector<String>& cookieHostNames, const Vector<String>& HSTSCacheHostNames, uint64_t callbackID)
630 {
631     if (websiteDataTypes.contains(WebsiteDataType::Cookies)) {
632         if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID))
633             networkStorageSession->deleteCookiesForHostnames(cookieHostNames);
634     }
635
636 #if PLATFORM(COCOA)
637     if (websiteDataTypes.contains(WebsiteDataType::HSTSCache)) {
638         if (auto* networkStorageSession = NetworkStorageSession::storageSession(sessionID))
639             deleteHSTSCacheForHostNames(*networkStorageSession, HSTSCacheHostNames);
640     }
641 #endif
642
643     auto clearTasksHandler = WTF::CallbackAggregator::create([this, callbackID] {
644         parentProcessConnection()->send(Messages::NetworkProcessProxy::DidDeleteWebsiteDataForOrigins(callbackID), 0);
645     });
646
647     if (websiteDataTypes.contains(WebsiteDataType::DOMCache)) {
648         for (auto& originData : originDatas)
649             CacheStorage::Engine::clearCachesForOrigin(sessionID, SecurityOriginData { originData }, [clearTasksHandler = clearTasksHandler.copyRef()] { });
650     }
651
652     if (websiteDataTypes.contains(WebsiteDataType::DiskCache) && !sessionID.isEphemeral())
653         clearDiskCacheEntries(originDatas, [clearTasksHandler = WTFMove(clearTasksHandler)] { });
654 }
655
656 void NetworkProcess::downloadRequest(PAL::SessionID sessionID, DownloadID downloadID, const ResourceRequest& request, const String& suggestedFilename)
657 {
658     downloadManager().startDownload(nullptr, sessionID, downloadID, request, suggestedFilename);
659 }
660
661 void NetworkProcess::resumeDownload(PAL::SessionID sessionID, DownloadID downloadID, const IPC::DataReference& resumeData, const String& path, WebKit::SandboxExtension::Handle&& sandboxExtensionHandle)
662 {
663     downloadManager().resumeDownload(sessionID, downloadID, resumeData, path, WTFMove(sandboxExtensionHandle));
664 }
665
666 void NetworkProcess::cancelDownload(DownloadID downloadID)
667 {
668     downloadManager().cancelDownload(downloadID);
669 }
670     
671 #if USE(PROTECTION_SPACE_AUTH_CALLBACK)
672 static uint64_t generateCanAuthenticateIdentifier()
673 {
674     static uint64_t lastLoaderID = 0;
675     return ++lastLoaderID;
676 }
677
678 void NetworkProcess::canAuthenticateAgainstProtectionSpace(NetworkResourceLoader& loader, const WebCore::ProtectionSpace& protectionSpace)
679 {
680     uint64_t loaderID = generateCanAuthenticateIdentifier();
681     m_waitingNetworkResourceLoaders.set(loaderID, loader);
682     parentProcessConnection()->send(Messages::NetworkProcessProxy::CanAuthenticateAgainstProtectionSpace(loaderID, loader.pageID(), loader.frameID(), protectionSpace), 0);
683 }
684
685 #if ENABLE(SERVER_PRECONNECT)
686 void NetworkProcess::canAuthenticateAgainstProtectionSpace(PreconnectTask& preconnectTask, const WebCore::ProtectionSpace& protectionSpace)
687 {
688     uint64_t loaderID = generateCanAuthenticateIdentifier();
689     m_waitingPreconnectTasks.set(loaderID, makeWeakPtr(preconnectTask));
690     parentProcessConnection()->send(Messages::NetworkProcessProxy::CanAuthenticateAgainstProtectionSpace(loaderID, preconnectTask.pageID(), preconnectTask.frameID(), protectionSpace), 0);
691 }
692 #endif
693
694 void NetworkProcess::continueCanAuthenticateAgainstProtectionSpace(uint64_t loaderID, bool canAuthenticate)
695 {
696     if (auto resourceLoader = m_waitingNetworkResourceLoaders.take(loaderID)) {
697         resourceLoader.value()->continueCanAuthenticateAgainstProtectionSpace(canAuthenticate);
698         return;
699     }
700 #if ENABLE(SERVER_PRECONNECT)
701     if (auto preconnectTask = m_waitingPreconnectTasks.take(loaderID)) {
702         preconnectTask->continueCanAuthenticateAgainstProtectionSpace(canAuthenticate);
703         return;
704     }
705 #endif
706 }
707
708 #endif
709
710 void NetworkProcess::continueWillSendRequest(DownloadID downloadID, WebCore::ResourceRequest&& request)
711 {
712     downloadManager().continueWillSendRequest(downloadID, WTFMove(request));
713 }
714
715 void NetworkProcess::pendingDownloadCanceled(DownloadID downloadID)
716 {
717     downloadProxyConnection()->send(Messages::DownloadProxy::DidCancel({ }), downloadID.downloadID());
718 }
719
720 void NetworkProcess::findPendingDownloadLocation(NetworkDataTask& networkDataTask, ResponseCompletionHandler&& completionHandler, const ResourceResponse& response)
721 {
722     uint64_t destinationID = networkDataTask.pendingDownloadID().downloadID();
723     downloadProxyConnection()->send(Messages::DownloadProxy::DidReceiveResponse(response), destinationID);
724
725     downloadManager().willDecidePendingDownloadDestination(networkDataTask, WTFMove(completionHandler));
726
727     // As per https://html.spec.whatwg.org/#as-a-download (step 2), the filename from the Content-Disposition header
728     // should override the suggested filename from the download attribute.
729     String suggestedFilename = response.isAttachmentWithFilename() ? response.suggestedFilename() : networkDataTask.suggestedFilename();
730     suggestedFilename = MIMETypeRegistry::appendFileExtensionIfNecessary(suggestedFilename, response.mimeType());
731
732     downloadProxyConnection()->send(Messages::DownloadProxy::DecideDestinationWithSuggestedFilenameAsync(networkDataTask.pendingDownloadID(), suggestedFilename), destinationID);
733 }
734
735 void NetworkProcess::continueDecidePendingDownloadDestination(DownloadID downloadID, String destination, SandboxExtension::Handle&& sandboxExtensionHandle, bool allowOverwrite)
736 {
737     if (destination.isEmpty())
738         downloadManager().cancelDownload(downloadID);
739     else
740         downloadManager().continueDecidePendingDownloadDestination(downloadID, destination, WTFMove(sandboxExtensionHandle), allowOverwrite);
741 }
742
743 void NetworkProcess::setCacheModel(uint32_t cm)
744 {
745     CacheModel cacheModel = static_cast<CacheModel>(cm);
746
747     if (m_hasSetCacheModel && (cacheModel == m_cacheModel))
748         return;
749
750     m_hasSetCacheModel = true;
751     m_cacheModel = cacheModel;
752
753     unsigned urlCacheMemoryCapacity = 0;
754     uint64_t urlCacheDiskCapacity = 0;
755     uint64_t diskFreeSize = 0;
756     if (WebCore::FileSystem::getVolumeFreeSpace(m_diskCacheDirectory, diskFreeSize)) {
757         // As a fudge factor, use 1000 instead of 1024, in case the reported byte
758         // count doesn't align exactly to a megabyte boundary.
759         diskFreeSize /= KB * 1000;
760         calculateURLCacheSizes(cacheModel, diskFreeSize, urlCacheMemoryCapacity, urlCacheDiskCapacity);
761     }
762
763     if (m_diskCacheSizeOverride >= 0)
764         urlCacheDiskCapacity = m_diskCacheSizeOverride;
765
766     if (m_cache)
767         m_cache->setCapacity(urlCacheDiskCapacity);
768 }
769
770 void NetworkProcess::setCanHandleHTTPSServerTrustEvaluation(bool value)
771 {
772     m_canHandleHTTPSServerTrustEvaluation = value;
773 }
774
775 void NetworkProcess::getNetworkProcessStatistics(uint64_t callbackID)
776 {
777     StatisticsData data;
778
779     auto& networkProcess = NetworkProcess::singleton();
780     data.statisticsNumbers.set("DownloadsActiveCount", networkProcess.downloadManager().activeDownloadCount());
781     data.statisticsNumbers.set("OutstandingAuthenticationChallengesCount", networkProcess.authenticationManager().outstandingAuthenticationChallengeCount());
782
783     parentProcessConnection()->send(Messages::WebProcessPool::DidGetStatistics(data, callbackID), 0);
784 }
785
786 void NetworkProcess::setAllowsAnySSLCertificateForWebSocket(bool allows)
787 {
788     DeprecatedGlobalSettings::setAllowsAnySSLCertificate(allows);
789 }
790
791 void NetworkProcess::logDiagnosticMessage(uint64_t webPageID, const String& message, const String& description, ShouldSample shouldSample)
792 {
793     if (!DiagnosticLoggingClient::shouldLogAfterSampling(shouldSample))
794         return;
795
796     parentProcessConnection()->send(Messages::NetworkProcessProxy::LogDiagnosticMessage(webPageID, message, description, ShouldSample::No), 0);
797 }
798
799 void NetworkProcess::logDiagnosticMessageWithResult(uint64_t webPageID, const String& message, const String& description, DiagnosticLoggingResultType result, ShouldSample shouldSample)
800 {
801     if (!DiagnosticLoggingClient::shouldLogAfterSampling(shouldSample))
802         return;
803
804     parentProcessConnection()->send(Messages::NetworkProcessProxy::LogDiagnosticMessageWithResult(webPageID, message, description, result, ShouldSample::No), 0);
805 }
806
807 void NetworkProcess::logDiagnosticMessageWithValue(uint64_t webPageID, const String& message, const String& description, double value, unsigned significantFigures, ShouldSample shouldSample)
808 {
809     if (!DiagnosticLoggingClient::shouldLogAfterSampling(shouldSample))
810         return;
811
812     parentProcessConnection()->send(Messages::NetworkProcessProxy::LogDiagnosticMessageWithValue(webPageID, message, description, value, significantFigures, ShouldSample::No), 0);
813 }
814
815 void NetworkProcess::terminate()
816 {
817 #if ENABLE(NETWORK_CAPTURE)
818     NetworkCapture::Manager::singleton().terminate();
819 #endif
820
821     platformTerminate();
822     ChildProcess::terminate();
823 }
824
825 void NetworkProcess::processDidTransitionToForeground()
826 {
827     platformProcessDidTransitionToForeground();
828 }
829
830 void NetworkProcess::processDidTransitionToBackground()
831 {
832     platformProcessDidTransitionToBackground();
833 }
834
835 // FIXME: We can remove this one by adapting RefCounter.
836 class TaskCounter : public RefCounted<TaskCounter> {
837 public:
838     explicit TaskCounter(Function<void()>&& callback) : m_callback(WTFMove(callback)) { }
839     ~TaskCounter() { m_callback(); };
840
841 private:
842     Function<void()> m_callback;
843 };
844
845 void NetworkProcess::actualPrepareToSuspend(ShouldAcknowledgeWhenReadyToSuspend shouldAcknowledgeWhenReadyToSuspend)
846 {
847     platformPrepareToSuspend();
848     lowMemoryHandler(Critical::Yes);
849
850     RefPtr<TaskCounter> delayedTaskCounter;
851     if (shouldAcknowledgeWhenReadyToSuspend == ShouldAcknowledgeWhenReadyToSuspend::Yes) {
852         delayedTaskCounter = adoptRef(new TaskCounter([this] {
853             RELEASE_LOG(ProcessSuspension, "%p - NetworkProcess::notifyProcessReadyToSuspend() Sending ProcessReadyToSuspend IPC message", this);
854             if (parentProcessConnection())
855                 parentProcessConnection()->send(Messages::NetworkProcessProxy::ProcessReadyToSuspend(), 0);
856         }));
857     }
858
859     platformSyncAllCookies([delayedTaskCounter] { });
860
861     for (auto& connection : m_webProcessConnections)
862         connection->cleanupForSuspension([delayedTaskCounter] { });
863 }
864
865 void NetworkProcess::processWillSuspendImminently(bool& handled)
866 {
867     actualPrepareToSuspend(ShouldAcknowledgeWhenReadyToSuspend::No);
868     handled = true;
869 }
870
871 void NetworkProcess::prepareToSuspend()
872 {
873     RELEASE_LOG(ProcessSuspension, "%p - NetworkProcess::prepareToSuspend()", this);
874     actualPrepareToSuspend(ShouldAcknowledgeWhenReadyToSuspend::Yes);
875 }
876
877 void NetworkProcess::cancelPrepareToSuspend()
878 {
879     // Although it is tempting to send a NetworkProcessProxy::DidCancelProcessSuspension message from here
880     // we do not because prepareToSuspend() already replied with a NetworkProcessProxy::ProcessReadyToSuspend
881     // message. And NetworkProcessProxy expects to receive either a NetworkProcessProxy::ProcessReadyToSuspend-
882     // or NetworkProcessProxy::DidCancelProcessSuspension- message, but not both.
883     RELEASE_LOG(ProcessSuspension, "%p - NetworkProcess::cancelPrepareToSuspend()", this);
884     platformProcessDidResume();
885     for (auto& connection : m_webProcessConnections)
886         connection->endSuspension();
887 }
888
889 void NetworkProcess::processDidResume()
890 {
891     RELEASE_LOG(ProcessSuspension, "%p - NetworkProcess::processDidResume()", this);
892     platformProcessDidResume();
893     for (auto& connection : m_webProcessConnections)
894         connection->endSuspension();
895 }
896
897 void NetworkProcess::prefetchDNS(const String& hostname)
898 {
899     WebCore::prefetchDNS(hostname);
900 }
901
902 void NetworkProcess::cacheStorageParameters(PAL::SessionID sessionID, CacheStorageParametersCallback&& callback)
903 {
904     m_cacheStorageParametersCallbacks.ensure(sessionID, [&] {
905         parentProcessConnection()->send(Messages::NetworkProcessProxy::RetrieveCacheStorageParameters { sessionID }, 0);
906         return Vector<CacheStorageParametersCallback> { };
907     }).iterator->value.append(WTFMove(callback));
908 }
909
910 void NetworkProcess::setCacheStorageParameters(PAL::SessionID sessionID, uint64_t quota, String&& cacheStorageDirectory, SandboxExtension::Handle&& handle)
911 {
912     auto iterator = m_cacheStorageParametersCallbacks.find(sessionID);
913     if (iterator == m_cacheStorageParametersCallbacks.end())
914         return;
915
916     SandboxExtension::consumePermanently(handle);
917     auto callbacks = WTFMove(iterator->value);
918     m_cacheStorageParametersCallbacks.remove(iterator);
919     for (auto& callback : callbacks)
920         callback(String { cacheStorageDirectory }, quota);
921 }
922
923 void NetworkProcess::preconnectTo(const WebCore::URL& url, WebCore::StoredCredentialsPolicy storedCredentialsPolicy)
924 {
925 #if ENABLE(SERVER_PRECONNECT)
926     NetworkLoadParameters parameters;
927     parameters.request = ResourceRequest { url };
928     parameters.sessionID = PAL::SessionID::defaultSessionID();
929     parameters.storedCredentialsPolicy = storedCredentialsPolicy;
930     parameters.shouldPreconnectOnly = PreconnectOnly::Yes;
931
932     new PreconnectTask(WTFMove(parameters));
933 #else
934     UNUSED_PARAM(url);
935     UNUSED_PARAM(storedCredentialsPolicy);
936 #endif
937 }
938
939 void NetworkProcess::registerURLSchemeAsSecure(const String& scheme) const
940 {
941     SchemeRegistry::registerURLSchemeAsSecure(scheme);
942 }
943
944 void NetworkProcess::registerURLSchemeAsBypassingContentSecurityPolicy(const String& scheme) const
945 {
946     SchemeRegistry::registerURLSchemeAsBypassingContentSecurityPolicy(scheme);
947 }
948
949 void NetworkProcess::registerURLSchemeAsLocal(const String& scheme) const
950 {
951     SchemeRegistry::registerURLSchemeAsLocal(scheme);
952 }
953
954 void NetworkProcess::registerURLSchemeAsNoAccess(const String& scheme) const
955 {
956     SchemeRegistry::registerURLSchemeAsNoAccess(scheme);
957 }
958
959 void NetworkProcess::registerURLSchemeAsDisplayIsolated(const String& scheme) const
960 {
961     SchemeRegistry::registerURLSchemeAsDisplayIsolated(scheme);
962 }
963
964 void NetworkProcess::registerURLSchemeAsCORSEnabled(const String& scheme) const
965 {
966     SchemeRegistry::registerURLSchemeAsCORSEnabled(scheme);
967 }
968
969 void NetworkProcess::registerURLSchemeAsCanDisplayOnlyIfCanRequest(const String& scheme) const
970 {
971     SchemeRegistry::registerAsCanDisplayOnlyIfCanRequest(scheme);
972 }
973
974 void NetworkProcess::didSyncAllCookies()
975 {
976     parentProcessConnection()->send(Messages::NetworkProcessProxy::DidSyncAllCookies(), 0);
977 }
978
979 #if !PLATFORM(COCOA)
980 void NetworkProcess::initializeProcess(const ChildProcessInitializationParameters&)
981 {
982 }
983
984 void NetworkProcess::initializeProcessName(const ChildProcessInitializationParameters&)
985 {
986 }
987
988 void NetworkProcess::initializeSandbox(const ChildProcessInitializationParameters&, SandboxInitializationParameters&)
989 {
990 }
991
992 void NetworkProcess::syncAllCookies()
993 {
994 }
995
996 void NetworkProcess::platformSyncAllCookies(CompletionHandler<void()>&& completionHandler)
997 {
998     completionHandler();
999 }
1000
1001 #endif
1002
1003 } // namespace WebKit