Make _WKWebsiteDataStoreConfiguration SPI for HSTS storage to replace _WKProcessPoolC...
[WebKit-https.git] / Source / WebKit / NetworkProcess / NetworkProcess.cpp
1 /*
2  * Copyright (C) 2012-2020 Apple Inc. All rights reserved.
3  * Copyright (C) 2018 Sony Interactive Entertainment Inc.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24  * THE POSSIBILITY OF SUCH DAMAGE.
25  */
26
27 #include "config.h"
28 #include "NetworkProcess.h"
29
30 #include "ArgumentCoders.h"
31 #include "Attachment.h"
32 #include "AuthenticationManager.h"
33 #include "AuxiliaryProcessMessages.h"
34 #include "DataReference.h"
35 #include "Download.h"
36 #include "DownloadProxyMessages.h"
37 #if ENABLE(LEGACY_CUSTOM_PROTOCOL_MANAGER)
38 #include "LegacyCustomProtocolManager.h"
39 #endif
40 #include "Logging.h"
41 #include "NetworkConnectionToWebProcess.h"
42 #include "NetworkContentRuleListManagerMessages.h"
43 #include "NetworkLoad.h"
44 #include "NetworkProcessCreationParameters.h"
45 #include "NetworkProcessPlatformStrategies.h"
46 #include "NetworkProcessProxyMessages.h"
47 #include "NetworkResourceLoader.h"
48 #include "NetworkSession.h"
49 #include "NetworkSessionCreationParameters.h"
50 #include "PreconnectTask.h"
51 #include "RemoteNetworkingContext.h"
52 #include "ShouldGrandfatherStatistics.h"
53 #include "StorageAccessStatus.h"
54 #include "StorageManagerSet.h"
55 #include "WebCookieManager.h"
56 #include "WebPageProxyMessages.h"
57 #include "WebProcessPoolMessages.h"
58 #include "WebResourceLoadStatisticsStore.h"
59 #include "WebSWOriginStore.h"
60 #include "WebSWServerConnection.h"
61 #include "WebSWServerToContextConnection.h"
62 #include "WebsiteDataFetchOption.h"
63 #include "WebsiteDataStore.h"
64 #include "WebsiteDataStoreParameters.h"
65 #include "WebsiteDataType.h"
66 #include <WebCore/ClientOrigin.h>
67 #include <WebCore/CookieJar.h>
68 #include <WebCore/DNS.h>
69 #include <WebCore/DeprecatedGlobalSettings.h>
70 #include <WebCore/DiagnosticLoggingClient.h>
71 #include <WebCore/HTTPCookieAcceptPolicy.h>
72 #include <WebCore/LegacySchemeRegistry.h>
73 #include <WebCore/LogInitialization.h>
74 #include <WebCore/MIMETypeRegistry.h>
75 #include <WebCore/NetworkStateNotifier.h>
76 #include <WebCore/NetworkStorageSession.h>
77 #include <WebCore/ResourceRequest.h>
78 #include <WebCore/RuntimeApplicationChecks.h>
79 #include <WebCore/RuntimeEnabledFeatures.h>
80 #include <WebCore/SecurityOrigin.h>
81 #include <WebCore/SecurityOriginData.h>
82 #include <WebCore/StorageQuotaManager.h>
83 #include <wtf/Algorithms.h>
84 #include <wtf/CallbackAggregator.h>
85 #include <wtf/OptionSet.h>
86 #include <wtf/ProcessPrivilege.h>
87 #include <wtf/RunLoop.h>
88 #include <wtf/UUID.h>
89 #include <wtf/UniqueRef.h>
90 #include <wtf/text/AtomString.h>
91
92 #if ENABLE(SEC_ITEM_SHIM)
93 #include "SecItemShim.h"
94 #endif
95
96 #include "NetworkCache.h"
97 #include "NetworkCacheCoders.h"
98
99 #if PLATFORM(COCOA)
100 #include "NetworkSessionCocoa.h"
101 #endif
102
103 #if USE(SOUP)
104 #include "NetworkSessionSoup.h"
105 #include <WebCore/DNSResolveQueueSoup.h>
106 #include <WebCore/SoupNetworkSession.h>
107 #endif
108
109 #if USE(CURL)
110 #include <WebCore/CurlContext.h>
111 #endif
112
113 #if ENABLE(SERVICE_WORKER)
114 #include "WebSWServerToContextConnectionMessages.h"
115 #endif
116
117 namespace WebKit {
118 using namespace WebCore;
119
120 static void callExitSoon(IPC::Connection*)
121 {
122     // If the connection has been closed and we haven't responded in the main thread for 10 seconds
123     // the process will exit forcibly.
124     auto watchdogDelay = 10_s;
125
126     WorkQueue::create("com.apple.WebKit.NetworkProcess.WatchDogQueue")->dispatchAfter(watchdogDelay, [] {
127         // We use _exit here since the watchdog callback is called from another thread and we don't want
128         // global destructors or atexit handlers to be called from this thread while the main thread is busy
129         // doing its thing.
130         RELEASE_LOG_ERROR(IPC, "Exiting process early due to unacknowledged closed-connection");
131         _exit(EXIT_FAILURE);
132     });
133 }
134
135 static inline MessagePortChannelRegistry createMessagePortChannelRegistry(NetworkProcess& networkProcess)
136 {
137     return MessagePortChannelRegistry { [&networkProcess](auto& messagePortIdentifier, auto processIdentifier, auto&& completionHandler) {
138         auto* connection = networkProcess.webProcessConnection(processIdentifier);
139         if (!connection) {
140             completionHandler(MessagePortChannelProvider::HasActivity::No);
141             return;
142         }
143
144         connection->checkProcessLocalPortForActivity(messagePortIdentifier, WTFMove(completionHandler));
145     } };
146 }
147
148 NetworkProcess::NetworkProcess(AuxiliaryProcessInitializationParameters&& parameters)
149     : m_downloadManager(*this)
150     , m_storageManagerSet(StorageManagerSet::create())
151 #if ENABLE(CONTENT_EXTENSIONS)
152     , m_networkContentRuleListManager(*this)
153 #endif
154 #if PLATFORM(IOS_FAMILY)
155     , m_webSQLiteDatabaseTracker([this](bool isHoldingLockedFiles) { parentProcessConnection()->send(Messages::NetworkProcessProxy::SetIsHoldingLockedFiles(isHoldingLockedFiles), 0); })
156 #endif
157     , m_messagePortChannelRegistry(createMessagePortChannelRegistry(*this))
158 {
159     NetworkProcessPlatformStrategies::initialize();
160
161     addSupplement<AuthenticationManager>();
162     addSupplement<WebCookieManager>();
163 #if ENABLE(LEGACY_CUSTOM_PROTOCOL_MANAGER)
164     addSupplement<LegacyCustomProtocolManager>();
165 #endif
166 #if PLATFORM(COCOA) && ENABLE(LEGACY_CUSTOM_PROTOCOL_MANAGER)
167     LegacyCustomProtocolManager::networkProcessCreated(*this);
168 #endif
169
170 #if USE(SOUP)
171     DNSResolveQueueSoup::setGlobalDefaultSoupSessionAccessor([this]() -> SoupSession* {
172         return static_cast<NetworkSessionSoup&>(*networkSession(PAL::SessionID::defaultSessionID())).soupSession();
173     });
174 #endif
175
176     NetworkStateNotifier::singleton().addListener([weakThis = makeWeakPtr(*this)](bool isOnLine) {
177         if (!weakThis)
178             return;
179         for (auto& webProcessConnection : weakThis->m_webProcessConnections.values())
180             webProcessConnection->setOnLineState(isOnLine);
181     });
182
183     initialize(WTFMove(parameters));
184 }
185
186 NetworkProcess::~NetworkProcess()
187 {
188     for (auto& callbacks : m_cacheStorageParametersCallbacks.values()) {
189         for (auto& callback : callbacks)
190             callback(String { });
191     }
192 }
193
194 AuthenticationManager& NetworkProcess::authenticationManager()
195 {
196     return *supplement<AuthenticationManager>();
197 }
198
199 DownloadManager& NetworkProcess::downloadManager()
200 {
201     return m_downloadManager;
202 }
203
204 void NetworkProcess::removeNetworkConnectionToWebProcess(NetworkConnectionToWebProcess& connection)
205 {
206     ASSERT(m_webProcessConnections.contains(connection.webProcessIdentifier()));
207     m_webProcessConnections.remove(connection.webProcessIdentifier());
208 }
209
210 bool NetworkProcess::shouldTerminate()
211 {
212     // Network process keeps session cookies and credentials, so it should never terminate (as long as UI process connection is alive).
213     return false;
214 }
215
216 void NetworkProcess::didReceiveMessage(IPC::Connection& connection, IPC::Decoder& decoder)
217 {
218     ASSERT(parentProcessConnection() == &connection);
219     if (parentProcessConnection() != &connection) {
220         WTFLogAlways("Ignored message '%s' because it did not come from the UIProcess (destination: %" PRIu64 ")", description(decoder.messageName()), decoder.destinationID());
221         ASSERT_NOT_REACHED();
222         return;
223     }
224
225     if (messageReceiverMap().dispatchMessage(connection, decoder))
226         return;
227
228     if (decoder.messageReceiverName() == Messages::AuxiliaryProcess::messageReceiverName()) {
229         AuxiliaryProcess::didReceiveMessage(connection, decoder);
230         return;
231     }
232
233 #if ENABLE(CONTENT_EXTENSIONS)
234     if (decoder.messageReceiverName() == Messages::NetworkContentRuleListManager::messageReceiverName()) {
235         m_networkContentRuleListManager.didReceiveMessage(connection, decoder);
236         return;
237     }
238 #endif
239
240     didReceiveNetworkProcessMessage(connection, decoder);
241 }
242
243 void NetworkProcess::didReceiveSyncMessage(IPC::Connection& connection, IPC::Decoder& decoder, std::unique_ptr<IPC::Encoder>& replyEncoder)
244 {
245     ASSERT(parentProcessConnection() == &connection);
246     if (parentProcessConnection() != &connection) {
247         WTFLogAlways("Ignored message '%s' because it did not come from the UIProcess (destination: %" PRIu64 ")", description(decoder.messageName()), decoder.destinationID());
248         ASSERT_NOT_REACHED();
249         return;
250     }
251
252     if (messageReceiverMap().dispatchSyncMessage(connection, decoder, replyEncoder))
253         return;
254
255     didReceiveSyncNetworkProcessMessage(connection, decoder, replyEncoder);
256 }
257
258 void NetworkProcess::didClose(IPC::Connection&)
259 {
260     ASSERT(RunLoop::isMain());
261
262     auto callbackAggregator = CallbackAggregator::create([this] {
263         ASSERT(RunLoop::isMain());
264         stopRunLoop();
265     });
266
267     // Make sure we flush all cookies and resource load statistics to disk before exiting.
268 #if ENABLE(RESOURCE_LOAD_STATISTICS)
269     forEachNetworkSession([&] (auto& networkSession) {
270         networkSession.flushAndDestroyPersistentStore([callbackAggregator] { });
271     });
272 #endif
273     platformSyncAllCookies([callbackAggregator] { });
274 }
275
276 void NetworkProcess::didCreateDownload()
277 {
278     disableTermination();
279 }
280
281 void NetworkProcess::didDestroyDownload()
282 {
283     enableTermination();
284 }
285
286 IPC::Connection* NetworkProcess::downloadProxyConnection()
287 {
288     return parentProcessConnection();
289 }
290
291 AuthenticationManager& NetworkProcess::downloadsAuthenticationManager()
292 {
293     return authenticationManager();
294 }
295
296 void NetworkProcess::lowMemoryHandler(Critical critical)
297 {
298     if (m_suppressMemoryPressureHandler)
299         return;
300
301     WTF::releaseFastMallocFreeMemory();
302
303     forEachNetworkSession([](auto& networkSession) {
304         networkSession.clearPrefetchCache();
305     });
306
307 #if ENABLE(SERVICE_WORKER)
308     for (auto& swServer : m_swServers.values())
309         swServer->handleLowMemoryWarning();
310 #endif
311 }
312
313 void NetworkProcess::initializeNetworkProcess(NetworkProcessCreationParameters&& parameters)
314 {
315 #if HAVE(SEC_KEY_PROXY)
316     WTF::setProcessPrivileges({ ProcessPrivilege::CanAccessRawCookies });
317 #else
318     WTF::setProcessPrivileges({ ProcessPrivilege::CanAccessRawCookies, ProcessPrivilege::CanAccessCredentials });
319 #endif
320     WebCore::NetworkStorageSession::permitProcessToUseCookieAPI(true);
321     platformInitializeNetworkProcess(parameters);
322
323     WTF::Thread::setCurrentThreadIsUserInitiated();
324     AtomString::init();
325
326     m_suppressMemoryPressureHandler = parameters.shouldSuppressMemoryPressureHandler;
327     if (!m_suppressMemoryPressureHandler) {
328         auto& memoryPressureHandler = MemoryPressureHandler::singleton();
329         memoryPressureHandler.setLowMemoryHandler([this] (Critical critical, Synchronous) {
330             lowMemoryHandler(critical);
331         });
332         memoryPressureHandler.install();
333     }
334
335     setCacheModel(parameters.cacheModel);
336
337 #if ENABLE(RESOURCE_LOAD_STATISTICS)
338     m_isITPDatabaseEnabled = parameters.shouldEnableITPDatabase;
339 #endif
340
341     setAdClickAttributionDebugMode(parameters.enableAdClickAttributionDebugMode);
342
343     SandboxExtension::consumePermanently(parameters.defaultDataStoreParameters.networkSessionParameters.resourceLoadStatisticsParameters.directoryExtensionHandle);
344
345     auto sessionID = parameters.defaultDataStoreParameters.networkSessionParameters.sessionID;
346     setSession(sessionID, NetworkSession::create(*this, WTFMove(parameters.defaultDataStoreParameters.networkSessionParameters)));
347
348     SandboxExtension::consumePermanently(parameters.defaultDataStoreParameters.cacheStorageDirectoryExtensionHandle);
349     addSessionStorageQuotaManager(sessionID, parameters.defaultDataStoreParameters.perOriginStorageQuota, parameters.defaultDataStoreParameters.perThirdPartyOriginStorageQuota, parameters.defaultDataStoreParameters.cacheStorageDirectory, parameters.defaultDataStoreParameters.cacheStorageDirectoryExtensionHandle);
350
351 #if ENABLE(INDEXED_DATABASE)
352     addIndexedDatabaseSession(sessionID, parameters.defaultDataStoreParameters.indexedDatabaseDirectory, parameters.defaultDataStoreParameters.indexedDatabaseDirectoryExtensionHandle);
353 #endif
354
355 #if ENABLE(SERVICE_WORKER)
356     bool serviceWorkerProcessTerminationDelayEnabled = true;
357     addServiceWorkerSession(PAL::SessionID::defaultSessionID(), serviceWorkerProcessTerminationDelayEnabled, WTFMove(parameters.serviceWorkerRegistrationDirectory), parameters.serviceWorkerRegistrationDirectoryExtensionHandle);
358 #endif
359
360     m_storageManagerSet->add(sessionID, parameters.defaultDataStoreParameters.localStorageDirectory, parameters.defaultDataStoreParameters.localStorageDirectoryExtensionHandle);
361
362     auto* defaultSession = networkSession(PAL::SessionID::defaultSessionID());
363     auto* defaultStorageSession = defaultSession->networkStorageSession();
364     for (const auto& cookie : parameters.defaultDataStoreParameters.pendingCookies)
365         defaultStorageSession->setCookie(cookie);
366
367     for (auto& supplement : m_supplements.values())
368         supplement->initialize(parameters);
369
370     for (auto& scheme : parameters.urlSchemesRegisteredAsSecure)
371         registerURLSchemeAsSecure(scheme);
372
373     for (auto& scheme : parameters.urlSchemesRegisteredAsBypassingContentSecurityPolicy)
374         registerURLSchemeAsBypassingContentSecurityPolicy(scheme);
375
376     for (auto& scheme : parameters.urlSchemesRegisteredAsLocal)
377         registerURLSchemeAsLocal(scheme);
378
379     for (auto& scheme : parameters.urlSchemesRegisteredAsNoAccess)
380         registerURLSchemeAsNoAccess(scheme);
381     
382     RELEASE_LOG(Process, "%p - NetworkProcess::initializeNetworkProcess: Presenting process = %d", this, WebCore::presentingApplicationPID());
383 }
384
385 void NetworkProcess::initializeConnection(IPC::Connection* connection)
386 {
387     AuxiliaryProcess::initializeConnection(connection);
388
389     // We give a chance for didClose() to get called on the main thread but forcefully call _exit() after a delay
390     // in case the main thread is unresponsive or didClose() takes too long.
391     connection->setDidCloseOnConnectionWorkQueueCallback(callExitSoon);
392
393     for (auto& supplement : m_supplements.values())
394         supplement->initializeConnection(connection);
395 }
396
397 void NetworkProcess::createNetworkConnectionToWebProcess(ProcessIdentifier identifier, PAL::SessionID sessionID, CompletionHandler<void(Optional<IPC::Attachment>&&, HTTPCookieAcceptPolicy)>&& completionHandler)
398 {
399     auto ipcConnection = createIPCConnectionPair();
400     if (!ipcConnection) {
401         completionHandler({ }, HTTPCookieAcceptPolicy::Never);
402         return;
403     }
404
405     auto newConnection = NetworkConnectionToWebProcess::create(*this, identifier, sessionID, ipcConnection->first);
406     auto& connection = newConnection.get();
407
408     ASSERT(!m_webProcessConnections.contains(identifier));
409     m_webProcessConnections.add(identifier, WTFMove(newConnection));
410
411     auto* storage = storageSession(sessionID);
412     completionHandler(WTFMove(ipcConnection->second), storage ? storage->cookieAcceptPolicy() : HTTPCookieAcceptPolicy::Never);
413
414     connection.setOnLineState(NetworkStateNotifier::singleton().onLine());
415
416     m_storageManagerSet->addConnection(connection.connection());
417
418     webIDBServer(sessionID).addConnection(connection.connection(), identifier);
419 }
420
421 void NetworkProcess::clearCachedCredentials()
422 {
423     defaultStorageSession().credentialStorage().clearCredentials();
424     if (auto* networkSession = this->networkSession(PAL::SessionID::defaultSessionID()))
425         networkSession->clearCredentials();
426     else
427         ASSERT_NOT_REACHED();
428
429     forEachNetworkSession([] (auto& networkSession) {
430         if (auto storageSession = networkSession.networkStorageSession())
431             storageSession->credentialStorage().clearCredentials();
432         networkSession.clearCredentials();
433     });
434 }
435
436 void NetworkProcess::addWebsiteDataStore(WebsiteDataStoreParameters&& parameters)
437 {
438     auto sessionID = parameters.networkSessionParameters.sessionID;
439
440     addSessionStorageQuotaManager(sessionID, parameters.perOriginStorageQuota, parameters.perThirdPartyOriginStorageQuota, parameters.cacheStorageDirectory, parameters.cacheStorageDirectoryExtensionHandle);
441
442 #if ENABLE(INDEXED_DATABASE)
443     addIndexedDatabaseSession(sessionID, parameters.indexedDatabaseDirectory, parameters.indexedDatabaseDirectoryExtensionHandle);
444 #endif
445
446 #if ENABLE(SERVICE_WORKER)
447     addServiceWorkerSession(sessionID, parameters.serviceWorkerProcessTerminationDelayEnabled, WTFMove(parameters.serviceWorkerRegistrationDirectory), parameters.serviceWorkerRegistrationDirectoryExtensionHandle);
448 #endif
449
450     m_storageManagerSet->add(sessionID, parameters.localStorageDirectory, parameters.localStorageDirectoryExtensionHandle);
451
452     RemoteNetworkingContext::ensureWebsiteDataStoreSession(*this, WTFMove(parameters));
453 }
454
455 void NetworkProcess::addSessionStorageQuotaManager(PAL::SessionID sessionID, uint64_t defaultQuota, uint64_t defaultThirdPartyQuota, const String& cacheRootPath, SandboxExtension::Handle& cacheRootPathHandle)
456 {
457     LockHolder locker(m_sessionStorageQuotaManagersLock);
458     auto isNewEntry = m_sessionStorageQuotaManagers.ensure(sessionID, [defaultQuota, defaultThirdPartyQuota, &cacheRootPath] {
459         return makeUnique<SessionStorageQuotaManager>(cacheRootPath, defaultQuota, defaultThirdPartyQuota);
460     }).isNewEntry;
461     if (isNewEntry) {
462         SandboxExtension::consumePermanently(cacheRootPathHandle);
463         if (!cacheRootPath.isEmpty())
464             postStorageTask(createCrossThreadTask(*this, &NetworkProcess::ensurePathExists, cacheRootPath));
465     }
466 }
467
468 void NetworkProcess::removeSessionStorageQuotaManager(PAL::SessionID sessionID)
469 {
470     LockHolder locker(m_sessionStorageQuotaManagersLock);
471     ASSERT(m_sessionStorageQuotaManagers.contains(sessionID));
472     m_sessionStorageQuotaManagers.remove(sessionID);
473 }
474
475 void NetworkProcess::forEachNetworkSession(const Function<void(NetworkSession&)>& functor)
476 {
477     for (auto& session : m_networkSessions.values())
478         functor(*session);
479 }
480
481 std::unique_ptr<WebCore::NetworkStorageSession> NetworkProcess::newTestingSession(const PAL::SessionID& sessionID)
482 {
483 #if PLATFORM(COCOA)
484     // Session name should be short enough for shared memory region name to be under the limit, otherwise sandbox rules won't work (see <rdar://problem/13642852>).
485     String sessionName = makeString("WebKit Test-", getCurrentProcessID());
486
487     auto session = adoptCF(WebCore::createPrivateStorageSession(sessionName.createCFString().get()));
488
489     RetainPtr<CFHTTPCookieStorageRef> cookieStorage;
490     if (WebCore::NetworkStorageSession::processMayUseCookieAPI()) {
491         ASSERT(hasProcessPrivilege(ProcessPrivilege::CanAccessRawCookies));
492         if (session)
493             cookieStorage = adoptCF(_CFURLStorageSessionCopyCookieStorage(kCFAllocatorDefault, session.get()));
494     }
495
496     return makeUnique<WebCore::NetworkStorageSession>(sessionID, WTFMove(session), WTFMove(cookieStorage));
497 #elif USE(CURL) || USE(SOUP)
498     return makeUnique<WebCore::NetworkStorageSession>(sessionID);
499 #endif
500 }
501
502 #if PLATFORM(COCOA)
503 void NetworkProcess::ensureSession(const PAL::SessionID& sessionID, bool shouldUseTestingNetworkSession, const String& identifierBase, RetainPtr<CFHTTPCookieStorageRef>&& cookieStorage)
504 #else
505 void NetworkProcess::ensureSession(const PAL::SessionID& sessionID, bool shouldUseTestingNetworkSession, const String& identifierBase)
506 #endif
507 {
508     ASSERT(sessionID != PAL::SessionID::defaultSessionID());
509
510     auto addResult = m_networkStorageSessions.add(sessionID, nullptr);
511     if (!addResult.isNewEntry)
512         return;
513
514     if (shouldUseTestingNetworkSession) {
515         addResult.iterator->value = newTestingSession(sessionID);
516         return;
517     }
518     
519 #if PLATFORM(COCOA)
520     RetainPtr<CFURLStorageSessionRef> storageSession;
521     RetainPtr<CFStringRef> cfIdentifier = makeString(identifierBase, ".PrivateBrowsing.", createCanonicalUUIDString()).createCFString();
522     if (sessionID.isEphemeral())
523         storageSession = adoptCF(createPrivateStorageSession(cfIdentifier.get()));
524     else
525         storageSession = WebCore::NetworkStorageSession::createCFStorageSessionForIdentifier(cfIdentifier.get());
526
527     if (NetworkStorageSession::processMayUseCookieAPI()) {
528         ASSERT(hasProcessPrivilege(ProcessPrivilege::CanAccessRawCookies));
529         if (!cookieStorage && storageSession)
530             cookieStorage = adoptCF(_CFURLStorageSessionCopyCookieStorage(kCFAllocatorDefault, storageSession.get()));
531     }
532
533     addResult.iterator->value = makeUnique<NetworkStorageSession>(sessionID, WTFMove(storageSession), WTFMove(cookieStorage));
534 #elif USE(CURL) || USE(SOUP)
535     addResult.iterator->value = makeUnique<NetworkStorageSession>(sessionID);
536 #endif
537 }
538
539 void NetworkProcess::cookieAcceptPolicyChanged(HTTPCookieAcceptPolicy newPolicy)
540 {
541     for (auto& connection : m_webProcessConnections.values())
542         connection->cookieAcceptPolicyChanged(newPolicy);
543 }
544
545 WebCore::NetworkStorageSession* NetworkProcess::storageSession(const PAL::SessionID& sessionID) const
546 {
547     if (sessionID == PAL::SessionID::defaultSessionID())
548         return &defaultStorageSession();
549     return m_networkStorageSessions.get(sessionID);
550 }
551
552 WebCore::NetworkStorageSession& NetworkProcess::defaultStorageSession() const
553 {
554     if (!m_defaultNetworkStorageSession)
555         m_defaultNetworkStorageSession = platformCreateDefaultStorageSession();
556     return *m_defaultNetworkStorageSession;
557 }
558
559 void NetworkProcess::forEachNetworkStorageSession(const Function<void(WebCore::NetworkStorageSession&)>& functor)
560 {
561     functor(defaultStorageSession());
562     for (auto& storageSession : m_networkStorageSessions.values())
563         functor(*storageSession);
564 }
565
566 NetworkSession* NetworkProcess::networkSession(PAL::SessionID sessionID) const
567 {
568     ASSERT(RunLoop::isMain());
569     return m_networkSessions.get(sessionID);
570 }
571
572 void NetworkProcess::setSession(PAL::SessionID sessionID, std::unique_ptr<NetworkSession>&& session)
573 {
574     ASSERT(RunLoop::isMain());
575     m_networkSessions.set(sessionID, WTFMove(session));
576 }
577
578 void NetworkProcess::destroySession(PAL::SessionID sessionID)
579 {
580     ASSERT(RunLoop::isMain());
581 #if !USE(SOUP) && !USE(CURL)
582     // cURL and Soup based ports destroy the default session right before the process exits to avoid leaking
583     // network resources like the cookies database.
584     ASSERT(sessionID != PAL::SessionID::defaultSessionID());
585 #endif
586
587     if (auto session = m_networkSessions.take(sessionID))
588         session->invalidateAndCancel();
589     m_networkStorageSessions.remove(sessionID);
590     m_sessionsControlledByAutomation.remove(sessionID);
591     CacheStorage::Engine::destroyEngine(*this, sessionID);
592
593 #if ENABLE(SERVICE_WORKER)
594     m_swServers.remove(sessionID);
595     m_serviceWorkerInfo.remove(sessionID);
596 #endif
597
598     m_storageManagerSet->remove(sessionID);
599     if (auto webIDBServer = m_webIDBServers.take(sessionID))
600         webIDBServer->close();
601 }
602
603 #if ENABLE(RESOURCE_LOAD_STATISTICS)
604 void NetworkProcess::dumpResourceLoadStatistics(PAL::SessionID sessionID, CompletionHandler<void(String)>&& completionHandler)
605 {
606     if (auto* networkSession = this->networkSession(sessionID)) {
607         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
608             resourceLoadStatistics->dumpResourceLoadStatistics(WTFMove(completionHandler));
609         else
610             completionHandler({ });
611     } else {
612         ASSERT_NOT_REACHED();
613         completionHandler({ });
614     }
615 }
616
617 void NetworkProcess::updatePrevalentDomainsToBlockCookiesFor(PAL::SessionID sessionID, const Vector<RegistrableDomain>& domainsToBlock, CompletionHandler<void()>&& completionHandler)
618 {
619     if (auto* networkStorageSession = storageSession(sessionID))
620         networkStorageSession->setPrevalentDomainsToBlockAndDeleteCookiesFor(domainsToBlock);
621     completionHandler();
622 }
623
624 void NetworkProcess::isGrandfathered(PAL::SessionID sessionID, const RegistrableDomain& domain, CompletionHandler<void(bool)>&& completionHandler)
625 {
626     if (auto* networkSession = this->networkSession(sessionID)) {
627         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
628             resourceLoadStatistics->isGrandfathered(domain, WTFMove(completionHandler));
629         else
630             completionHandler(false);
631     } else {
632         ASSERT_NOT_REACHED();
633         completionHandler(false);
634     }
635 }
636
637 void NetworkProcess::isPrevalentResource(PAL::SessionID sessionID, const RegistrableDomain& domain, CompletionHandler<void(bool)>&& completionHandler)
638 {
639     if (auto* networkSession = this->networkSession(sessionID)) {
640         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
641             resourceLoadStatistics->isPrevalentResource(domain, WTFMove(completionHandler));
642         else
643             completionHandler(false);
644     } else {
645         ASSERT_NOT_REACHED();
646         completionHandler(false);
647     }
648 }
649
650 void NetworkProcess::isVeryPrevalentResource(PAL::SessionID sessionID, const RegistrableDomain& domain, CompletionHandler<void(bool)>&& completionHandler)
651 {
652     if (auto* networkSession = this->networkSession(sessionID)) {
653         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
654             resourceLoadStatistics->isVeryPrevalentResource(domain, WTFMove(completionHandler));
655         else
656             completionHandler(false);
657     } else {
658         ASSERT_NOT_REACHED();
659         completionHandler(false);
660     }
661 }
662
663 void NetworkProcess::setAgeCapForClientSideCookies(PAL::SessionID sessionID, Optional<Seconds> seconds, CompletionHandler<void()>&& completionHandler)
664 {
665     if (auto* networkStorageSession = storageSession(sessionID))
666         networkStorageSession->setAgeCapForClientSideCookies(seconds);
667     completionHandler();
668 }
669
670 void NetworkProcess::setGrandfathered(PAL::SessionID sessionID, const RegistrableDomain& domain, bool isGrandfathered, CompletionHandler<void()>&& completionHandler)
671 {
672     if (auto* networkSession = this->networkSession(sessionID)) {
673         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
674             resourceLoadStatistics->setGrandfathered(domain, isGrandfathered, WTFMove(completionHandler));
675         else
676             completionHandler();
677     } else {
678         ASSERT_NOT_REACHED();
679         completionHandler();
680     }
681 }
682
683 void NetworkProcess::setUseITPDatabase(PAL::SessionID sessionID, bool value, CompletionHandler<void()>&& completionHandler)
684 {
685     if (auto* networkSession = this->networkSession(sessionID)) {
686         if (m_isITPDatabaseEnabled != value) {
687             m_isITPDatabaseEnabled = value;
688             networkSession->recreateResourceLoadStatisticStore(WTFMove(completionHandler));
689         } else
690             completionHandler();
691     } else {
692         ASSERT_NOT_REACHED();
693         completionHandler();
694     }
695 }
696
697 void NetworkProcess::setPrevalentResource(PAL::SessionID sessionID, const RegistrableDomain& domain, CompletionHandler<void()>&& completionHandler)
698 {
699     if (auto* networkSession = this->networkSession(sessionID)) {
700         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
701             resourceLoadStatistics->setPrevalentResource(domain, WTFMove(completionHandler));
702         else
703             completionHandler();
704     } else {
705         ASSERT_NOT_REACHED();
706         completionHandler();
707     }
708 }
709
710 void NetworkProcess::setPrevalentResourceForDebugMode(PAL::SessionID sessionID, const RegistrableDomain& domain, CompletionHandler<void()>&& completionHandler)
711 {
712     if (auto* networkSession = this->networkSession(sessionID)) {
713         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
714             resourceLoadStatistics->setPrevalentResourceForDebugMode(domain, WTFMove(completionHandler));
715         else
716             completionHandler();
717     } else {
718         ASSERT_NOT_REACHED();
719         completionHandler();
720     }
721 }
722
723 void NetworkProcess::setVeryPrevalentResource(PAL::SessionID sessionID, const RegistrableDomain& domain, CompletionHandler<void()>&& completionHandler)
724 {
725     if (auto* networkSession = this->networkSession(sessionID)) {
726         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
727             resourceLoadStatistics->setVeryPrevalentResource(domain, WTFMove(completionHandler));
728         else
729             completionHandler();
730     } else {
731         ASSERT_NOT_REACHED();
732         completionHandler();
733     }
734 }
735
736 void NetworkProcess::clearPrevalentResource(PAL::SessionID sessionID, const RegistrableDomain& domain, CompletionHandler<void()>&& completionHandler)
737 {
738     if (auto* networkSession = this->networkSession(sessionID)) {
739         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
740             resourceLoadStatistics->clearPrevalentResource(domain, WTFMove(completionHandler));
741         else
742             completionHandler();
743     } else {
744         ASSERT_NOT_REACHED();
745         completionHandler();
746     }
747 }
748
749 void NetworkProcess::submitTelemetry(PAL::SessionID sessionID, CompletionHandler<void()>&& completionHandler)
750 {
751     if (auto* networkSession = this->networkSession(sessionID)) {
752         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
753             resourceLoadStatistics->submitTelemetry(WTFMove(completionHandler));
754         else
755             completionHandler();
756     } else {
757         ASSERT_NOT_REACHED();
758         completionHandler();
759     }
760 }
761
762 void NetworkProcess::scheduleCookieBlockingUpdate(PAL::SessionID sessionID, CompletionHandler<void()>&& completionHandler)
763 {
764     if (auto* networkSession = this->networkSession(sessionID)) {
765         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
766             resourceLoadStatistics->scheduleCookieBlockingUpdate(WTFMove(completionHandler));
767         else
768             completionHandler();
769     } else {
770         ASSERT_NOT_REACHED();
771         completionHandler();
772     }
773 }
774
775 void NetworkProcess::scheduleClearInMemoryAndPersistent(PAL::SessionID sessionID, Optional<WallTime> modifiedSince, ShouldGrandfatherStatistics shouldGrandfather, CompletionHandler<void()>&& completionHandler)
776 {
777     if (auto* networkSession = this->networkSession(sessionID)) {
778         networkSession->clearIsolatedSessions();
779         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics()) {
780             if (modifiedSince)
781                 resourceLoadStatistics->scheduleClearInMemoryAndPersistent(modifiedSince.value(), shouldGrandfather, WTFMove(completionHandler));
782             else
783                 resourceLoadStatistics->scheduleClearInMemoryAndPersistent(shouldGrandfather, WTFMove(completionHandler));
784         } else
785             completionHandler();
786     } else {
787         ASSERT_NOT_REACHED();
788         completionHandler();
789     }
790 }
791
792 void NetworkProcess::getResourceLoadStatisticsDataSummary(PAL::SessionID sessionID, CompletionHandler<void(Vector<WebResourceLoadStatisticsStore::ThirdPartyData>&&)>&& completionHandler)
793 {
794     if (auto* networkSession = this->networkSession(sessionID)) {
795         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
796             resourceLoadStatistics->aggregatedThirdPartyData(WTFMove(completionHandler));
797         else
798             completionHandler({ });
799     } else {
800         ASSERT_NOT_REACHED();
801         completionHandler({ });
802     }
803 }
804
805 void NetworkProcess::resetParametersToDefaultValues(PAL::SessionID sessionID, CompletionHandler<void()>&& completionHandler)
806 {
807     if (auto* networkSession = this->networkSession(sessionID)) {
808         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
809             resourceLoadStatistics->resetParametersToDefaultValues(WTFMove(completionHandler));
810         else
811             completionHandler();
812     } else {
813         ASSERT_NOT_REACHED();
814         completionHandler();
815     }
816 }
817
818 void NetworkProcess::scheduleStatisticsAndDataRecordsProcessing(PAL::SessionID sessionID, CompletionHandler<void()>&& completionHandler)
819 {
820     if (auto* networkSession = this->networkSession(sessionID)) {
821         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
822             resourceLoadStatistics->scheduleStatisticsAndDataRecordsProcessing(WTFMove(completionHandler));
823         else
824             completionHandler();
825     } else {
826         ASSERT_NOT_REACHED();
827         completionHandler();
828     }
829 }
830
831 void NetworkProcess::statisticsDatabaseHasAllTables(PAL::SessionID sessionID, CompletionHandler<void(bool)>&& completionHandler)
832 {
833     if (auto* networkSession = this->networkSession(sessionID)) {
834         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
835             resourceLoadStatistics->statisticsDatabaseHasAllTables(WTFMove(completionHandler));
836         else
837             completionHandler(false);
838     } else {
839         ASSERT_NOT_REACHED();
840         completionHandler(false);
841     }
842 }
843
844 void NetworkProcess::setNotifyPagesWhenDataRecordsWereScanned(PAL::SessionID sessionID, bool value, CompletionHandler<void()>&& completionHandler)
845 {
846     if (auto* networkSession = this->networkSession(sessionID)) {
847         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
848             resourceLoadStatistics->setNotifyPagesWhenDataRecordsWereScanned(value, WTFMove(completionHandler));
849         else
850             completionHandler();
851     } else {
852         ASSERT_NOT_REACHED();
853         completionHandler();
854     }
855 }
856
857 void NetworkProcess::setIsRunningResourceLoadStatisticsTest(PAL::SessionID sessionID, bool value, CompletionHandler<void()>&& completionHandler)
858 {
859     if (auto* networkSession = this->networkSession(sessionID)) {
860         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
861             resourceLoadStatistics->setIsRunningTest(value, WTFMove(completionHandler));
862         else
863             completionHandler();
864     } else {
865         ASSERT_NOT_REACHED();
866         completionHandler();
867     }
868 }
869
870 void NetworkProcess::setNotifyPagesWhenTelemetryWasCaptured(PAL::SessionID sessionID, bool value, CompletionHandler<void()>&& completionHandler)
871 {
872     if (auto* networkSession = this->networkSession(sessionID)) {
873         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
874             resourceLoadStatistics->setNotifyPagesWhenTelemetryWasCaptured(value, WTFMove(completionHandler));
875         else
876             completionHandler();
877     } else {
878         ASSERT_NOT_REACHED();
879         completionHandler();
880     }
881 }
882
883 void NetworkProcess::setSubframeUnderTopFrameDomain(PAL::SessionID sessionID, const RegistrableDomain& subFrameDomain, const RegistrableDomain& topFrameDomain, CompletionHandler<void()>&& completionHandler)
884 {
885     if (auto* networkSession = this->networkSession(sessionID)) {
886         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
887             resourceLoadStatistics->setSubframeUnderTopFrameDomain(subFrameDomain, topFrameDomain, WTFMove(completionHandler));
888         else
889             completionHandler();
890     } else {
891         ASSERT_NOT_REACHED();
892         completionHandler();
893     }
894 }
895
896 void NetworkProcess::isRegisteredAsRedirectingTo(PAL::SessionID sessionID, const RegistrableDomain& domainRedirectedFrom, const RegistrableDomain& domainRedirectedTo, CompletionHandler<void(bool)>&& completionHandler)
897 {
898     if (auto* networkSession = this->networkSession(sessionID)) {
899         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
900             resourceLoadStatistics->isRegisteredAsRedirectingTo(domainRedirectedFrom, domainRedirectedTo, WTFMove(completionHandler));
901         else
902             completionHandler(false);
903     } else {
904         ASSERT_NOT_REACHED();
905         completionHandler(false);
906     }
907 }
908
909 void NetworkProcess::isRegisteredAsSubFrameUnder(PAL::SessionID sessionID, const RegistrableDomain& subFrameDomain, const RegistrableDomain& topFrameDomain, CompletionHandler<void(bool)>&& completionHandler)
910 {
911     if (auto* networkSession = this->networkSession(sessionID)) {
912         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
913             resourceLoadStatistics->isRegisteredAsSubFrameUnder(subFrameDomain, topFrameDomain, WTFMove(completionHandler));
914         else
915             completionHandler(false);
916     } else {
917         ASSERT_NOT_REACHED();
918         completionHandler(false);
919     }
920 }
921
922 void NetworkProcess::setSubresourceUnderTopFrameDomain(PAL::SessionID sessionID, const RegistrableDomain& subresourceDomain, const RegistrableDomain& topFrameDomain, CompletionHandler<void()>&& completionHandler)
923 {
924     if (auto* networkSession = this->networkSession(sessionID)) {
925         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
926             resourceLoadStatistics->setSubresourceUnderTopFrameDomain(subresourceDomain, topFrameDomain, WTFMove(completionHandler));
927         else
928             completionHandler();
929     } else {
930         ASSERT_NOT_REACHED();
931         completionHandler();
932     }
933 }
934
935 void NetworkProcess::setSubresourceUniqueRedirectTo(PAL::SessionID sessionID, const RegistrableDomain& subresourceDomain, const RegistrableDomain& domainRedirectedTo, CompletionHandler<void()>&& completionHandler)
936 {
937     if (auto* networkSession = this->networkSession(sessionID)) {
938         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
939             resourceLoadStatistics->setSubresourceUniqueRedirectTo(subresourceDomain, domainRedirectedTo, WTFMove(completionHandler));
940         else
941             completionHandler();
942     } else {
943         ASSERT_NOT_REACHED();
944         completionHandler();
945     }
946 }
947
948 void NetworkProcess::setSubresourceUniqueRedirectFrom(PAL::SessionID sessionID, const RegistrableDomain& subresourceDomain, const RegistrableDomain& domainRedirectedFrom, CompletionHandler<void()>&& completionHandler)
949 {
950     if (auto* networkSession = this->networkSession(sessionID)) {
951         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
952             resourceLoadStatistics->setSubresourceUniqueRedirectFrom(subresourceDomain, domainRedirectedFrom, WTFMove(completionHandler));
953         else
954             completionHandler();
955     } else {
956         ASSERT_NOT_REACHED();
957         completionHandler();
958     }
959 }
960
961 void NetworkProcess::isRegisteredAsSubresourceUnder(PAL::SessionID sessionID, const RegistrableDomain& subresourceDomain, const RegistrableDomain& topFrameDomain, CompletionHandler<void(bool)>&& completionHandler)
962 {
963     if (auto* networkSession = this->networkSession(sessionID)) {
964         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
965             resourceLoadStatistics->isRegisteredAsSubresourceUnder(subresourceDomain, topFrameDomain, WTFMove(completionHandler));
966         else
967             completionHandler(false);
968     } else {
969         ASSERT_NOT_REACHED();
970         completionHandler(false);
971     }
972 }
973
974 void NetworkProcess::setTopFrameUniqueRedirectTo(PAL::SessionID sessionID, const RegistrableDomain& topFrameDomain, const RegistrableDomain& domainRedirectedTo, CompletionHandler<void()>&& completionHandler)
975 {
976     if (auto* networkSession = this->networkSession(sessionID)) {
977         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
978             resourceLoadStatistics->setTopFrameUniqueRedirectTo(topFrameDomain, domainRedirectedTo, WTFMove(completionHandler));
979         else
980             completionHandler();
981     } else {
982         ASSERT_NOT_REACHED();
983         completionHandler();
984     }
985 }
986
987 void NetworkProcess::setTopFrameUniqueRedirectFrom(PAL::SessionID sessionID, const RegistrableDomain& topFrameDomain, const RegistrableDomain& domainRedirectedFrom, CompletionHandler<void()>&& completionHandler)
988 {
989     if (auto* networkSession = this->networkSession(sessionID)) {
990         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
991             resourceLoadStatistics->setTopFrameUniqueRedirectFrom(topFrameDomain, domainRedirectedFrom, WTFMove(completionHandler));
992         else
993             completionHandler();
994     } else {
995         ASSERT_NOT_REACHED();
996         completionHandler();
997     }
998 }
999     
1000     
1001 void NetworkProcess::setLastSeen(PAL::SessionID sessionID, const RegistrableDomain& domain, Seconds seconds, CompletionHandler<void()>&& completionHandler)
1002 {
1003     if (auto* networkSession = this->networkSession(sessionID)) {
1004         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1005             resourceLoadStatistics->setLastSeen(domain, seconds, WTFMove(completionHandler));
1006         else
1007             completionHandler();
1008     } else {
1009         ASSERT_NOT_REACHED();
1010         completionHandler();
1011     }
1012 }
1013
1014 void NetworkProcess::domainIDExistsInDatabase(PAL::SessionID sessionID, int domainID, CompletionHandler<void(bool)>&& completionHandler)
1015 {
1016     if (auto* networkSession = this->networkSession(sessionID)) {
1017         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1018             resourceLoadStatistics->domainIDExistsInDatabase(domainID, WTFMove(completionHandler));
1019         else
1020             completionHandler(false);
1021     } else {
1022         ASSERT_NOT_REACHED();
1023         completionHandler(false);
1024     }
1025 }
1026
1027 void NetworkProcess::mergeStatisticForTesting(PAL::SessionID sessionID, const RegistrableDomain& domain, const RegistrableDomain& topFrameDomain1, const RegistrableDomain& topFrameDomain2, Seconds lastSeen, bool hadUserInteraction, Seconds mostRecentUserInteraction, bool isGrandfathered, bool isPrevalent, bool isVeryPrevalent, unsigned dataRecordsRemoved, CompletionHandler<void()>&& completionHandler)
1028 {
1029     if (auto* networkSession = this->networkSession(sessionID)) {
1030         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1031             resourceLoadStatistics->mergeStatisticForTesting(domain, topFrameDomain1, topFrameDomain2, lastSeen, hadUserInteraction, mostRecentUserInteraction, isGrandfathered, isPrevalent, isVeryPrevalent, dataRecordsRemoved, WTFMove(completionHandler));
1032         else
1033             completionHandler();
1034     } else {
1035         ASSERT_NOT_REACHED();
1036         completionHandler();
1037     }
1038 }
1039
1040 void NetworkProcess::insertExpiredStatisticForTesting(PAL::SessionID sessionID, const RegistrableDomain& domain, bool hadUserInteraction, bool isScheduledForAllButCookieDataRemoval, bool isPrevalent, CompletionHandler<void()>&& completionHandler)
1041 {
1042     if (auto* networkSession = this->networkSession(sessionID)) {
1043         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1044             resourceLoadStatistics->insertExpiredStatisticForTesting(domain, hadUserInteraction, isScheduledForAllButCookieDataRemoval, isPrevalent, WTFMove(completionHandler));
1045         else
1046             completionHandler();
1047     } else {
1048         ASSERT_NOT_REACHED();
1049         completionHandler();
1050     }
1051 }
1052
1053 void NetworkProcess::getAllStorageAccessEntries(PAL::SessionID sessionID, CompletionHandler<void(Vector<String> domains)>&& completionHandler)
1054 {
1055     if (auto* networkStorageSession = storageSession(sessionID))
1056         completionHandler(networkStorageSession->getAllStorageAccessEntries());
1057     else {
1058         ASSERT_NOT_REACHED();
1059         completionHandler({ });
1060     }
1061 }
1062
1063 void NetworkProcess::logFrameNavigation(PAL::SessionID sessionID, const RegistrableDomain& targetDomain, const RegistrableDomain& topFrameDomain, const RegistrableDomain& sourceDomain, bool isRedirect, bool isMainFrame, Seconds delayAfterMainFrameDocumentLoad, bool wasPotentiallyInitiatedByUser)
1064 {
1065     if (auto* networkSession = this->networkSession(sessionID)) {
1066         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1067             resourceLoadStatistics->logFrameNavigation(targetDomain, topFrameDomain, sourceDomain, isRedirect, isMainFrame, delayAfterMainFrameDocumentLoad, wasPotentiallyInitiatedByUser);
1068     } else
1069         ASSERT_NOT_REACHED();
1070 }
1071
1072 void NetworkProcess::logUserInteraction(PAL::SessionID sessionID, const RegistrableDomain& domain, CompletionHandler<void()>&& completionHandler)
1073 {
1074     if (auto* networkSession = this->networkSession(sessionID)) {
1075         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1076             resourceLoadStatistics->logUserInteraction(domain, WTFMove(completionHandler));
1077         else
1078             completionHandler();
1079     } else {
1080         ASSERT_NOT_REACHED();
1081         completionHandler();
1082     }
1083 }
1084
1085 void NetworkProcess::hadUserInteraction(PAL::SessionID sessionID, const RegistrableDomain& domain, CompletionHandler<void(bool)>&& completionHandler)
1086 {
1087     if (auto* networkSession = this->networkSession(sessionID)) {
1088         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1089             resourceLoadStatistics->hasHadUserInteraction(domain, WTFMove(completionHandler));
1090         else
1091             completionHandler(false);
1092     } else {
1093         ASSERT_NOT_REACHED();
1094         completionHandler(false);
1095     }
1096 }
1097
1098 void NetworkProcess::isRelationshipOnlyInDatabaseOnce(PAL::SessionID sessionID, const RegistrableDomain& subDomain, const RegistrableDomain& topDomain, CompletionHandler<void(bool)>&& completionHandler)
1099 {
1100     if (auto* networkSession = this->networkSession(sessionID)) {
1101         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1102             resourceLoadStatistics->isRelationshipOnlyInDatabaseOnce(subDomain, topDomain, WTFMove(completionHandler));
1103         else
1104             completionHandler(false);
1105     } else {
1106         ASSERT_NOT_REACHED();
1107         completionHandler(false);
1108     }
1109 }
1110
1111 void NetworkProcess::clearUserInteraction(PAL::SessionID sessionID, const RegistrableDomain& domain, CompletionHandler<void()>&& completionHandler)
1112 {
1113     if (auto* networkSession = this->networkSession(sessionID)) {
1114         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1115             resourceLoadStatistics->clearUserInteraction(domain, WTFMove(completionHandler));
1116         else
1117             completionHandler();
1118     } else {
1119         ASSERT_NOT_REACHED();
1120         completionHandler();
1121     }
1122 }
1123
1124 void NetworkProcess::hasLocalStorage(PAL::SessionID sessionID, const RegistrableDomain& domain, CompletionHandler<void(bool)>&& completionHandler)
1125 {
1126     if (m_storageManagerSet->contains(sessionID)) {
1127         m_storageManagerSet->getLocalStorageOrigins(sessionID, [domain, completionHandler = WTFMove(completionHandler)](auto&& origins) mutable {
1128             completionHandler(WTF::anyOf(origins, [&domain](auto& origin) {
1129                 return domain.matches(origin);
1130             }));
1131         });
1132         return;
1133     }
1134     completionHandler(false);
1135 }
1136
1137 void NetworkProcess::setCacheMaxAgeCapForPrevalentResources(PAL::SessionID sessionID, Seconds seconds, CompletionHandler<void()>&& completionHandler)
1138 {
1139     if (auto* networkStorageSession = storageSession(sessionID))
1140         networkStorageSession->setCacheMaxAgeCapForPrevalentResources(Seconds { seconds });
1141     else
1142         ASSERT_NOT_REACHED();
1143     completionHandler();
1144 }
1145
1146 void NetworkProcess::setGrandfatheringTime(PAL::SessionID sessionID, Seconds seconds, CompletionHandler<void()>&& completionHandler)
1147 {
1148     if (auto* networkSession = this->networkSession(sessionID)) {
1149         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1150             resourceLoadStatistics->setGrandfatheringTime(seconds, WTFMove(completionHandler));
1151         else
1152             completionHandler();
1153     } else {
1154         ASSERT_NOT_REACHED();
1155         completionHandler();
1156     }
1157 }
1158
1159 void NetworkProcess::setMaxStatisticsEntries(PAL::SessionID sessionID, uint64_t maximumEntryCount, CompletionHandler<void()>&& completionHandler)
1160 {
1161     if (auto* networkSession = this->networkSession(sessionID)) {
1162         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1163             resourceLoadStatistics->setMaxStatisticsEntries(maximumEntryCount, WTFMove(completionHandler));
1164         else
1165             completionHandler();
1166     } else {
1167         ASSERT_NOT_REACHED();
1168         completionHandler();
1169     }
1170 }
1171
1172 void NetworkProcess::setMinimumTimeBetweenDataRecordsRemoval(PAL::SessionID sessionID, Seconds seconds, CompletionHandler<void()>&& completionHandler)
1173 {
1174     if (auto* networkSession = this->networkSession(sessionID)) {
1175         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1176             resourceLoadStatistics->setMinimumTimeBetweenDataRecordsRemoval(seconds, WTFMove(completionHandler));
1177         else
1178             completionHandler();
1179     } else {
1180         ASSERT_NOT_REACHED();
1181         completionHandler();
1182     }
1183 }
1184
1185 void NetworkProcess::setPruneEntriesDownTo(PAL::SessionID sessionID, uint64_t pruneTargetCount, CompletionHandler<void()>&& completionHandler)
1186 {
1187     if (auto* networkSession = this->networkSession(sessionID)) {
1188         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1189             resourceLoadStatistics->setPruneEntriesDownTo(pruneTargetCount, WTFMove(completionHandler));
1190         else
1191             completionHandler();
1192     } else {
1193         ASSERT_NOT_REACHED();
1194         completionHandler();
1195     }
1196 }
1197
1198 void NetworkProcess::setTimeToLiveUserInteraction(PAL::SessionID sessionID, Seconds seconds, CompletionHandler<void()>&& completionHandler)
1199 {
1200     if (auto* networkSession = this->networkSession(sessionID)) {
1201         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1202             resourceLoadStatistics->setTimeToLiveUserInteraction(seconds, WTFMove(completionHandler));
1203         else
1204             completionHandler();
1205     } else {
1206         ASSERT_NOT_REACHED();
1207         completionHandler();
1208     }
1209 }
1210
1211 void NetworkProcess::setShouldClassifyResourcesBeforeDataRecordsRemoval(PAL::SessionID sessionID, bool value, CompletionHandler<void()>&& completionHandler)
1212 {
1213     if (auto* networkSession = this->networkSession(sessionID)) {
1214         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1215             resourceLoadStatistics->setShouldClassifyResourcesBeforeDataRecordsRemoval(value, WTFMove(completionHandler));
1216         else
1217             completionHandler();
1218     } else {
1219         ASSERT_NOT_REACHED();
1220         completionHandler();
1221     }
1222 }
1223
1224 void NetworkProcess::setResourceLoadStatisticsEnabled(PAL::SessionID sessionID, bool enabled)
1225 {
1226     if (auto* networkSession = this->networkSession(sessionID))
1227         networkSession->setResourceLoadStatisticsEnabled(enabled);
1228 }
1229
1230 void NetworkProcess::setResourceLoadStatisticsLogTestingEvent(bool enabled)
1231 {
1232     forEachNetworkSession([enabled](auto& networkSession) {
1233         networkSession.setResourceLoadStatisticsLogTestingEvent(enabled);
1234     });
1235 }
1236
1237 void NetworkProcess::setResourceLoadStatisticsDebugMode(PAL::SessionID sessionID, bool debugMode, CompletionHandler<void()>&& completionHandler)
1238 {
1239     if (auto* networkSession = this->networkSession(sessionID)) {
1240         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1241             resourceLoadStatistics->setResourceLoadStatisticsDebugMode(debugMode, WTFMove(completionHandler));
1242         else
1243             completionHandler();
1244     } else {
1245         ASSERT_NOT_REACHED();
1246         completionHandler();
1247     }
1248 }
1249
1250 void NetworkProcess::isResourceLoadStatisticsEphemeral(PAL::SessionID sessionID, CompletionHandler<void(bool)>&& completionHandler) const
1251 {
1252     if (auto* networkSession = this->networkSession(sessionID)) {
1253         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics()) {
1254             completionHandler(resourceLoadStatistics->isEphemeral());
1255             return;
1256         }
1257     } else
1258         ASSERT_NOT_REACHED();
1259     completionHandler(false);
1260 }
1261
1262 void NetworkProcess::resetCacheMaxAgeCapForPrevalentResources(PAL::SessionID sessionID, CompletionHandler<void()>&& completionHandler)
1263 {
1264     if (auto* networkStorageSession = storageSession(sessionID))
1265         networkStorageSession->resetCacheMaxAgeCapForPrevalentResources();
1266     else
1267         ASSERT_NOT_REACHED();
1268     completionHandler();
1269 }
1270
1271 void NetworkProcess::didCommitCrossSiteLoadWithDataTransfer(PAL::SessionID sessionID, const RegistrableDomain& fromDomain, const RegistrableDomain& toDomain, OptionSet<WebCore::CrossSiteNavigationDataTransfer::Flag> navigationDataTransfer, WebPageProxyIdentifier webPageProxyID, WebCore::PageIdentifier webPageID)
1272 {
1273     ASSERT(!navigationDataTransfer.isEmpty());
1274
1275     if (auto* networkStorageSession = storageSession(sessionID)) {
1276         if (!networkStorageSession->shouldBlockThirdPartyCookies(fromDomain))
1277             return;
1278
1279         if (navigationDataTransfer.contains(CrossSiteNavigationDataTransfer::Flag::DestinationLinkDecoration))
1280             networkStorageSession->didCommitCrossSiteLoadWithDataTransferFromPrevalentResource(toDomain, webPageID);
1281
1282         if (navigationDataTransfer.contains(CrossSiteNavigationDataTransfer::Flag::ReferrerLinkDecoration))
1283             parentProcessConnection()->send(Messages::NetworkProcessProxy::DidCommitCrossSiteLoadWithDataTransferFromPrevalentResource(webPageProxyID), 0);
1284     } else
1285         ASSERT_NOT_REACHED();
1286
1287     if (navigationDataTransfer.contains(CrossSiteNavigationDataTransfer::Flag::DestinationLinkDecoration)) {
1288         if (auto* networkSession = this->networkSession(sessionID)) {
1289             if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1290                 resourceLoadStatistics->logCrossSiteLoadWithLinkDecoration(fromDomain, toDomain, [] { });
1291         } else
1292             ASSERT_NOT_REACHED();
1293     }
1294 }
1295
1296 void NetworkProcess::setCrossSiteLoadWithLinkDecorationForTesting(PAL::SessionID sessionID, const RegistrableDomain& fromDomain, const RegistrableDomain& toDomain, CompletionHandler<void()>&& completionHandler)
1297 {
1298     if (auto* networkSession = this->networkSession(sessionID)) {
1299         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1300             resourceLoadStatistics->logCrossSiteLoadWithLinkDecoration(fromDomain, toDomain, WTFMove(completionHandler));
1301         else
1302             completionHandler();
1303     } else {
1304         ASSERT_NOT_REACHED();
1305         completionHandler();
1306     }
1307 }
1308
1309 void NetworkProcess::resetCrossSiteLoadsWithLinkDecorationForTesting(PAL::SessionID sessionID, CompletionHandler<void()>&& completionHandler)
1310 {
1311     if (auto* networkStorageSession = storageSession(sessionID))
1312         networkStorageSession->resetCrossSiteLoadsWithLinkDecorationForTesting();
1313     else
1314         ASSERT_NOT_REACHED();
1315     completionHandler();
1316 }
1317
1318 void NetworkProcess::hasIsolatedSession(PAL::SessionID sessionID, const WebCore::RegistrableDomain& domain, CompletionHandler<void(bool)>&& completionHandler) const
1319 {
1320     bool result = false;
1321     if (auto* networkSession = this->networkSession(sessionID))
1322         result = networkSession->hasIsolatedSession(domain);
1323     completionHandler(result);
1324 }
1325
1326 void NetworkProcess::setAppBoundDomainsForResourceLoadStatistics(PAL::SessionID sessionID, HashSet<WebCore::RegistrableDomain>&& appBoundDomains, CompletionHandler<void()>&& completionHandler)
1327 {
1328     if (auto* networkSession = this->networkSession(sessionID)) {
1329         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics()) {
1330             resourceLoadStatistics->setAppBoundDomains(WTFMove(appBoundDomains), WTFMove(completionHandler));
1331             return;
1332         }
1333     }
1334     ASSERT_NOT_REACHED();
1335     completionHandler();
1336 }
1337
1338 void NetworkProcess::setShouldDowngradeReferrerForTesting(bool enabled, CompletionHandler<void()>&& completionHandler)
1339 {
1340     forEachNetworkSession([enabled](auto& networkSession) {
1341         networkSession.setShouldDowngradeReferrerForTesting(enabled);
1342     });
1343     completionHandler();
1344 }
1345
1346 void NetworkProcess::setThirdPartyCookieBlockingMode(PAL::SessionID sessionID, WebCore::ThirdPartyCookieBlockingMode blockingMode, CompletionHandler<void()>&& completionHandler)
1347 {
1348     if (auto* networkSession = this->networkSession(sessionID))
1349         networkSession->setThirdPartyCookieBlockingMode(blockingMode);
1350     else
1351         ASSERT_NOT_REACHED();
1352     completionHandler();
1353 }
1354
1355 void NetworkProcess::setShouldEnbleSameSiteStrictEnforcementForTesting(PAL::SessionID sessionID, WebCore::SameSiteStrictEnforcementEnabled enabled, CompletionHandler<void()>&& completionHandler)
1356 {
1357     if (auto* networkSession = this->networkSession(sessionID))
1358         networkSession->setShouldEnbleSameSiteStrictEnforcement(enabled);
1359     else
1360         ASSERT_NOT_REACHED();
1361     completionHandler();
1362 }
1363
1364 void NetworkProcess::setFirstPartyWebsiteDataRemovalModeForTesting(PAL::SessionID sessionID, WebCore::FirstPartyWebsiteDataRemovalMode mode, CompletionHandler<void()>&& completionHandler)
1365 {
1366     if (auto* networkSession = this->networkSession(sessionID)) {
1367         if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1368             resourceLoadStatistics->setFirstPartyWebsiteDataRemovalMode(mode, WTFMove(completionHandler));
1369         else
1370             completionHandler();
1371     } else {
1372         ASSERT_NOT_REACHED();
1373         completionHandler();
1374     }
1375 }
1376
1377 void NetworkProcess::setToSameSiteStrictCookiesForTesting(PAL::SessionID sessionID, const WebCore::RegistrableDomain& domain, CompletionHandler<void()>&& completionHandler)
1378 {
1379     if (auto* networkStorageSession = storageSession(sessionID))
1380         networkStorageSession->setAllCookiesToSameSiteStrict(domain, WTFMove(completionHandler));
1381     else {
1382         ASSERT_NOT_REACHED();
1383         completionHandler();
1384     }
1385 }
1386 #endif // ENABLE(RESOURCE_LOAD_STATISTICS)
1387
1388 void NetworkProcess::setAdClickAttributionDebugMode(bool debugMode)
1389 {
1390     if (RuntimeEnabledFeatures::sharedFeatures().adClickAttributionDebugModeEnabled() == debugMode)
1391         return;
1392
1393     RuntimeEnabledFeatures::sharedFeatures().setAdClickAttributionDebugModeEnabled(debugMode);
1394
1395     String message = debugMode ? "[Ad Click Attribution] Turned Debug Mode on."_s : "[Ad Click Attribution] Turned Debug Mode off."_s;
1396     for (auto& networkConnectionToWebProcess : m_webProcessConnections.values()) {
1397         if (networkConnectionToWebProcess->sessionID().isEphemeral())
1398             continue;
1399         networkConnectionToWebProcess->broadcastConsoleMessage(MessageSource::AdClickAttribution, MessageLevel::Info, message);
1400     }
1401 }
1402
1403 void NetworkProcess::preconnectTo(PAL::SessionID sessionID, WebPageProxyIdentifier webPageProxyID, WebCore::PageIdentifier webPageID, const URL& url, const String& userAgent, WebCore::StoredCredentialsPolicy storedCredentialsPolicy, Optional<NavigatingToAppBoundDomain> isNavigatingToAppBoundDomain)
1404 {
1405 #if ENABLE(SERVER_PRECONNECT)
1406 #if ENABLE(LEGACY_CUSTOM_PROTOCOL_MANAGER)
1407     if (supplement<LegacyCustomProtocolManager>()->supportsScheme(url.protocol().toString()))
1408         return;
1409 #endif
1410
1411     NetworkLoadParameters parameters;
1412     parameters.request = ResourceRequest { url };
1413     parameters.webPageProxyID = webPageProxyID;
1414     parameters.webPageID = webPageID;
1415     parameters.isNavigatingToAppBoundDomain = isNavigatingToAppBoundDomain;
1416     if (!userAgent.isEmpty()) {
1417         // FIXME: we add user-agent to the preconnect request because otherwise the preconnect
1418         // gets thrown away by CFNetwork when using an HTTPS proxy (<rdar://problem/59434166>).
1419         parameters.request.setHTTPUserAgent(userAgent);
1420     }
1421     parameters.storedCredentialsPolicy = storedCredentialsPolicy;
1422     parameters.shouldPreconnectOnly = PreconnectOnly::Yes;
1423
1424     new PreconnectTask(*this, sessionID, WTFMove(parameters), [](const WebCore::ResourceError&) { });
1425 #else
1426     UNUSED_PARAM(url);
1427     UNUSED_PARAM(userAgent);
1428     UNUSED_PARAM(storedCredentialsPolicy);
1429 #endif
1430 }
1431
1432 bool NetworkProcess::sessionIsControlledByAutomation(PAL::SessionID sessionID) const
1433 {
1434     return m_sessionsControlledByAutomation.contains(sessionID);
1435 }
1436
1437 void NetworkProcess::setSessionIsControlledByAutomation(PAL::SessionID sessionID, bool controlled)
1438 {
1439     if (controlled)
1440         m_sessionsControlledByAutomation.add(sessionID);
1441     else
1442         m_sessionsControlledByAutomation.remove(sessionID);
1443 }
1444
1445 static void fetchDiskCacheEntries(NetworkCache::Cache* cache, PAL::SessionID sessionID, OptionSet<WebsiteDataFetchOption> fetchOptions, CompletionHandler<void(Vector<WebsiteData::Entry>)>&& completionHandler)
1446 {
1447     if (!cache) {
1448         RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler)] () mutable {
1449             completionHandler({ });
1450         });
1451         return;
1452     }
1453     
1454     HashMap<SecurityOriginData, uint64_t> originsAndSizes;
1455     cache->traverse([fetchOptions, completionHandler = WTFMove(completionHandler), originsAndSizes = WTFMove(originsAndSizes)](auto* traversalEntry) mutable {
1456         if (!traversalEntry) {
1457             Vector<WebsiteData::Entry> entries;
1458
1459             for (auto& originAndSize : originsAndSizes)
1460                 entries.append(WebsiteData::Entry { originAndSize.key, WebsiteDataType::DiskCache, originAndSize.value });
1461
1462             RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler), entries = WTFMove(entries)] () mutable {
1463                 completionHandler(entries);
1464             });
1465
1466             return;
1467         }
1468
1469         auto url = traversalEntry->entry.response().url();
1470         auto result = originsAndSizes.add({url.protocol().toString(), url.host().toString(), url.port()}, 0);
1471
1472         if (fetchOptions.contains(WebsiteDataFetchOption::ComputeSizes))
1473             result.iterator->value += traversalEntry->entry.sourceStorageRecord().header.size() + traversalEntry->recordInfo.bodySize;
1474     });
1475 }
1476
1477 void NetworkProcess::fetchWebsiteData(PAL::SessionID sessionID, OptionSet<WebsiteDataType> websiteDataTypes, OptionSet<WebsiteDataFetchOption> fetchOptions, CallbackID callbackID)
1478 {
1479     struct CallbackAggregator final : public ThreadSafeRefCounted<CallbackAggregator> {
1480         explicit CallbackAggregator(Function<void (WebsiteData)>&& completionHandler)
1481             : m_completionHandler(WTFMove(completionHandler))
1482         {
1483         }
1484
1485         ~CallbackAggregator()
1486         {
1487             RunLoop::main().dispatch([completionHandler = WTFMove(m_completionHandler), websiteData = WTFMove(m_websiteData)] () mutable {
1488                 completionHandler(websiteData);
1489             });
1490         }
1491
1492         CompletionHandler<void(WebsiteData)> m_completionHandler;
1493         WebsiteData m_websiteData;
1494     };
1495
1496     auto callbackAggregator = adoptRef(*new CallbackAggregator([this, callbackID] (WebsiteData websiteData) {
1497         parentProcessConnection()->send(Messages::NetworkProcessProxy::DidFetchWebsiteData(callbackID, websiteData), 0);
1498     }));
1499
1500     if (websiteDataTypes.contains(WebsiteDataType::Cookies)) {
1501         if (auto* networkStorageSession = storageSession(sessionID))
1502             networkStorageSession->getHostnamesWithCookies(callbackAggregator->m_websiteData.hostNamesWithCookies);
1503     }
1504
1505     if (websiteDataTypes.contains(WebsiteDataType::Credentials)) {
1506         if (storageSession(sessionID)) {
1507             auto securityOrigins = storageSession(sessionID)->credentialStorage().originsWithCredentials();
1508             for (auto& securityOrigin : securityOrigins)
1509                 callbackAggregator->m_websiteData.entries.append({ securityOrigin, WebsiteDataType::Credentials, 0 });
1510         }
1511         auto securityOrigins = WebCore::CredentialStorage::originsWithSessionCredentials();
1512         for (auto& securityOrigin : securityOrigins)
1513             callbackAggregator->m_websiteData.entries.append({ securityOrigin, WebsiteDataType::Credentials, 0 });
1514     }
1515
1516     if (websiteDataTypes.contains(WebsiteDataType::DOMCache)) {
1517         CacheStorage::Engine::fetchEntries(*this, sessionID, fetchOptions.contains(WebsiteDataFetchOption::ComputeSizes), [callbackAggregator](auto entries) mutable {
1518             callbackAggregator->m_websiteData.entries.appendVector(entries);
1519         });
1520     }
1521
1522     if (websiteDataTypes.contains(WebsiteDataType::SessionStorage) && m_storageManagerSet->contains(sessionID)) {
1523         m_storageManagerSet->getSessionStorageOrigins(sessionID, [callbackAggregator](auto&& origins) {
1524             while (!origins.isEmpty())
1525                 callbackAggregator->m_websiteData.entries.append(WebsiteData::Entry { origins.takeAny(), WebsiteDataType::SessionStorage, 0 });
1526         });
1527     }
1528
1529     if (websiteDataTypes.contains(WebsiteDataType::LocalStorage) && m_storageManagerSet->contains(sessionID)) {
1530         m_storageManagerSet->getLocalStorageOrigins(sessionID, [callbackAggregator](auto&& origins) {
1531             while (!origins.isEmpty())
1532                 callbackAggregator->m_websiteData.entries.append(WebsiteData::Entry { origins.takeAny(), WebsiteDataType::LocalStorage, 0 });
1533         });
1534     }
1535
1536 #if PLATFORM(COCOA) || USE(SOUP)
1537     if (websiteDataTypes.contains(WebsiteDataType::HSTSCache))
1538         callbackAggregator->m_websiteData.hostNamesWithHSTSCache = hostNamesWithHSTSCache(sessionID);
1539 #endif
1540
1541 #if ENABLE(INDEXED_DATABASE)
1542     auto path = m_idbDatabasePaths.get(sessionID);
1543     if (!path.isEmpty() && websiteDataTypes.contains(WebsiteDataType::IndexedDBDatabases)) {
1544         // FIXME: Pick the right database store based on the session ID.
1545         postStorageTask(CrossThreadTask([this, callbackAggregator, path = crossThreadCopy(path)]() mutable {
1546             RunLoop::main().dispatch([callbackAggregator = WTFMove(callbackAggregator), securityOrigins = indexedDatabaseOrigins(path)] {
1547                 for (const auto& securityOrigin : securityOrigins)
1548                     callbackAggregator->m_websiteData.entries.append({ securityOrigin, WebsiteDataType::IndexedDBDatabases, 0 });
1549             });
1550         }));
1551     }
1552 #endif
1553
1554 #if ENABLE(SERVICE_WORKER)
1555     path = m_serviceWorkerInfo.get(sessionID).databasePath;
1556     if (!path.isEmpty() && websiteDataTypes.contains(WebsiteDataType::ServiceWorkerRegistrations)) {
1557         swServerForSession(sessionID).getOriginsWithRegistrations([callbackAggregator](const HashSet<SecurityOriginData>& securityOrigins) mutable {
1558             for (auto& origin : securityOrigins)
1559                 callbackAggregator->m_websiteData.entries.append({ origin, WebsiteDataType::ServiceWorkerRegistrations, 0 });
1560         });
1561     }
1562 #endif
1563     if (websiteDataTypes.contains(WebsiteDataType::DiskCache)) {
1564         forEachNetworkSession([sessionID, fetchOptions, &callbackAggregator](auto& session) {
1565             fetchDiskCacheEntries(session.cache(), sessionID, fetchOptions, [callbackAggregator](auto entries) mutable {
1566                 callbackAggregator->m_websiteData.entries.appendVector(entries);
1567             });
1568         });
1569     }
1570
1571 #if HAVE(CFNETWORK_ALTERNATIVE_SERVICE)
1572     if (websiteDataTypes.contains(WebsiteDataType::AlternativeServices)) {
1573         if (auto* session = networkSession(sessionID)) {
1574             for (auto& origin : session->hostNamesWithAlternativeServices())
1575                 callbackAggregator->m_websiteData.entries.append({ origin, WebsiteDataType::AlternativeServices, 0 });
1576         }
1577     }
1578 #endif
1579
1580 #if ENABLE(RESOURCE_LOAD_STATISTICS)
1581     if (websiteDataTypes.contains(WebsiteDataType::ResourceLoadStatistics)) {
1582         if (auto* session = networkSession(sessionID)) {
1583             if (auto* resourceLoadStatistics = session->resourceLoadStatistics()) {
1584                 resourceLoadStatistics->registrableDomains([callbackAggregator](auto&& domains) mutable {
1585                     while (!domains.isEmpty())
1586                         callbackAggregator->m_websiteData.registrableDomainsWithResourceLoadStatistics.add(domains.takeLast());
1587                 });
1588             }
1589         }
1590     }
1591 #endif
1592 }
1593
1594 void NetworkProcess::deleteWebsiteData(PAL::SessionID sessionID, OptionSet<WebsiteDataType> websiteDataTypes, WallTime modifiedSince, CallbackID callbackID)
1595 {
1596 #if PLATFORM(COCOA) || USE(SOUP)
1597     if (websiteDataTypes.contains(WebsiteDataType::HSTSCache))
1598         clearHSTSCache(sessionID, modifiedSince);
1599 #endif
1600
1601     if (websiteDataTypes.contains(WebsiteDataType::Cookies)) {
1602         if (auto* networkStorageSession = storageSession(sessionID))
1603             networkStorageSession->deleteAllCookiesModifiedSince(modifiedSince);
1604     }
1605
1606     if (websiteDataTypes.contains(WebsiteDataType::Credentials)) {
1607         if (auto* session = storageSession(sessionID))
1608             session->credentialStorage().clearCredentials();
1609         WebCore::CredentialStorage::clearSessionCredentials();
1610     }
1611
1612     auto clearTasksHandler = WTF::CallbackAggregator::create([this, callbackID] {
1613         parentProcessConnection()->send(Messages::NetworkProcessProxy::DidDeleteWebsiteData(callbackID), 0);
1614     });
1615
1616     if (websiteDataTypes.contains(WebsiteDataType::DOMCache))
1617         CacheStorage::Engine::clearAllCaches(*this, sessionID, [clearTasksHandler] { });
1618
1619     if (websiteDataTypes.contains(WebsiteDataType::SessionStorage) && m_storageManagerSet->contains(sessionID))
1620         m_storageManagerSet->deleteSessionStorage(sessionID, [clearTasksHandler] { });
1621
1622     if (websiteDataTypes.contains(WebsiteDataType::LocalStorage) && m_storageManagerSet->contains(sessionID))
1623         m_storageManagerSet->deleteLocalStorageModifiedSince(sessionID, modifiedSince, [clearTasksHandler] { });
1624
1625 #if ENABLE(INDEXED_DATABASE)
1626     if (websiteDataTypes.contains(WebsiteDataType::IndexedDBDatabases) && !sessionID.isEphemeral())
1627         webIDBServer(sessionID).closeAndDeleteDatabasesModifiedSince(modifiedSince, [clearTasksHandler] { });
1628 #endif
1629
1630 #if ENABLE(SERVICE_WORKER)
1631     if (websiteDataTypes.contains(WebsiteDataType::ServiceWorkerRegistrations) && !sessionID.isEphemeral())
1632         swServerForSession(sessionID).clearAll([clearTasksHandler] { });
1633 #endif
1634
1635 #if ENABLE(RESOURCE_LOAD_STATISTICS)
1636     if (websiteDataTypes.contains(WebsiteDataType::ResourceLoadStatistics)) {
1637         if (auto* networkSession = this->networkSession(sessionID)) {
1638             if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics()) {
1639                 auto deletedTypesRaw = websiteDataTypes.toRaw();
1640                 auto monitoredTypesRaw = WebResourceLoadStatisticsStore::monitoredDataTypes().toRaw();
1641                 
1642                 // If we are deleting all of the data types that the resource load statistics store monitors
1643                 // we do not need to re-grandfather old data.
1644                 auto shouldGrandfather = ((monitoredTypesRaw & deletedTypesRaw) == monitoredTypesRaw) ? ShouldGrandfatherStatistics::No : ShouldGrandfatherStatistics::Yes;
1645                 
1646                 resourceLoadStatistics->scheduleClearInMemoryAndPersistent(modifiedSince, shouldGrandfather, [clearTasksHandler] { });
1647             }
1648         }
1649     }
1650 #endif
1651
1652     if (websiteDataTypes.contains(WebsiteDataType::DiskCache) && !sessionID.isEphemeral())
1653         clearDiskCache(modifiedSince, [clearTasksHandler = WTFMove(clearTasksHandler)] { });
1654
1655     if (websiteDataTypes.contains(WebsiteDataType::AdClickAttributions)) {
1656         if (auto* networkSession = this->networkSession(sessionID))
1657             networkSession->clearAdClickAttribution();
1658     }
1659
1660 #if HAVE(CFNETWORK_ALTERNATIVE_SERVICE)
1661     if (websiteDataTypes.contains(WebsiteDataType::AlternativeServices)) {
1662         if (auto* networkSession = this->networkSession(sessionID))
1663             networkSession->clearAlternativeServices(modifiedSince);
1664     }
1665 #endif
1666 }
1667
1668 static void clearDiskCacheEntries(NetworkCache::Cache* cache, const Vector<SecurityOriginData>& origins, CompletionHandler<void()>&& completionHandler)
1669 {
1670     if (!cache) {
1671         RunLoop::main().dispatch(WTFMove(completionHandler));
1672         return;
1673     }
1674
1675     HashSet<RefPtr<SecurityOrigin>> originsToDelete;
1676     for (auto& origin : origins)
1677         originsToDelete.add(origin.securityOrigin());
1678
1679     Vector<NetworkCache::Key> cacheKeysToDelete;
1680     cache->traverse([cache, completionHandler = WTFMove(completionHandler), originsToDelete = WTFMove(originsToDelete), cacheKeysToDelete = WTFMove(cacheKeysToDelete)](auto* traversalEntry) mutable {
1681         if (traversalEntry) {
1682             if (originsToDelete.contains(SecurityOrigin::create(traversalEntry->entry.response().url())))
1683                 cacheKeysToDelete.append(traversalEntry->entry.key());
1684             return;
1685         }
1686
1687         cache->remove(cacheKeysToDelete, WTFMove(completionHandler));
1688         return;
1689     });
1690 }
1691
1692 void NetworkProcess::deleteWebsiteDataForOrigins(PAL::SessionID sessionID, OptionSet<WebsiteDataType> websiteDataTypes, const Vector<SecurityOriginData>& originDatas, const Vector<String>& cookieHostNames, const Vector<String>& HSTSCacheHostNames, const Vector<RegistrableDomain>& registrableDomains, CallbackID callbackID)
1693 {
1694     if (websiteDataTypes.contains(WebsiteDataType::Cookies)) {
1695         if (auto* networkStorageSession = storageSession(sessionID))
1696             networkStorageSession->deleteCookiesForHostnames(cookieHostNames);
1697     }
1698
1699 #if PLATFORM(COCOA) || USE(SOUP)
1700     if (websiteDataTypes.contains(WebsiteDataType::HSTSCache))
1701         deleteHSTSCacheForHostNames(sessionID, HSTSCacheHostNames);
1702 #endif
1703
1704 #if HAVE(CFNETWORK_ALTERNATIVE_SERVICE)
1705     if (websiteDataTypes.contains(WebsiteDataType::AlternativeServices)) {
1706         if (auto* networkSession = this->networkSession(sessionID)) {
1707             Vector<String> hosts;
1708             hosts.reserveInitialCapacity(originDatas.size());
1709             for (auto& origin : originDatas)
1710                 hosts.uncheckedAppend(origin.host);
1711             networkSession->deleteAlternativeServicesForHostNames(hosts);
1712         }
1713     }
1714 #endif
1715
1716     if (websiteDataTypes.contains(WebsiteDataType::AdClickAttributions)) {
1717         if (auto* networkSession = this->networkSession(sessionID)) {
1718             for (auto& originData : originDatas)
1719                 networkSession->clearAdClickAttributionForRegistrableDomain(RegistrableDomain::uncheckedCreateFromHost(originData.host));
1720         }
1721     }
1722     
1723     auto clearTasksHandler = WTF::CallbackAggregator::create([this, callbackID] {
1724         parentProcessConnection()->send(Messages::NetworkProcessProxy::DidDeleteWebsiteDataForOrigins(callbackID), 0);
1725     });
1726
1727     if (websiteDataTypes.contains(WebsiteDataType::DOMCache)) {
1728         for (auto& originData : originDatas)
1729             CacheStorage::Engine::clearCachesForOrigin(*this, sessionID, SecurityOriginData { originData }, [clearTasksHandler] { });
1730     }
1731
1732     if (websiteDataTypes.contains(WebsiteDataType::SessionStorage) && m_storageManagerSet->contains(sessionID))
1733         m_storageManagerSet->deleteSessionStorageForOrigins(sessionID, originDatas, [clearTasksHandler] { });
1734
1735     if (websiteDataTypes.contains(WebsiteDataType::LocalStorage) && m_storageManagerSet->contains(sessionID))
1736         m_storageManagerSet->deleteLocalStorageForOrigins(sessionID, originDatas, [clearTasksHandler] { });
1737
1738 #if ENABLE(INDEXED_DATABASE)
1739     if (websiteDataTypes.contains(WebsiteDataType::IndexedDBDatabases) && !sessionID.isEphemeral())
1740         webIDBServer(sessionID).closeAndDeleteDatabasesForOrigins(originDatas, [clearTasksHandler] { });
1741 #endif
1742
1743 #if ENABLE(SERVICE_WORKER)
1744     if (websiteDataTypes.contains(WebsiteDataType::ServiceWorkerRegistrations) && !sessionID.isEphemeral()) {
1745         auto& server = swServerForSession(sessionID);
1746         for (auto& originData : originDatas)
1747             server.clear(originData, [clearTasksHandler] { });
1748     }
1749 #endif
1750
1751     if (websiteDataTypes.contains(WebsiteDataType::DiskCache) && !sessionID.isEphemeral()) {
1752         forEachNetworkSession([originDatas, &clearTasksHandler](auto& session) {
1753             clearDiskCacheEntries(session.cache(), originDatas, [clearTasksHandler] { });
1754         });
1755     }
1756
1757     if (websiteDataTypes.contains(WebsiteDataType::Credentials)) {
1758         if (auto* session = storageSession(sessionID)) {
1759             for (auto& originData : originDatas)
1760                 session->credentialStorage().removeCredentialsWithOrigin(originData);
1761         }
1762         WebCore::CredentialStorage::removeSessionCredentialsWithOrigins(originDatas);
1763     }
1764
1765 #if ENABLE(RESOURCE_LOAD_STATISTICS)
1766     if (websiteDataTypes.contains(WebsiteDataType::ResourceLoadStatistics)) {
1767         if (auto* networkSession = this->networkSession(sessionID)) {
1768             for (auto& domain : registrableDomains) {
1769                 if (auto* resourceLoadStatistics = networkSession->resourceLoadStatistics())
1770                     resourceLoadStatistics->removeDataForDomain(domain, [clearTasksHandler] { });
1771             }
1772         }
1773     }
1774 #endif
1775 }
1776
1777 #if ENABLE(RESOURCE_LOAD_STATISTICS)
1778 static Vector<String> filterForRegistrableDomains(const Vector<RegistrableDomain>& registrableDomains, const HashSet<String>& foundValues)
1779 {
1780     Vector<String> result;
1781     for (const auto& value : foundValues) {
1782         if (registrableDomains.contains(RegistrableDomain::uncheckedCreateFromHost(value)))
1783             result.append(value);
1784     }
1785     
1786     return result;
1787 }
1788
1789 static Vector<WebsiteData::Entry> filterForRegistrableDomains(const Vector<RegistrableDomain>& registrableDomains, const Vector<WebsiteData::Entry>& foundValues)
1790 {
1791     Vector<WebsiteData::Entry> result;
1792     for (const auto& value : foundValues) {
1793         if (registrableDomains.contains(RegistrableDomain::uncheckedCreateFromHost(value.origin.host)))
1794             result.append(value);
1795     }
1796     
1797     return result;
1798 }
1799
1800 static Vector<WebCore::SecurityOriginData> filterForRegistrableDomains(const HashSet<WebCore::SecurityOriginData>& origins, const Vector<RegistrableDomain>& domainsToDelete, HashSet<RegistrableDomain>& domainsDeleted)
1801 {
1802     Vector<SecurityOriginData> originsDeleted;
1803     for (const auto& origin : origins) {
1804         auto domain = RegistrableDomain::uncheckedCreateFromHost(origin.host);
1805         if (!domainsToDelete.contains(domain))
1806             continue;
1807         originsDeleted.append(origin);
1808         domainsDeleted.add(domain);
1809     }
1810
1811     return originsDeleted;
1812 }
1813
1814 void NetworkProcess::deleteAndRestrictWebsiteDataForRegistrableDomains(PAL::SessionID sessionID, OptionSet<WebsiteDataType> websiteDataTypes, RegistrableDomainsToDeleteOrRestrictWebsiteDataFor&& domains, bool shouldNotifyPage, CompletionHandler<void(const HashSet<RegistrableDomain>&)>&& completionHandler)
1815 {
1816     OptionSet<WebsiteDataFetchOption> fetchOptions = WebsiteDataFetchOption::DoNotCreateProcesses;
1817
1818     struct CallbackAggregator final : public ThreadSafeRefCounted<CallbackAggregator> {
1819         explicit CallbackAggregator(CompletionHandler<void(const HashSet<RegistrableDomain>&)>&& completionHandler)
1820             : m_completionHandler(WTFMove(completionHandler))
1821         {
1822         }
1823         
1824         ~CallbackAggregator()
1825         {
1826             RunLoop::main().dispatch([completionHandler = WTFMove(m_completionHandler), domains = WTFMove(m_domains)] () mutable {
1827                 completionHandler(domains);
1828             });
1829         }
1830         
1831         CompletionHandler<void(const HashSet<RegistrableDomain>&)> m_completionHandler;
1832         HashSet<RegistrableDomain> m_domains;
1833     };
1834     
1835     auto callbackAggregator = adoptRef(*new CallbackAggregator([this, completionHandler = WTFMove(completionHandler), shouldNotifyPage] (const HashSet<RegistrableDomain>& domainsWithData) mutable {
1836         if (shouldNotifyPage)
1837             parentProcessConnection()->send(Messages::NetworkProcessProxy::NotifyWebsiteDataDeletionForRegistrableDomainsFinished(), 0);
1838         
1839         RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler), domainsWithData = crossThreadCopy(domainsWithData)] () mutable {
1840             completionHandler(domainsWithData);
1841         });
1842     }));
1843
1844     HashSet<String> hostNamesWithCookies;
1845     HashSet<String> hostNamesWithHSTSCache;
1846     Vector<String> hostnamesWithCookiesToDelete;
1847     auto domainsToDeleteAllNonCookieWebsiteDataFor = domains.domainsToDeleteAllNonCookieWebsiteDataFor;
1848     if (websiteDataTypes.contains(WebsiteDataType::Cookies)) {
1849         if (auto* networkStorageSession = storageSession(sessionID)) {
1850             networkStorageSession->getHostnamesWithCookies(hostNamesWithCookies);
1851
1852             hostnamesWithCookiesToDelete = filterForRegistrableDomains(domains.domainsToDeleteAllCookiesFor, hostNamesWithCookies);
1853             networkStorageSession->deleteCookiesForHostnames(hostnamesWithCookiesToDelete, WebCore::IncludeHttpOnlyCookies::Yes);
1854
1855             for (const auto& host : hostnamesWithCookiesToDelete)
1856                 callbackAggregator->m_domains.add(RegistrableDomain::uncheckedCreateFromHost(host));
1857
1858             hostnamesWithCookiesToDelete = filterForRegistrableDomains(domains.domainsToDeleteAllButHttpOnlyCookiesFor, hostNamesWithCookies);
1859             networkStorageSession->deleteCookiesForHostnames(hostnamesWithCookiesToDelete, WebCore::IncludeHttpOnlyCookies::No);
1860
1861             for (const auto& host : hostnamesWithCookiesToDelete)
1862                 callbackAggregator->m_domains.add(RegistrableDomain::uncheckedCreateFromHost(host));
1863         }
1864     }
1865
1866     Vector<String> hostnamesWithHSTSToDelete;
1867 #if PLATFORM(COCOA) || USE(SOUP)
1868     if (websiteDataTypes.contains(WebsiteDataType::HSTSCache)) {
1869         hostNamesWithHSTSCache = this->hostNamesWithHSTSCache(sessionID);
1870         hostnamesWithHSTSToDelete = filterForRegistrableDomains(domainsToDeleteAllNonCookieWebsiteDataFor, hostNamesWithHSTSCache);
1871
1872         for (const auto& host : hostnamesWithHSTSToDelete)
1873             callbackAggregator->m_domains.add(RegistrableDomain::uncheckedCreateFromHost(host));
1874
1875         deleteHSTSCacheForHostNames(sessionID, hostnamesWithHSTSToDelete);
1876     }
1877 #endif
1878
1879 #if HAVE(CFNETWORK_ALTERNATIVE_SERVICE)
1880     if (websiteDataTypes.contains(WebsiteDataType::AlternativeServices)) {
1881         if (auto* networkSession = this->networkSession(sessionID)) {
1882             Vector<String> registrableDomainsToDelete;
1883             registrableDomainsToDelete.reserveInitialCapacity(domainsToDeleteAllNonCookieWebsiteDataFor.size());
1884             for (auto& domain : domainsToDeleteAllNonCookieWebsiteDataFor)
1885                 registrableDomainsToDelete.uncheckedAppend(domain.string());
1886             networkSession->deleteAlternativeServicesForHostNames(registrableDomainsToDelete);
1887         }
1888     }
1889 #endif
1890
1891     if (websiteDataTypes.contains(WebsiteDataType::Credentials)) {
1892         if (auto* session = storageSession(sessionID)) {
1893             auto origins = session->credentialStorage().originsWithCredentials();
1894             auto originsToDelete = filterForRegistrableDomains(origins, domainsToDeleteAllNonCookieWebsiteDataFor, callbackAggregator->m_domains);
1895             for (auto& origin : originsToDelete)
1896                 session->credentialStorage().removeCredentialsWithOrigin(origin);
1897         }
1898
1899         auto origins = WebCore::CredentialStorage::originsWithSessionCredentials();
1900         auto originsToDelete = filterForRegistrableDomains(origins, domainsToDeleteAllNonCookieWebsiteDataFor, callbackAggregator->m_domains);
1901         WebCore::CredentialStorage::removeSessionCredentialsWithOrigins(originsToDelete);
1902     }
1903     
1904     if (websiteDataTypes.contains(WebsiteDataType::DOMCache)) {
1905         CacheStorage::Engine::fetchEntries(*this, sessionID, fetchOptions.contains(WebsiteDataFetchOption::ComputeSizes), [this, domainsToDeleteAllNonCookieWebsiteDataFor, sessionID, callbackAggregator](auto entries) mutable {
1906             
1907             auto entriesToDelete = filterForRegistrableDomains(domainsToDeleteAllNonCookieWebsiteDataFor, entries);
1908
1909             for (const auto& entry : entriesToDelete)
1910                 callbackAggregator->m_domains.add(RegistrableDomain::uncheckedCreateFromHost(entry.origin.host));
1911
1912             for (auto& entry : entriesToDelete)
1913                 CacheStorage::Engine::clearCachesForOrigin(*this, sessionID, SecurityOriginData { entry.origin }, [callbackAggregator] { });
1914         });
1915     }
1916
1917     if (m_storageManagerSet->contains(sessionID)) {
1918         if (websiteDataTypes.contains(WebsiteDataType::SessionStorage)) {
1919             m_storageManagerSet->getSessionStorageOrigins(sessionID, [protectedThis = makeRef(*this), this, sessionID, callbackAggregator, domainsToDeleteAllNonCookieWebsiteDataFor](auto&& origins) {
1920                 auto originsToDelete = filterForRegistrableDomains(origins, domainsToDeleteAllNonCookieWebsiteDataFor, callbackAggregator->m_domains);
1921                 m_storageManagerSet->deleteSessionStorageForOrigins(sessionID, originsToDelete, [callbackAggregator] { });
1922             });
1923         }
1924
1925         if (websiteDataTypes.contains(WebsiteDataType::LocalStorage)) {
1926             m_storageManagerSet->getLocalStorageOrigins(sessionID, [protectedThis = makeRef(*this), this, sessionID, callbackAggregator, domainsToDeleteAllNonCookieWebsiteDataFor](auto&& origins) {
1927                 auto originsToDelete = filterForRegistrableDomains(origins, domainsToDeleteAllNonCookieWebsiteDataFor, callbackAggregator->m_domains);
1928                 m_storageManagerSet->deleteLocalStorageForOrigins(sessionID, originsToDelete, [callbackAggregator] { });
1929             });
1930         }
1931     }
1932
1933 #if ENABLE(INDEXED_DATABASE)
1934     auto path = m_idbDatabasePaths.get(sessionID);
1935     if (!path.isEmpty() && websiteDataTypes.contains(WebsiteDataType::IndexedDBDatabases)) {
1936         // FIXME: Pick the right database store based on the session ID.
1937         postStorageTask(CrossThreadTask([this, sessionID, callbackAggregator, path = crossThreadCopy(path), domainsToDeleteAllNonCookieWebsiteDataFor = crossThreadCopy(domainsToDeleteAllNonCookieWebsiteDataFor)]() mutable {
1938             RunLoop::main().dispatch([this, sessionID, domainsToDeleteAllNonCookieWebsiteDataFor = WTFMove(domainsToDeleteAllNonCookieWebsiteDataFor), callbackAggregator, securityOrigins = indexedDatabaseOrigins(path)] {
1939                 Vector<SecurityOriginData> entriesToDelete;
1940                 for (const auto& securityOrigin : securityOrigins) {
1941                     auto domain = RegistrableDomain::uncheckedCreateFromHost(securityOrigin.host);
1942                     if (!domainsToDeleteAllNonCookieWebsiteDataFor.contains(domain))
1943                         continue;
1944
1945                     entriesToDelete.append(securityOrigin);
1946                     callbackAggregator->m_domains.add(domain);
1947                 }
1948
1949                 webIDBServer(sessionID).closeAndDeleteDatabasesForOrigins(entriesToDelete, [callbackAggregator] { });
1950             });
1951         }));
1952     }
1953 #endif
1954     
1955 #if ENABLE(SERVICE_WORKER)
1956     path = m_serviceWorkerInfo.get(sessionID).databasePath;
1957     if (!path.isEmpty() && websiteDataTypes.contains(WebsiteDataType::ServiceWorkerRegistrations)) {
1958         swServerForSession(sessionID).getOriginsWithRegistrations([this, sessionID, domainsToDeleteAllNonCookieWebsiteDataFor, callbackAggregator](const HashSet<SecurityOriginData>& securityOrigins) mutable {
1959             for (auto& securityOrigin : securityOrigins) {
1960                 if (!domainsToDeleteAllNonCookieWebsiteDataFor.contains(RegistrableDomain::uncheckedCreateFromHost(securityOrigin.host)))
1961                     continue;
1962                 callbackAggregator->m_domains.add(RegistrableDomain::uncheckedCreateFromHost(securityOrigin.host));
1963                 swServerForSession(sessionID).clear(securityOrigin, [callbackAggregator] { });
1964             }
1965         });
1966     }
1967 #endif
1968
1969     if (websiteDataTypes.contains(WebsiteDataType::DiskCache)) {
1970         forEachNetworkSession([sessionID, fetchOptions, &domainsToDeleteAllNonCookieWebsiteDataFor, &callbackAggregator](auto& session) {
1971             fetchDiskCacheEntries(session.cache(), sessionID, fetchOptions, [domainsToDeleteAllNonCookieWebsiteDataFor, callbackAggregator, session = makeWeakPtr(&session)](auto entries) mutable {
1972                 if (!session)
1973                     return;
1974
1975                 Vector<SecurityOriginData> entriesToDelete;
1976                 for (auto& entry : entries) {
1977                     if (!domainsToDeleteAllNonCookieWebsiteDataFor.contains(RegistrableDomain::uncheckedCreateFromHost(entry.origin.host)))
1978                         continue;
1979                     entriesToDelete.append(entry.origin);
1980                     callbackAggregator->m_domains.add(RegistrableDomain::uncheckedCreateFromHost(entry.origin.host));
1981                 }
1982                 clearDiskCacheEntries(session->cache(), entriesToDelete, [callbackAggregator] { });
1983             });
1984         });
1985     }
1986
1987     auto dataTypesForUIProcess = WebsiteData::filter(websiteDataTypes, WebsiteDataProcessType::UI);
1988     if (!dataTypesForUIProcess.isEmpty() && !domainsToDeleteAllNonCookieWebsiteDataFor.isEmpty()) {
1989         CompletionHandler<void(const HashSet<RegistrableDomain>&)> completionHandler = [callbackAggregator] (const HashSet<RegistrableDomain>& domains) {
1990             for (auto& domain : domains)
1991                 callbackAggregator->m_domains.add(domain);
1992         };
1993         parentProcessConnection()->sendWithAsyncReply(Messages::NetworkProcessProxy::DeleteWebsiteDataInUIProcessForRegistrableDomains(sessionID, dataTypesForUIProcess, fetchOptions, domainsToDeleteAllNonCookieWebsiteDataFor), WTFMove(completionHandler));
1994     }
1995 }
1996
1997 void NetworkProcess::deleteCookiesForTesting(PAL::SessionID sessionID, RegistrableDomain domain, bool includeHttpOnlyCookies, CompletionHandler<void()>&& completionHandler)
1998 {
1999     OptionSet<WebsiteDataType> cookieType = WebsiteDataType::Cookies;
2000     RegistrableDomainsToDeleteOrRestrictWebsiteDataFor toDeleteFor;
2001     if (includeHttpOnlyCookies)
2002         toDeleteFor.domainsToDeleteAllCookiesFor.append(domain);
2003     else
2004         toDeleteFor.domainsToDeleteAllButHttpOnlyCookiesFor.append(domain);
2005
2006     deleteAndRestrictWebsiteDataForRegistrableDomains(sessionID, cookieType, WTFMove(toDeleteFor), true, [completionHandler = WTFMove(completionHandler)] (const HashSet<RegistrableDomain>& domainsDeletedFor) mutable {
2007         UNUSED_PARAM(domainsDeletedFor);
2008         completionHandler();
2009     });
2010 }
2011
2012 void NetworkProcess::registrableDomainsWithWebsiteData(PAL::SessionID sessionID, OptionSet<WebsiteDataType> websiteDataTypes, bool shouldNotifyPage, CompletionHandler<void(HashSet<RegistrableDomain>&&)>&& completionHandler)
2013 {
2014     OptionSet<WebsiteDataFetchOption> fetchOptions = WebsiteDataFetchOption::DoNotCreateProcesses;
2015     
2016     struct CallbackAggregator final : public ThreadSafeRefCounted<CallbackAggregator> {
2017         explicit CallbackAggregator(CompletionHandler<void(HashSet<RegistrableDomain>&&)>&& completionHandler)
2018             : m_completionHandler(WTFMove(completionHandler))
2019         {
2020         }
2021         
2022         ~CallbackAggregator()
2023         {
2024             RunLoop::main().dispatch([completionHandler = WTFMove(m_completionHandler), websiteData = WTFMove(m_websiteData)] () mutable {
2025                 HashSet<RegistrableDomain> domains;
2026                 for (const auto& hostnameWithCookies : websiteData.hostNamesWithCookies)
2027                     domains.add(RegistrableDomain::uncheckedCreateFromHost(hostnameWithCookies));
2028
2029                 for (const auto& hostnameWithHSTS : websiteData.hostNamesWithHSTSCache)
2030                     domains.add(RegistrableDomain::uncheckedCreateFromHost(hostnameWithHSTS));
2031
2032                 for (const auto& entry : websiteData.entries)
2033                     domains.add(RegistrableDomain::uncheckedCreateFromHost(entry.origin.host));
2034
2035                 completionHandler(WTFMove(domains));
2036             });
2037         }
2038         
2039         CompletionHandler<void(HashSet<RegistrableDomain>&&)> m_completionHandler;
2040         WebsiteData m_websiteData;
2041     };
2042     
2043     auto callbackAggregator = adoptRef(*new CallbackAggregator([this, completionHandler = WTFMove(completionHandler), shouldNotifyPage] (HashSet<RegistrableDomain>&& domainsWithData) mutable {
2044         if (shouldNotifyPage)
2045             parentProcessConnection()->send(Messages::NetworkProcessProxy::NotifyWebsiteDataScanForRegistrableDomainsFinished(), 0);
2046
2047         RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler), domainsWithData = crossThreadCopy(domainsWithData)] () mutable {
2048             completionHandler(WTFMove(domainsWithData));
2049         });
2050     }));
2051     
2052     auto& websiteData = callbackAggregator->m_websiteData;
2053     
2054     if (websiteDataTypes.contains(WebsiteDataType::Cookies)) {
2055         if (auto* networkStorageSession = storageSession(sessionID))
2056             networkStorageSession->getHostnamesWithCookies(websiteData.hostNamesWithCookies);
2057     }
2058     
2059 #if PLATFORM(COCOA) || USE(SOUP)
2060     if (websiteDataTypes.contains(WebsiteDataType::HSTSCache))
2061         websiteData.hostNamesWithHSTSCache = hostNamesWithHSTSCache(sessionID);
2062 #endif
2063
2064     if (websiteDataTypes.contains(WebsiteDataType::Credentials)) {
2065         if (auto* networkStorageSession = storageSession(sessionID)) {
2066             auto securityOrigins = networkStorageSession->credentialStorage().originsWithCredentials();
2067             for (auto& securityOrigin : securityOrigins)
2068                 callbackAggregator->m_websiteData.entries.append({ securityOrigin, WebsiteDataType::Credentials, 0 });
2069         }
2070     }
2071     
2072     if (websiteDataTypes.contains(WebsiteDataType::DOMCache)) {
2073         CacheStorage::Engine::fetchEntries(*this, sessionID, fetchOptions.contains(WebsiteDataFetchOption::ComputeSizes), [callbackAggregator](auto entries) mutable {
2074             callbackAggregator->m_websiteData.entries.appendVector(entries);
2075         });
2076     }
2077     
2078 #if ENABLE(INDEXED_DATABASE)
2079     auto path = m_idbDatabasePaths.get(sessionID);
2080     if (!path.isEmpty() && websiteDataTypes.contains(WebsiteDataType::IndexedDBDatabases)) {
2081         // FIXME: Pick the right database store based on the session ID.
2082         postStorageTask(CrossThreadTask([this, callbackAggregator, path = crossThreadCopy(path)]() mutable {
2083             RunLoop::main().dispatch([callbackAggregator, securityOrigins = indexedDatabaseOrigins(path)] {
2084                 for (const auto& securityOrigin : securityOrigins)
2085                     callbackAggregator->m_websiteData.entries.append({ securityOrigin, WebsiteDataType::IndexedDBDatabases, 0 });
2086             });
2087         }));
2088     }
2089 #endif
2090     
2091 #if ENABLE(SERVICE_WORKER)
2092     path = m_serviceWorkerInfo.get(sessionID).databasePath;
2093     if (!path.isEmpty() && websiteDataTypes.contains(WebsiteDataType::ServiceWorkerRegistrations)) {
2094         swServerForSession(sessionID).getOriginsWithRegistrations([callbackAggregator](const HashSet<SecurityOriginData>& securityOrigins) mutable {
2095             for (auto& securityOrigin : securityOrigins)
2096                 callbackAggregator->m_websiteData.entries.append({ securityOrigin, WebsiteDataType::ServiceWorkerRegistrations, 0 });
2097         });
2098     }
2099 #endif
2100     
2101     if (websiteDataTypes.contains(WebsiteDataType::DiskCache)) {
2102         forEachNetworkSession([sessionID, fetchOptions, &callbackAggregator](auto& session) {
2103             fetchDiskCacheEntries(session.cache(), sessionID, fetchOptions, [callbackAggregator](auto entries) mutable {
2104                 callbackAggregator->m_websiteData.entries.appendVector(entries);
2105             });
2106         });
2107     }
2108 }
2109 #endif // ENABLE(RESOURCE_LOAD_STATISTICS)
2110
2111 CacheStorage::Engine* NetworkProcess::findCacheEngine(const PAL::SessionID& sessionID)
2112 {
2113     return m_cacheEngines.get(sessionID);
2114 }
2115
2116 CacheStorage::Engine& NetworkProcess::ensureCacheEngine(const PAL::SessionID& sessionID, Function<Ref<CacheStorage::Engine>()>&& functor)
2117 {
2118     return m_cacheEngines.ensure(sessionID, WTFMove(functor)).iterator->value;
2119 }
2120
2121 void NetworkProcess::removeCacheEngine(const PAL::SessionID& sessionID)
2122 {
2123     m_cacheEngines.remove(sessionID);
2124 }
2125
2126 void NetworkProcess::downloadRequest(PAL::SessionID sessionID, DownloadID downloadID, const ResourceRequest& request, Optional<NavigatingToAppBoundDomain> isNavigatingToAppBoundDomain, const String& suggestedFilename)
2127 {
2128     downloadManager().startDownload(sessionID, downloadID, request, isNavigatingToAppBoundDomain, suggestedFilename);
2129 }
2130
2131 void NetworkProcess::resumeDownload(PAL::SessionID sessionID, DownloadID downloadID, const IPC::DataReference& resumeData, const String& path, WebKit::SandboxExtension::Handle&& sandboxExtensionHandle)
2132 {
2133     downloadManager().resumeDownload(sessionID, downloadID, resumeData, path, WTFMove(sandboxExtensionHandle));
2134 }
2135
2136 void NetworkProcess::cancelDownload(DownloadID downloadID)
2137 {
2138     downloadManager().cancelDownload(downloadID);
2139 }
2140
2141 #if PLATFORM(COCOA)
2142 void NetworkProcess::publishDownloadProgress(DownloadID downloadID, const URL& url, SandboxExtension::Handle&& sandboxExtensionHandle)
2143 {
2144     downloadManager().publishDownloadProgress(downloadID, url, WTFMove(sandboxExtensionHandle));
2145 }
2146 #endif
2147
2148 void NetworkProcess::continueWillSendRequest(DownloadID downloadID, WebCore::ResourceRequest&& request)
2149 {
2150     downloadManager().continueWillSendRequest(downloadID, WTFMove(request));
2151 }
2152
2153 void NetworkProcess::pendingDownloadCanceled(DownloadID downloadID)
2154 {
2155     downloadProxyConnection()->send(Messages::DownloadProxy::DidCancel({ }), downloadID.downloadID());
2156 }
2157
2158 void NetworkProcess::findPendingDownloadLocation(NetworkDataTask& networkDataTask, ResponseCompletionHandler&& completionHandler, const ResourceResponse& response)
2159 {
2160     uint64_t destinationID = networkDataTask.pendingDownloadID().downloadID();
2161     downloadProxyConnection()->send(Messages::DownloadProxy::DidReceiveResponse(response), destinationID);
2162
2163     downloadManager().willDecidePendingDownloadDestination(networkDataTask, WTFMove(completionHandler));
2164
2165     // As per https://html.spec.whatwg.org/#as-a-download (step 2), the filename from the Content-Disposition header
2166     // should override the suggested filename from the download attribute.
2167     String suggestedFilename = response.isAttachmentWithFilename() ? response.suggestedFilename() : networkDataTask.suggestedFilename();
2168     suggestedFilename = MIMETypeRegistry::appendFileExtensionIfNecessary(suggestedFilename, response.mimeType());
2169
2170     downloadProxyConnection()->send(Messages::DownloadProxy::DecideDestinationWithSuggestedFilenameAsync(networkDataTask.pendingDownloadID(), suggestedFilename), destinationID);
2171 }
2172
2173 void NetworkProcess::continueDecidePendingDownloadDestination(DownloadID downloadID, String destination, SandboxExtension::Handle&& sandboxExtensionHandle, bool allowOverwrite)
2174 {
2175     if (destination.isEmpty())
2176         downloadManager().cancelDownload(downloadID);
2177     else
2178         downloadManager().continueDecidePendingDownloadDestination(downloadID, destination, WTFMove(sandboxExtensionHandle), allowOverwrite);
2179 }
2180
2181 void NetworkProcess::setCacheModelSynchronouslyForTesting(CacheModel cacheModel, CompletionHandler<void()>&& completionHandler)
2182 {
2183     setCacheModel(cacheModel);
2184     completionHandler();
2185 }
2186
2187 void NetworkProcess::setCacheModel(CacheModel cacheModel)
2188 {
2189     if (m_hasSetCacheModel && (cacheModel == m_cacheModel))
2190         return;
2191
2192     m_hasSetCacheModel = true;
2193     m_cacheModel = cacheModel;
2194
2195     forEachNetworkSession([](auto& session) {
2196         if (auto* cache = session.cache())
2197             cache->updateCapacity();
2198     });
2199 }
2200
2201 void NetworkProcess::setAllowsAnySSLCertificateForWebSocket(bool allows, CompletionHandler<void()>&& completionHandler)
2202 {
2203     DeprecatedGlobalSettings::setAllowsAnySSLCertificate(allows);
2204     completionHandler();
2205 }
2206
2207 void NetworkProcess::logDiagnosticMessage(WebPageProxyIdentifier webPageProxyID, const String& message, const String& description, ShouldSample shouldSample)
2208 {
2209     if (!DiagnosticLoggingClient::shouldLogAfterSampling(shouldSample))
2210         return;
2211
2212     parentProcessConnection()->send(Messages::NetworkProcessProxy::LogDiagnosticMessage(webPageProxyID, message, description, ShouldSample::No), 0);
2213 }
2214
2215 void NetworkProcess::logDiagnosticMessageWithResult(WebPageProxyIdentifier webPageProxyID, const String& message, const String& description, DiagnosticLoggingResultType result, ShouldSample shouldSample)
2216 {
2217     if (!DiagnosticLoggingClient::shouldLogAfterSampling(shouldSample))
2218         return;
2219
2220     parentProcessConnection()->send(Messages::NetworkProcessProxy::LogDiagnosticMessageWithResult(webPageProxyID, message, description, result, ShouldSample::No), 0);
2221 }
2222
2223 void NetworkProcess::logDiagnosticMessageWithValue(WebPageProxyIdentifier webPageProxyID, const String& message, const String& description, double value, unsigned significantFigures, ShouldSample shouldSample)
2224 {
2225     if (!DiagnosticLoggingClient::shouldLogAfterSampling(shouldSample))
2226         return;
2227
2228     parentProcessConnection()->send(Messages::NetworkProcessProxy::LogDiagnosticMessageWithValue(webPageProxyID, message, description, value, significantFigures, ShouldSample::No), 0);
2229 }
2230
2231 void NetworkProcess::terminate()
2232 {
2233     platformTerminate();
2234     AuxiliaryProcess::terminate();
2235 }
2236
2237 void NetworkProcess::processDidTransitionToForeground()
2238 {
2239     platformProcessDidTransitionToForeground();
2240 }
2241
2242 void NetworkProcess::processDidTransitionToBackground()
2243 {
2244     platformProcessDidTransitionToBackground();
2245 }
2246
2247 void NetworkProcess::processWillSuspendImminentlyForTestingSync(CompletionHandler<void()>&& completionHandler)
2248 {
2249     prepareToSuspend(true, WTFMove(completionHandler));
2250 }
2251
2252 void NetworkProcess::prepareToSuspend(bool isSuspensionImminent, CompletionHandler<void()>&& completionHandler)
2253 {
2254     RELEASE_LOG(ProcessSuspension, "%p - NetworkProcess::prepareToSuspend(), isSuspensionImminent: %d", this, isSuspensionImminent);
2255
2256 #if PLATFORM(IOS_FAMILY) && ENABLE(INDEXED_DATABASE)
2257     for (auto& server : m_webIDBServers.values())
2258         server->suspend(isSuspensionImminent ? WebIDBServer::ShouldForceStop::Yes : WebIDBServer::ShouldForceStop::No);
2259 #endif
2260
2261 #if PLATFORM(IOS_FAMILY)
2262     m_webSQLiteDatabaseTracker.setIsSuspended(true);
2263 #endif
2264
2265     lowMemoryHandler(Critical::Yes);
2266
2267     RefPtr<CallbackAggregator> callbackAggregator = CallbackAggregator::create([this, completionHandler = WTFMove(completionHandler)]() mutable {
2268         RELEASE_LOG(ProcessSuspension, "%p - NetworkProcess::prepareToSuspend() Process is ready to suspend", this);
2269         completionHandler();
2270     });
2271     
2272 #if ENABLE(RESOURCE_LOAD_STATISTICS)
2273     WebResourceLoadStatisticsStore::suspend([callbackAggregator] { });
2274 #endif
2275
2276     platformSyncAllCookies([callbackAggregator] { });
2277
2278     for (auto& connection : m_webProcessConnections.values())
2279         connection->cleanupForSuspension([callbackAggregator] { });
2280
2281 #if ENABLE(SERVICE_WORKER)
2282     for (auto& server : m_swServers.values()) {
2283         ASSERT(m_swServers.get(server->sessionID()) == server.get());
2284         server->startSuspension([callbackAggregator] { });
2285     }
2286 #endif
2287
2288     m_storageManagerSet->suspend([callbackAggregator] { });
2289 }
2290
2291 void NetworkProcess::applicationDidEnterBackground()
2292 {
2293     m_downloadManager.applicationDidEnterBackground();
2294 }
2295
2296 void NetworkProcess::applicationWillEnterForeground()
2297 {
2298     m_downloadManager.applicationWillEnterForeground();
2299 }
2300
2301 void NetworkProcess::processDidResume()
2302 {
2303     RELEASE_LOG(ProcessSuspension, "%p - NetworkProcess::processDidResume()", this);
2304     resume();
2305 }
2306
2307 void NetworkProcess::resume()
2308 {
2309 #if PLATFORM(IOS_FAMILY)
2310     m_webSQLiteDatabaseTracker.setIsSuspended(false);
2311 #endif
2312
2313     for (auto& connection : m_webProcessConnections.values())
2314         connection->endSuspension();
2315
2316 #if ENABLE(RESOURCE_LOAD_STATISTICS)
2317     WebResourceLoadStatisticsStore::resume();
2318 #endif
2319     
2320 #if ENABLE(SERVICE_WORKER)
2321     for (auto& server : m_swServers.values())
2322         server->endSuspension();
2323 #endif
2324 #if PLATFORM(IOS_FAMILY) && ENABLE(INDEXED_DATABASE)
2325     for (auto& server : m_webIDBServers.values())
2326         server->resume();
2327 #endif
2328
2329     m_storageManagerSet->resume();
2330 }
2331
2332 void NetworkProcess::prefetchDNS(const String& hostname)
2333 {
2334     WebCore::prefetchDNS(hostname);
2335 }
2336
2337 void NetworkProcess::cacheStorageRootPath(PAL::SessionID sessionID, CacheStorageRootPathCallback&& callback)
2338 {
2339     m_cacheStorageParametersCallbacks.ensure(sessionID, [&] {
2340         parentProcessConnection()->send(Messages::NetworkProcessProxy::RetrieveCacheStorageParameters { sessionID }, 0);
2341         return Vector<CacheStorageRootPathCallback> { };
2342     }).iterator->value.append(WTFMove(callback));
2343 }
2344
2345 void NetworkProcess::setCacheStorageParameters(PAL::SessionID sessionID, String&& cacheStorageDirectory, SandboxExtension::Handle&& handle)
2346 {
2347     auto iterator = m_cacheStorageParametersCallbacks.find(sessionID);
2348     if (iterator == m_cacheStorageParametersCallbacks.end())
2349         return;
2350
2351     SandboxExtension::consumePermanently(handle);
2352     auto callbacks = WTFMove(iterator->value);
2353     m_cacheStorageParametersCallbacks.remove(iterator);
2354     for (auto& callback : callbacks)
2355         callback(String { cacheStorageDirectory });
2356 }
2357
2358 void NetworkProcess::registerURLSchemeAsSecure(const String& scheme) const
2359 {
2360     LegacySchemeRegistry::registerURLSchemeAsSecure(scheme);
2361 }
2362
2363 void NetworkProcess::registerURLSchemeAsBypassingContentSecurityPolicy(const String& scheme) const
2364 {
2365     LegacySchemeRegistry::registerURLSchemeAsBypassingContentSecurityPolicy(scheme);
2366 }
2367
2368 void NetworkProcess::registerURLSchemeAsLocal(const String& scheme) const
2369 {
2370     LegacySchemeRegistry::registerURLSchemeAsLocal(scheme);
2371 }
2372
2373 void NetworkProcess::registerURLSchemeAsNoAccess(const String& scheme) const
2374 {
2375     LegacySchemeRegistry::registerURLSchemeAsNoAccess(scheme);
2376 }
2377
2378 void NetworkProcess::didSyncAllCookies()
2379 {
2380     parentProcessConnection()->send(Messages::NetworkProcessProxy::DidSyncAllCookies(), 0);
2381 }
2382
2383 #if ENABLE(INDEXED_DATABASE)
2384 Ref<WebIDBServer> NetworkProcess::createWebIDBServer(PAL::SessionID sessionID)
2385 {
2386     String path;
2387     if (!sessionID.isEphemeral()) {
2388         ASSERT(m_idbDatabasePaths.contains(sessionID));
2389         path = m_idbDatabasePaths.get(sessionID);
2390     }
2391
2392     return WebIDBServer::create(sessionID, path, [this, weakThis = makeWeakPtr(this), sessionID](const auto& origin, uint64_t spaceRequested) {
2393         RefPtr<StorageQuotaManager> storageQuotaManager = weakThis ? this->storageQuotaManager(sessionID, origin) : nullptr;
2394         return storageQuotaManager ? storageQuotaManager->requestSpaceOnBackgroundThread(spaceRequested) : StorageQuotaManager::Decision::Deny;
2395     });
2396 }
2397
2398 WebIDBServer& NetworkProcess::webIDBServer(PAL::SessionID sessionID)
2399 {
2400     return *m_webIDBServers.ensure(sessionID, [this, sessionID] {
2401         return this->createWebIDBServer(sessionID);
2402     }).iterator->value;
2403 }
2404
2405 void NetworkProcess::ensurePathExists(const String& path)
2406 {
2407     ASSERT(!RunLoop::isMain());
2408     
2409     if (!FileSystem::makeAllDirectories(path))
2410         LOG_ERROR("Failed to make all directories for path '%s'", path.utf8().data());
2411 }
2412
2413 void NetworkProcess::postStorageTask(CrossThreadTask&& task)
2414 {
2415     ASSERT(RunLoop::isMain());
2416     
2417     LockHolder locker(m_storageTaskMutex);
2418     
2419     m_storageTasks.append(WTFMove(task));
2420     
2421     m_storageTaskQueue->dispatch([this] {
2422         performNextStorageTask();
2423     });
2424 }
2425
2426 void NetworkProcess::performNextStorageTask()
2427 {
2428     ASSERT(!RunLoop::isMain());
2429     
2430     CrossThreadTask task;
2431     {
2432         LockHolder locker(m_storageTaskMutex);
2433         ASSERT(!m_storageTasks.isEmpty());
2434         task = m_storageTasks.takeFirst();
2435     }
2436     
2437     task.performTask();
2438 }
2439
2440 void NetworkProcess::collectIndexedDatabaseOriginsForVersion(const String& path, HashSet<WebCore::SecurityOriginData>& securityOrigins)
2441 {
2442     if (path.isEmpty())
2443         return;
2444
2445     for (auto& topOriginPath : FileSystem::listDirectory(path, "*")) {
2446         auto databaseIdentifier = FileSystem::pathGetFileName(topOriginPath);
2447         if (auto securityOrigin = SecurityOriginData::fromDatabaseIdentifier(databaseIdentifier)) {
2448             securityOrigins.add(WTFMove(*securityOrigin));
2449         
2450             for (auto& originPath : FileSystem::listDirectory(topOriginPath, "*")) {
2451                 databaseIdentifier = FileSystem::pathGetFileName(originPath);
2452                 if (auto securityOrigin = SecurityOriginData::fromDatabaseIdentifier(databaseIdentifier))
2453                     securityOrigins.add(WTFMove(*securityOrigin));
2454             }
2455         }
2456     }
2457 }
2458
2459 HashSet<WebCore::SecurityOriginData> NetworkProcess::indexedDatabaseOrigins(const String& path)
2460 {
2461     if (path.isEmpty())
2462         return { };
2463     
2464     HashSet<WebCore::SecurityOriginData> securityOrigins;
2465     collectIndexedDatabaseOriginsForVersion(FileSystem::pathByAppendingComponent(path, "v0"), securityOrigins);
2466     collectIndexedDatabaseOriginsForVersion(FileSystem::pathByAppendingComponent(path, "v1"), securityOrigins);
2467
2468     return securityOrigins;
2469 }
2470
2471 void NetworkProcess::addIndexedDatabaseSession(PAL::SessionID sessionID, String& indexedDatabaseDirectory, SandboxExtension::Handle& handle)
2472 {
2473     // *********
2474     // IMPORTANT: Do not change the directory structure for indexed databases on disk without first consulting a reviewer from Apple (<rdar://problem/17454712>)
2475     // *********
2476     auto addResult = m_idbDatabasePaths.add(sessionID, indexedDatabaseDirectory);
2477     if (addResult.isNewEntry) {
2478         SandboxExtension::consumePermanently(handle);
2479         if (!indexedDatabaseDirectory.isEmpty())
2480             postStorageTask(createCrossThreadTask(*this, &NetworkProcess::ensurePathExists, indexedDatabaseDirectory));
2481         setSessionStorageQuotaManagerIDBRootPath(sessionID, indexedDatabaseDirectory);
2482     }
2483 }
2484
2485 void NetworkProcess::setSessionStorageQuotaManagerIDBRootPath(PAL::SessionID sessionID, const String& idbRootPath)
2486 {
2487     LockHolder locker(m_sessionStorageQuotaManagersLock);
2488     auto* sessionStorageQuotaManager = m_sessionStorageQuotaManagers.get(sessionID);
2489     ASSERT(sessionStorageQuotaManager);
2490     sessionStorageQuotaManager->setIDBRootPath(idbRootPath);
2491 }
2492
2493 #endif // ENABLE(INDEXED_DATABASE)
2494
2495 void NetworkProcess::syncLocalStorage(CompletionHandler<void()>&& completionHandler)
2496 {
2497     m_storageManagerSet->waitUntilSyncingLocalStorageFinished();
2498     completionHandler();
2499 }
2500
2501 void NetworkProcess::clearLegacyPrivateBrowsingLocalStorage()
2502 {
2503     if (m_storageManagerSet->contains(PAL::SessionID::legacyPrivateSessionID()))
2504         m_storageManagerSet->deleteLocalStorageModifiedSince(PAL::SessionID::legacyPrivateSessionID(), -WallTime::infinity(), []() { });
2505 }
2506
2507 void NetworkProcess::resetQuota(PAL::SessionID sessionID, CompletionHandler<void()>&& completionHandler)
2508 {
2509     LockHolder locker(m_sessionStorageQuotaManagersLock);
2510     if (auto* sessionStorageQuotaManager = m_sessionStorageQuotaManagers.get(sessionID)) {
2511         for (auto storageQuotaManager : sessionStorageQuotaManager->existingStorageQuotaManagers())
2512             storageQuotaManager->resetQuotaForTesting();
2513     }
2514     completionHandler();
2515 }
2516
2517 void NetworkProcess::renameOriginInWebsiteData(PAL::SessionID sessionID, const URL& oldName, const URL& newName, OptionSet<WebsiteDataType> dataTypes, CompletionHandler<void()>&& completionHandler)
2518 {
2519     auto aggregator = CallbackAggregator::create(WTFMove(completionHandler));
2520
2521     if (dataTypes.contains(WebsiteDataType::LocalStorage)) {
2522         if (m_storageManagerSet->contains(sessionID))
2523             m_storageManagerSet->renameOrigin(sessionID, oldName, newName, [aggregator] { });
2524     }
2525 }
2526
2527 #if ENABLE(SERVICE_WORKER)
2528 void NetworkProcess::forEachSWServer(const Function<void(SWServer&)>& callback)
2529 {
2530     for (auto& server : m_swServers.values())
2531         callback(*server);
2532 }
2533
2534 SWServer& NetworkProcess::swServerForSession(PAL::SessionID sessionID)
2535 {
2536     auto result = m_swServers.ensure(sessionID, [&] {
2537         auto info = m_serviceWorkerInfo.get(sessionID);
2538         auto path = info.databasePath;
2539         // There should already be a registered path for this PAL::SessionID.
2540         // If there's not, then where did this PAL::SessionID come from?
2541         ASSERT(sessionID.isEphemeral() || !path.isEmpty());
2542
2543         return makeUnique<SWServer>(makeUniqueRef<WebSWOriginStore>(), info.processTerminationDelayEnabled, WTFMove(path), sessionID, parentProcessHasServiceWorkerEntitlement(), [this, sessionID](auto&& jobData, bool shouldRefreshCache, auto&& request, auto&& completionHandler) mutable {
2544             ServiceWorkerSoftUpdateLoader::start(networkSession(sessionID), WTFMove(jobData), shouldRefreshCache, WTFMove(request), WTFMove(completionHandler));
2545         }, [this, sessionID](auto& registrableDomain, auto&& completionHandler) {
2546             ASSERT(!registrableDomain.isEmpty());
2547             parentProcessConnection()->sendWithAsyncReply(Messages::NetworkProcessProxy::EstablishWorkerContextConnectionToNetworkProcess { registrableDomain, sessionID }, WTFMove(completionHandler), 0);
2548         }, [this, sessionID](auto&& completionHandler) {
2549             parentProcessConnection()->sendWithAsyncReply(Messages::NetworkProcessProxy::GetAppBoundDomains { sessionID }, WTFMove(completionHandler), 0);
2550         });
2551     });
2552     return *result.iterator->value;
2553 }
2554
2555 WebSWOriginStore* NetworkProcess::existingSWOriginStoreForSession(PAL::SessionID sessionID) const
2556 {
2557     auto* swServer = m_swServers.get(sessionID);
2558     if (!swServer)
2559         return nullptr;
2560     return &static_cast<WebSWOriginStore&>(swServer->originStore());
2561 }
2562
2563 void NetworkProcess::registerSWServerConnection(WebSWServerConnection& connection)
2564 {
2565     auto* store = existingSWOriginStoreForSession(connection.sessionID());
2566     ASSERT(store);
2567     if (store)
2568         store->registerSWServerConnection(connection);
2569 }
2570
2571 void NetworkProcess::unregisterSWServerConnection(WebSWServerConnection& connection)
2572 {
2573     if (auto* store = existingSWOriginStoreForSession(connection.sessionID()))
2574         store->unregisterSWServerConnection(connection);
2575 }
2576
2577 void NetworkProcess::addServiceWorkerSession(PAL::SessionID sessionID, bool processTerminationDelayEnabled, String&& serviceWorkerRegistrationDirectory, const SandboxExtension::Handle& handle)
2578 {
2579     ServiceWorkerInfo info {
2580         WTFMove(serviceWorkerRegistrationDirectory),
2581         processTerminationDelayEnabled
2582     };
2583     auto addResult = m_serviceWorkerInfo.add(sessionID, WTFMove(info));
2584     if (addResult.isNewEntry) {
2585         SandboxExtension::consumePermanently(handle);
2586         if (!addResult.iterator->value.databasePath.isEmpty())
2587             postStorageTask(createCrossThreadTask(*this, &NetworkProcess::ensurePathExists, addResult.iterator->value.databasePath));
2588     }
2589 }
2590 #endif // ENABLE(SERVICE_WORKER)
2591
2592 void NetworkProcess::requestStorageSpace(PAL::SessionID sessionID, const ClientOrigin& origin, uint64_t quota, uint64_t currentSize, uint64_t spaceRequired, CompletionHandler<void(Optional<uint64_t>)>&& callback)
2593 {
2594     parentProcessConnection()->sendWithAsyncReply(Messages::NetworkProcessProxy::RequestStorageSpace { sessionID, origin, quota, currentSize, spaceRequired }, WTFMove(callback), 0);
2595 }
2596
2597 RefPtr<StorageQuotaManager> NetworkProcess::storageQuotaManager(PAL::SessionID sessionID, const ClientOrigin& origin)
2598 {
2599     LockHolder locker(m_sessionStorageQuotaManagersLock);
2600     auto* sessionStorageQuotaManager = m_sessionStorageQuotaManagers.get(sessionID);
2601     if (!sessionStorageQuotaManager)
2602         return nullptr;
2603
2604     String idbRootPath;
2605 #if ENABLE(INDEXED_DATABASE)
2606     idbRootPath = sessionStorageQuotaManager->idbRootPath();
2607 #endif
2608     StorageQuotaManager::UsageGetter usageGetter = [cacheRootPath = sessionStorageQuotaManager->cacheRootPath().isolatedCopy(), idbRootPath = idbRootPath.isolatedCopy(), origin = origin.isolatedCopy()]() {
2609         ASSERT(!isMainThread());    
2610
2611         uint64_t usage = CacheStorage::Engine::diskUsage(cacheRootPath, origin);
2612 #if ENABLE(INDEXED_DATABASE)
2613         usage += IDBServer::IDBServer::diskUsage(idbRootPath, origin);
2614 #endif
2615
2616         return usage;
2617     };
2618     StorageQuotaManager::QuotaIncreaseRequester quotaIncreaseRequester = [this, weakThis = makeWeakPtr(*this), sessionID, origin] (uint64_t currentQuota, uint64_t currentSpace, uint64_t requestedIncrease, auto&& callback) {
2619         ASSERT(isMainThread());
2620         if (!weakThis)
2621             callback({ });
2622         requestStorageSpace(sessionID, origin, currentQuota, currentSpace, requestedIncrease, WTFMove(callback));
2623     };
2624
2625     return sessionStorageQuotaManager->ensureOriginStorageQuotaManager(origin, sessionStorageQuotaManager->defaultQuota(origin), WTFMove(usageGetter), WTFMove(quotaIncreaseRequester)).ptr();
2626 }
2627
2628 #if !PLATFORM(COCOA)
2629 void NetworkProcess::initializeProcess(const AuxiliaryProcessInitializationParameters&)
2630 {
2631 }
2632
2633 void NetworkProcess::initializeProcessName(const AuxiliaryProcessInitializationParameters&)
2634 {
2635 }
2636
2637 void NetworkProcess::initializeSandbox(const AuxiliaryProcessInitializationParameters&, SandboxInitializationParameters&)
2638 {
2639 }
2640
2641 void NetworkProcess::syncAllCookies()
2642 {
2643 }
2644
2645 void NetworkProcess::platformSyncAllCookies(CompletionHandler<void()>&& completionHandler)
2646 {
2647     completionHandler();
2648 }
2649
2650 #endif
2651
2652 void NetworkProcess::storeAdClickAttribution(PAL::SessionID sessionID, WebCore::AdClickAttribution&& adClickAttribution)
2653 {
2654     if (auto* session = networkSession(sessionID))
2655         session->storeAdClickAttribution(WTFMove(adClickAttribution));
2656 }
2657
2658 void NetworkProcess::dumpAdClickAttribution(PAL::SessionID sessionID, CompletionHandler<void(String)>&& completionHandler)
2659 {
2660     if (auto* session = networkSession(sessionID))
2661         return session->dumpAdClickAttribution(WTFMove(completionHandler));
2662
2663     completionHandler({ });
2664 }
2665
2666 void NetworkProcess::clearAdClickAttribution(PAL::SessionID sessionID, CompletionHandler<void()>&& completionHandler)
2667 {
2668     if (auto* session = networkSession(sessionID))
2669         session->clearAdClickAttribution();
2670     
2671     completionHandler();
2672 }
2673
2674 void NetworkProcess::setAdClickAttributionOverrideTimerForTesting(PAL::SessionID sessionID, bool value, CompletionHandler<void()>&& completionHandler)
2675 {
2676     if (auto* session = networkSession(sessionID))
2677         session->setAdClickAttributionOverrideTimerForTesting(value);
2678     
2679     completionHandler();
2680 }
2681
2682 void NetworkProcess::setAdClickAttributionConversionURLForTesting(PAL::SessionID sessionID, URL&& url, CompletionHandler<void()>&& completionHandler)
2683 {
2684     if (auto* session = networkSession(sessionID))
2685         session->setAdClickAttributionConversionURLForTesting(WTFMove(url));
2686     
2687     completionHandler();
2688 }
2689
2690 void NetworkProcess::markAdClickAttributionsAsExpiredForTesting(PAL::SessionID sessionID, CompletionHandler<void()>&& completionHandler)
2691 {
2692     if (auto* session = networkSession(sessionID))
2693         session->markAdClickAttributionsAsExpiredForTesting();
2694
2695     completionHandler();
2696 }
2697
2698 void NetworkProcess::addKeptAliveLoad(Ref<NetworkResourceLoader>&& loader)
2699 {
2700     if (auto* session = networkSession(loader->sessionID()))
2701         session->addKeptAliveLoad(WTFMove(loader));
2702 }
2703
2704 void NetworkProcess::removeKeptAliveLoad(NetworkResourceLoader& loader)
2705 {
2706     if (auto* session = networkSession(loader.sessionID()))
2707         session->removeKeptAliveLoad(loader);
2708 }
2709
2710 void NetworkProcess::getLocalStorageOriginDetails(PAL::SessionID sessionID, CompletionHandler<void(Vector<LocalStorageDatabaseTracker::OriginDetails>&&)>&& completionHandler)
2711 {
2712     if (!m_storageManagerSet->contains(sessionID)) {
2713         LOG_ERROR("Cannot get local storage information for an unknown session");
2714         completionHandler({ });
2715         return;
2716     }
2717
2718     m_storageManagerSet->getLocalStorageOriginDetails(sessionID, WTFMove(completionHandler));
2719 }
2720
2721 void NetworkProcess::connectionToWebProcessClosed(IPC::Connection& connection, PAL::SessionID sessionID)
2722 {
2723     m_storageManagerSet->removeConnection(connection);
2724     if (auto* webIDBServer = m_webIDBServers.get(sessionID))
2725         webIDBServer->removeConnection(connection);
2726 }
2727
2728 NetworkConnectionToWebProcess* NetworkProcess::webProcessConnection(ProcessIdentifier identifier) const
2729 {
2730     return m_webProcessConnections.get(identifier);
2731 }
2732
2733 const Seconds NetworkProcess::defaultServiceWorkerFetchTimeout = 70_s;
2734 void NetworkProcess::setServiceWorkerFetchTimeoutForTesting(Seconds timeout, CompletionHandler<void()>&& completionHandler)
2735 {
2736     m_serviceWorkerFetchTimeout = timeout;
2737     completionHandler();
2738 }
2739
2740 void NetworkProcess::resetServiceWorkerFetchTimeoutForTesting(CompletionHandler<void()>&& completionHandler)
2741 {
2742     m_serviceWorkerFetchTimeout = defaultServiceWorkerFetchTimeout;
2743     completionHandler();
2744 }
2745
2746 void NetworkProcess::hasAppBoundSession(PAL::SessionID sessionID, CompletionHandler<void(bool)>&& completionHandler) const
2747 {
2748     bool result = false;
2749     if (auto* networkSession = this->networkSession(sessionID))
2750         result = networkSession->hasAppBoundSession();
2751     completionHandler(result);
2752 }
2753
2754 void NetworkProcess::clearAppBoundSession(PAL::SessionID sessionID, CompletionHandler<void()>&& completionHandler)
2755 {
2756     if (auto* networkSession = this->networkSession(sessionID)) {
2757         networkSession->clearAppBoundSession();
2758         completionHandler();
2759     } else {
2760         ASSERT_NOT_REACHED();
2761         completionHandler();
2762     }
2763 }
2764
2765 void NetworkProcess::broadcastConsoleMessage(PAL::SessionID sessionID, JSC::MessageSource source, JSC::MessageLevel level, const String& message)
2766 {
2767     for (auto& networkConnectionToWebProcess : m_webProcessConnections.values()) {
2768         if (networkConnectionToWebProcess->sessionID() == sessionID)
2769             networkConnectionToWebProcess->broadcastConsoleMessage(source, level, message);
2770     }
2771 }
2772
2773 void NetworkProcess::updateBundleIdentifier(String&& bundleIdentifier, CompletionHandler<void()>&& completionHandler)
2774 {
2775 #if PLATFORM(COCOA)
2776     WebCore::clearApplicationBundleIdentifierTestingOverride();
2777     WebCore::setApplicationBundleIdentifier(bundleIdentifier);
2778 #endif
2779     completionHandler();
2780 }
2781
2782 void NetworkProcess::clearBundleIdentifier(CompletionHandler<void()>&& completionHandler)
2783 {
2784 #if PLATFORM(COCOA)
2785     WebCore::clearApplicationBundleIdentifierTestingOverride();
2786 #endif
2787     completionHandler();
2788 }
2789
2790 } // namespace WebKit