Resource Load Statistics: Make network process calls only for the process pool that...
[WebKit-https.git] / Source / WebKit / UIProcess / WebsiteData / WebsiteDataStore.cpp
1 /*
2  * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WebsiteDataStore.h"
28
29 #include "APIProcessPoolConfiguration.h"
30 #include "APIWebsiteDataRecord.h"
31 #include "APIWebsiteDataStore.h"
32 #include "NetworkProcessMessages.h"
33 #include "StorageManager.h"
34 #include "StorageProcessCreationParameters.h"
35 #include "StorageProcessMessages.h"
36 #include "WebCookieManagerProxy.h"
37 #include "WebProcessMessages.h"
38 #include "WebProcessPool.h"
39 #include "WebResourceLoadStatisticsStore.h"
40 #include "WebResourceLoadStatisticsStoreMessages.h"
41 #include "WebsiteData.h"
42 #include "WebsiteDataStoreParameters.h"
43 #include <WebCore/ApplicationCacheStorage.h>
44 #include <WebCore/DatabaseTracker.h>
45 #include <WebCore/FileSystem.h>
46 #include <WebCore/HTMLMediaElement.h>
47 #include <WebCore/OriginLock.h>
48 #include <WebCore/SecurityOrigin.h>
49 #include <WebCore/SecurityOriginData.h>
50 #include <wtf/CallbackAggregator.h>
51 #include <wtf/CompletionHandler.h>
52 #include <wtf/CrossThreadCopier.h>
53 #include <wtf/ProcessPrivilege.h>
54 #include <wtf/RunLoop.h>
55
56 #if ENABLE(NETSCAPE_PLUGIN_API)
57 #include "PluginProcessManager.h"
58 #endif
59
60 #if HAVE(SEC_KEY_PROXY)
61 #include "SecKeyProxyStore.h"
62 #endif
63
64 namespace WebKit {
65
66 static bool allowsWebsiteDataRecordsForAllOrigins;
67 void WebsiteDataStore::allowWebsiteDataRecordsForAllOrigins()
68 {
69     allowsWebsiteDataRecordsForAllOrigins = true;
70 }
71
72 static HashMap<PAL::SessionID, WebsiteDataStore*>& nonDefaultDataStores()
73 {
74     RELEASE_ASSERT(isUIThread());
75     static NeverDestroyed<HashMap<PAL::SessionID, WebsiteDataStore*>> map;
76     return map;
77 }
78
79 Ref<WebsiteDataStore> WebsiteDataStore::createNonPersistent()
80 {
81     return adoptRef(*new WebsiteDataStore(PAL::SessionID::generateEphemeralSessionID()));
82 }
83
84 Ref<WebsiteDataStore> WebsiteDataStore::create(Configuration configuration, PAL::SessionID sessionID)
85 {
86     return adoptRef(*new WebsiteDataStore(WTFMove(configuration), sessionID));
87 }
88
89 WebsiteDataStore::Configuration::Configuration()
90     : resourceLoadStatisticsDirectory(API::WebsiteDataStore::defaultResourceLoadStatisticsDirectory())
91 {
92 }
93
94 WebsiteDataStore::WebsiteDataStore(Configuration configuration, PAL::SessionID sessionID)
95     : m_sessionID(sessionID)
96     , m_configuration(WTFMove(configuration))
97     , m_storageManager(StorageManager::create(m_configuration.localStorageDirectory))
98     , m_queue(WorkQueue::create("com.apple.WebKit.WebsiteDataStore"))
99 {
100     WTF::setProcessPrivileges(allPrivileges());
101     maybeRegisterWithSessionIDMap();
102     platformInitialize();
103 }
104
105 WebsiteDataStore::WebsiteDataStore(PAL::SessionID sessionID)
106     : m_sessionID(sessionID)
107     , m_configuration()
108     , m_queue(WorkQueue::create("com.apple.WebKit.WebsiteDataStore"))
109 {
110     maybeRegisterWithSessionIDMap();
111     platformInitialize();
112 }
113
114 WebsiteDataStore::~WebsiteDataStore()
115 {
116     platformDestroy();
117
118     if (m_sessionID.isValid() && m_sessionID != PAL::SessionID::defaultSessionID()) {
119         ASSERT(nonDefaultDataStores().get(m_sessionID) == this);
120         nonDefaultDataStores().remove(m_sessionID);
121         for (auto& processPool : WebProcessPool::allProcessPools()) {
122             processPool->sendToStorageProcess(Messages::StorageProcess::DestroySession(m_sessionID));
123             if (auto* networkProcess = processPool->networkProcess())
124                 networkProcess->removeSession(m_sessionID);
125         }
126     }
127 }
128
129 void WebsiteDataStore::maybeRegisterWithSessionIDMap()
130 {
131     if (m_sessionID.isValid() && m_sessionID != PAL::SessionID::defaultSessionID()) {
132         auto result = nonDefaultDataStores().add(m_sessionID, this);
133         ASSERT_UNUSED(result, result.isNewEntry);
134     }
135 }
136
137 WebsiteDataStore* WebsiteDataStore::existingNonDefaultDataStoreForSessionID(PAL::SessionID sessionID)
138 {
139     return sessionID.isValid() && sessionID != PAL::SessionID::defaultSessionID() ? nonDefaultDataStores().get(sessionID) : nullptr;
140 }
141
142 WebProcessPool* WebsiteDataStore::processPoolForCookieStorageOperations()
143 {
144     auto pools = processPools(1, false);
145     return pools.isEmpty() ? nullptr : pools.begin()->get();
146 }
147
148 void WebsiteDataStore::resolveDirectoriesIfNecessary()
149 {
150     if (m_hasResolvedDirectories)
151         return;
152     m_hasResolvedDirectories = true;
153
154     // Resolve directory paths.
155     if (!m_configuration.applicationCacheDirectory.isEmpty())
156         m_resolvedConfiguration.applicationCacheDirectory = resolveAndCreateReadWriteDirectoryForSandboxExtension(m_configuration.applicationCacheDirectory);
157     if (!m_configuration.mediaCacheDirectory.isEmpty())
158         m_resolvedConfiguration.mediaCacheDirectory = resolveAndCreateReadWriteDirectoryForSandboxExtension(m_configuration.mediaCacheDirectory);
159     if (!m_configuration.mediaKeysStorageDirectory.isEmpty())
160         m_resolvedConfiguration.mediaKeysStorageDirectory = resolveAndCreateReadWriteDirectoryForSandboxExtension(m_configuration.mediaKeysStorageDirectory);
161     if (!m_configuration.webSQLDatabaseDirectory.isEmpty())
162         m_resolvedConfiguration.webSQLDatabaseDirectory = resolveAndCreateReadWriteDirectoryForSandboxExtension(m_configuration.webSQLDatabaseDirectory);
163     if (!m_configuration.indexedDBDatabaseDirectory.isEmpty())
164         m_resolvedConfiguration.indexedDBDatabaseDirectory = resolveAndCreateReadWriteDirectoryForSandboxExtension(m_configuration.indexedDBDatabaseDirectory);
165     if (!m_configuration.resourceLoadStatisticsDirectory.isEmpty())
166         m_resolvedConfiguration.resourceLoadStatisticsDirectory = resolveAndCreateReadWriteDirectoryForSandboxExtension(m_configuration.resourceLoadStatisticsDirectory);
167     if (!m_configuration.serviceWorkerRegistrationDirectory.isEmpty() && m_resolvedConfiguration.serviceWorkerRegistrationDirectory.isEmpty())
168         m_resolvedConfiguration.serviceWorkerRegistrationDirectory = resolveAndCreateReadWriteDirectoryForSandboxExtension(m_configuration.serviceWorkerRegistrationDirectory);
169     if (!m_configuration.javaScriptConfigurationDirectory.isEmpty())
170         m_resolvedConfiguration.javaScriptConfigurationDirectory = resolvePathForSandboxExtension(m_configuration.javaScriptConfigurationDirectory);
171     if (!m_configuration.cacheStorageDirectory.isEmpty() && m_resolvedConfiguration.cacheStorageDirectory.isEmpty())
172         m_resolvedConfiguration.cacheStorageDirectory = resolvePathForSandboxExtension(m_configuration.cacheStorageDirectory);
173
174     // Resolve directories for file paths.
175     if (!m_configuration.cookieStorageFile.isEmpty()) {
176         m_resolvedConfiguration.cookieStorageFile = resolveAndCreateReadWriteDirectoryForSandboxExtension(WebCore::FileSystem::directoryName(m_configuration.cookieStorageFile));
177         m_resolvedConfiguration.cookieStorageFile = WebCore::FileSystem::pathByAppendingComponent(m_resolvedConfiguration.cookieStorageFile, WebCore::FileSystem::pathGetFileName(m_configuration.cookieStorageFile));
178     }
179 }
180
181 void WebsiteDataStore::cloneSessionData(WebPageProxy& sourcePage, WebPageProxy& newPage)
182 {
183     auto& sourceDataStore = sourcePage.websiteDataStore();
184     auto& newDataStore = newPage.websiteDataStore();
185
186     // FIXME: Handle this.
187     if (&sourceDataStore != &newDataStore)
188         return;
189
190     if (!sourceDataStore.m_storageManager)
191         return;
192
193     sourceDataStore.m_storageManager->cloneSessionStorageNamespace(sourcePage.pageID(), newPage.pageID());
194 }
195
196 enum class ProcessAccessType {
197     None,
198     OnlyIfLaunched,
199     Launch,
200 };
201
202 static ProcessAccessType computeNetworkProcessAccessTypeForDataFetch(OptionSet<WebsiteDataType> dataTypes, bool isNonPersistentStore)
203 {
204     ProcessAccessType processAccessType = ProcessAccessType::None;
205
206     if (dataTypes.contains(WebsiteDataType::Cookies)) {
207         if (isNonPersistentStore)
208             processAccessType = std::max(processAccessType, ProcessAccessType::OnlyIfLaunched);
209         else
210             processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
211     }
212
213     if (dataTypes.contains(WebsiteDataType::DiskCache) && !isNonPersistentStore)
214         processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
215
216     if (dataTypes.contains(WebsiteDataType::DOMCache))
217         processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
218
219     return processAccessType;
220 }
221
222 static ProcessAccessType computeWebProcessAccessTypeForDataFetch(OptionSet<WebsiteDataType> dataTypes, bool isNonPersistentStore)
223 {
224     UNUSED_PARAM(isNonPersistentStore);
225
226     ProcessAccessType processAccessType = ProcessAccessType::None;
227
228     if (dataTypes.contains(WebsiteDataType::MemoryCache))
229         return ProcessAccessType::OnlyIfLaunched;
230
231     return processAccessType;
232 }
233
234 void WebsiteDataStore::fetchData(OptionSet<WebsiteDataType> dataTypes, OptionSet<WebsiteDataFetchOption> fetchOptions, Function<void(Vector<WebsiteDataRecord>)>&& completionHandler)
235 {
236     fetchDataAndApply(dataTypes, fetchOptions, nullptr, WTFMove(completionHandler));
237 }
238
239 void WebsiteDataStore::fetchDataAndApply(OptionSet<WebsiteDataType> dataTypes, OptionSet<WebsiteDataFetchOption> fetchOptions, RefPtr<WorkQueue>&& queue, Function<void(Vector<WebsiteDataRecord>)>&& apply)
240 {
241     struct CallbackAggregator final : ThreadSafeRefCounted<CallbackAggregator> {
242         explicit CallbackAggregator(OptionSet<WebsiteDataFetchOption> fetchOptions, RefPtr<WorkQueue>&& queue, Function<void(Vector<WebsiteDataRecord>)>&& apply, WebsiteDataStore& dataStore)
243             : fetchOptions(fetchOptions)
244             , queue(WTFMove(queue))
245             , apply(WTFMove(apply))
246             , protectedDataStore(dataStore)
247         {
248         }
249
250         ~CallbackAggregator()
251         {
252             ASSERT(!pendingCallbacks);
253         }
254
255         void addPendingCallback()
256         {
257             pendingCallbacks++;
258         }
259
260         void removePendingCallback(WebsiteData websiteData)
261         {
262             ASSERT(pendingCallbacks);
263             --pendingCallbacks;
264
265             for (auto& entry : websiteData.entries) {
266                 auto displayName = WebsiteDataRecord::displayNameForOrigin(entry.origin);
267                 if (!displayName) {
268                     if (!allowsWebsiteDataRecordsForAllOrigins)
269                         continue;
270
271                     displayName = makeString(entry.origin.protocol, " ", entry.origin.host);
272                 }
273
274                 auto& record = m_websiteDataRecords.add(displayName, WebsiteDataRecord { }).iterator->value;
275                 if (!record.displayName)
276                     record.displayName = WTFMove(displayName);
277
278                 record.add(entry.type, entry.origin);
279
280                 if (fetchOptions.contains(WebsiteDataFetchOption::ComputeSizes)) {
281                     if (!record.size)
282                         record.size = WebsiteDataRecord::Size { 0, { } };
283
284                     record.size->totalSize += entry.size;
285                     record.size->typeSizes.add(static_cast<unsigned>(entry.type), 0).iterator->value += entry.size;
286                 }
287             }
288
289             for (auto& hostName : websiteData.hostNamesWithCookies) {
290                 auto displayName = WebsiteDataRecord::displayNameForCookieHostName(hostName);
291                 if (!displayName)
292                     continue;
293
294                 auto& record = m_websiteDataRecords.add(displayName, WebsiteDataRecord { }).iterator->value;
295                 if (!record.displayName)
296                     record.displayName = WTFMove(displayName);
297
298                 record.addCookieHostName(hostName);
299             }
300
301 #if ENABLE(NETSCAPE_PLUGIN_API)
302             for (auto& hostName : websiteData.hostNamesWithPluginData) {
303                 auto displayName = WebsiteDataRecord::displayNameForPluginDataHostName(hostName);
304                 if (!displayName)
305                     continue;
306
307                 auto& record = m_websiteDataRecords.add(displayName, WebsiteDataRecord { }).iterator->value;
308                 if (!record.displayName)
309                     record.displayName = WTFMove(displayName);
310
311                 record.addPluginDataHostName(hostName);
312             }
313 #endif
314
315             for (auto& origin : websiteData.originsWithCredentials) {
316                 auto& record = m_websiteDataRecords.add(origin, WebsiteDataRecord { }).iterator->value;
317                 
318                 record.addOriginWithCredential(origin);
319             }
320
321             callIfNeeded();
322         }
323
324         void callIfNeeded()
325         {
326             if (pendingCallbacks)
327                 return;
328
329             Vector<WebsiteDataRecord> records;
330             records.reserveInitialCapacity(m_websiteDataRecords.size());
331             for (auto& record : m_websiteDataRecords.values())
332                 records.uncheckedAppend(WTFMove(record));
333
334             auto processRecords = [apply = WTFMove(apply), records = WTFMove(records)] () mutable {
335                 apply(WTFMove(records));
336             };
337
338             if (queue)
339                 queue->dispatch(WTFMove(processRecords));
340             else
341                 RunLoop::main().dispatch(WTFMove(processRecords));
342         }
343
344         const OptionSet<WebsiteDataFetchOption> fetchOptions;
345
346         unsigned pendingCallbacks = 0;
347         RefPtr<WorkQueue> queue;
348         Function<void(Vector<WebsiteDataRecord>)> apply;
349
350         HashMap<String, WebsiteDataRecord> m_websiteDataRecords;
351         Ref<WebsiteDataStore> protectedDataStore;
352     };
353
354     RefPtr<CallbackAggregator> callbackAggregator = adoptRef(new CallbackAggregator(fetchOptions, WTFMove(queue), WTFMove(apply), *this));
355
356 #if ENABLE(VIDEO)
357     if (dataTypes.contains(WebsiteDataType::DiskCache)) {
358         callbackAggregator->addPendingCallback();
359         m_queue->dispatch([mediaCacheDirectory = m_configuration.mediaCacheDirectory.isolatedCopy(), callbackAggregator] {
360             // FIXME: Make HTMLMediaElement::originsInMediaCache return a collection of SecurityOriginDatas.
361             HashSet<RefPtr<WebCore::SecurityOrigin>> origins = WebCore::HTMLMediaElement::originsInMediaCache(mediaCacheDirectory);
362             WebsiteData websiteData;
363             
364             for (auto& origin : origins) {
365                 WebsiteData::Entry entry { origin->data(), WebsiteDataType::DiskCache, 0 };
366                 websiteData.entries.append(WTFMove(entry));
367             }
368             
369             RunLoop::main().dispatch([callbackAggregator, origins = WTFMove(origins), websiteData = WTFMove(websiteData)]() mutable {
370                 callbackAggregator->removePendingCallback(WTFMove(websiteData));
371             });
372         });
373     }
374 #endif
375
376     auto networkProcessAccessType = computeNetworkProcessAccessTypeForDataFetch(dataTypes, !isPersistent());
377     if (networkProcessAccessType != ProcessAccessType::None) {
378         for (auto& processPool : processPools()) {
379             switch (networkProcessAccessType) {
380             case ProcessAccessType::OnlyIfLaunched:
381                 if (!processPool->networkProcess())
382                     continue;
383                 break;
384
385             case ProcessAccessType::Launch:
386                 processPool->ensureNetworkProcess(this);
387                 break;
388
389             case ProcessAccessType::None:
390                 ASSERT_NOT_REACHED();
391             }
392
393             callbackAggregator->addPendingCallback();
394             processPool->networkProcess()->fetchWebsiteData(m_sessionID, dataTypes, fetchOptions, [callbackAggregator, processPool](WebsiteData websiteData) {
395                 callbackAggregator->removePendingCallback(WTFMove(websiteData));
396             });
397         }
398     }
399
400     auto webProcessAccessType = computeWebProcessAccessTypeForDataFetch(dataTypes, !isPersistent());
401     if (webProcessAccessType != ProcessAccessType::None) {
402         for (auto& process : processes()) {
403             switch (webProcessAccessType) {
404             case ProcessAccessType::OnlyIfLaunched:
405                 if (!process->canSendMessage())
406                     continue;
407                 break;
408
409             case ProcessAccessType::Launch:
410                 // FIXME: Handle this.
411                 ASSERT_NOT_REACHED();
412                 break;
413
414             case ProcessAccessType::None:
415                 ASSERT_NOT_REACHED();
416             }
417
418             callbackAggregator->addPendingCallback();
419             process->fetchWebsiteData(m_sessionID, dataTypes, [callbackAggregator](WebsiteData websiteData) {
420                 callbackAggregator->removePendingCallback(WTFMove(websiteData));
421             });
422         }
423     }
424
425     if (dataTypes.contains(WebsiteDataType::SessionStorage) && m_storageManager) {
426         callbackAggregator->addPendingCallback();
427
428         m_storageManager->getSessionStorageOrigins([callbackAggregator](HashSet<WebCore::SecurityOriginData>&& origins) {
429             WebsiteData websiteData;
430
431             while (!origins.isEmpty())
432                 websiteData.entries.append(WebsiteData::Entry { origins.takeAny(), WebsiteDataType::SessionStorage, 0 });
433
434             callbackAggregator->removePendingCallback(WTFMove(websiteData));
435         });
436     }
437
438     if (dataTypes.contains(WebsiteDataType::LocalStorage) && m_storageManager) {
439         callbackAggregator->addPendingCallback();
440
441         m_storageManager->getLocalStorageOrigins([callbackAggregator](HashSet<WebCore::SecurityOriginData>&& origins) {
442             WebsiteData websiteData;
443
444             while (!origins.isEmpty())
445                 websiteData.entries.append(WebsiteData::Entry { origins.takeAny(), WebsiteDataType::LocalStorage, 0 });
446
447             callbackAggregator->removePendingCallback(WTFMove(websiteData));
448         });
449     }
450
451     if (dataTypes.contains(WebsiteDataType::OfflineWebApplicationCache) && isPersistent()) {
452         callbackAggregator->addPendingCallback();
453
454         m_queue->dispatch([fetchOptions, applicationCacheDirectory = m_configuration.applicationCacheDirectory.isolatedCopy(), applicationCacheFlatFileSubdirectoryName = m_configuration.applicationCacheFlatFileSubdirectoryName.isolatedCopy(), callbackAggregator] {
455             auto storage = WebCore::ApplicationCacheStorage::create(applicationCacheDirectory, applicationCacheFlatFileSubdirectoryName);
456
457             WebsiteData websiteData;
458
459             // FIXME: getOriginsWithCache should return a collection of SecurityOriginDatas.
460             auto origins = storage->originsWithCache();
461
462             for (auto& origin : origins) {
463                 uint64_t size = fetchOptions.contains(WebsiteDataFetchOption::ComputeSizes) ? storage->diskUsageForOrigin(origin) : 0;
464                 WebsiteData::Entry entry { origin->data(), WebsiteDataType::OfflineWebApplicationCache, size };
465
466                 websiteData.entries.append(WTFMove(entry));
467             }
468
469             RunLoop::main().dispatch([callbackAggregator, origins = WTFMove(origins), websiteData = WTFMove(websiteData)]() mutable {
470                 callbackAggregator->removePendingCallback(WTFMove(websiteData));
471             });
472         });
473     }
474
475     if (dataTypes.contains(WebsiteDataType::WebSQLDatabases) && isPersistent()) {
476         callbackAggregator->addPendingCallback();
477
478         m_queue->dispatch([webSQLDatabaseDirectory = m_configuration.webSQLDatabaseDirectory.isolatedCopy(), callbackAggregator] {
479             auto origins = WebCore::DatabaseTracker::trackerWithDatabasePath(webSQLDatabaseDirectory)->origins();
480             RunLoop::main().dispatch([callbackAggregator, origins = WTFMove(origins)]() mutable {
481                 WebsiteData websiteData;
482                 for (auto& origin : origins)
483                     websiteData.entries.append(WebsiteData::Entry { WTFMove(origin), WebsiteDataType::WebSQLDatabases, 0 });
484                 callbackAggregator->removePendingCallback(WTFMove(websiteData));
485             });
486         });
487     }
488
489     if ((dataTypes.contains(WebsiteDataType::IndexedDBDatabases)
490 #if ENABLE(SERVICE_WORKER)
491         || dataTypes.contains(WebsiteDataType::ServiceWorkerRegistrations)
492 #endif
493         ) && isPersistent()) {
494         for (auto& processPool : processPools()) {
495             processPool->ensureStorageProcessAndWebsiteDataStore(this);
496
497             callbackAggregator->addPendingCallback();
498             processPool->storageProcess()->fetchWebsiteData(m_sessionID, dataTypes, [callbackAggregator, processPool](WebsiteData websiteData) {
499                 callbackAggregator->removePendingCallback(WTFMove(websiteData));
500             });
501         }
502     }
503
504     if (dataTypes.contains(WebsiteDataType::MediaKeys) && isPersistent()) {
505         callbackAggregator->addPendingCallback();
506
507         m_queue->dispatch([mediaKeysStorageDirectory = m_configuration.mediaKeysStorageDirectory.isolatedCopy(), callbackAggregator] {
508             auto origins = mediaKeyOrigins(mediaKeysStorageDirectory);
509
510             RunLoop::main().dispatch([callbackAggregator, origins = WTFMove(origins)]() mutable {
511                 WebsiteData websiteData;
512                 for (auto& origin : origins)
513                     websiteData.entries.append(WebsiteData::Entry { origin, WebsiteDataType::MediaKeys, 0 });
514
515                 callbackAggregator->removePendingCallback(WTFMove(websiteData));
516             });
517         });
518     }
519
520 #if ENABLE(NETSCAPE_PLUGIN_API)
521     if (dataTypes.contains(WebsiteDataType::PlugInData) && isPersistent()) {
522         class State {
523         public:
524             static void fetchData(Ref<CallbackAggregator>&& callbackAggregator, Vector<PluginModuleInfo>&& plugins)
525             {
526                 new State(WTFMove(callbackAggregator), WTFMove(plugins));
527             }
528
529         private:
530             State(Ref<CallbackAggregator>&& callbackAggregator, Vector<PluginModuleInfo>&& plugins)
531                 : m_callbackAggregator(WTFMove(callbackAggregator))
532                 , m_plugins(WTFMove(plugins))
533             {
534                 m_callbackAggregator->addPendingCallback();
535
536                 fetchWebsiteDataForNextPlugin();
537             }
538
539             ~State()
540             {
541                 ASSERT(m_plugins.isEmpty());
542             }
543
544             void fetchWebsiteDataForNextPlugin()
545             {
546                 if (m_plugins.isEmpty()) {
547                     WebsiteData websiteData;
548                     websiteData.hostNamesWithPluginData = WTFMove(m_hostNames);
549
550                     m_callbackAggregator->removePendingCallback(WTFMove(websiteData));
551
552                     delete this;
553                     return;
554                 }
555
556                 auto plugin = m_plugins.takeLast();
557                 PluginProcessManager::singleton().fetchWebsiteData(plugin, m_callbackAggregator->fetchOptions, [this](Vector<String> hostNames) {
558                     for (auto& hostName : hostNames)
559                         m_hostNames.add(WTFMove(hostName));
560                     fetchWebsiteDataForNextPlugin();
561                 });
562             }
563
564             Ref<CallbackAggregator> m_callbackAggregator;
565             Vector<PluginModuleInfo> m_plugins;
566             HashSet<String> m_hostNames;
567         };
568
569         State::fetchData(*callbackAggregator, plugins());
570     }
571 #endif
572
573     callbackAggregator->callIfNeeded();
574 }
575
576 void WebsiteDataStore::fetchDataForTopPrivatelyControlledDomains(OptionSet<WebsiteDataType> dataTypes, OptionSet<WebsiteDataFetchOption> fetchOptions, const Vector<String>& topPrivatelyControlledDomains, Function<void(Vector<WebsiteDataRecord>&&, HashSet<String>&&)>&& completionHandler)
577 {
578     fetchDataAndApply(dataTypes, fetchOptions, m_queue.copyRef(), [topPrivatelyControlledDomains = crossThreadCopy(topPrivatelyControlledDomains), completionHandler = WTFMove(completionHandler)] (auto&& existingDataRecords) mutable {
579         ASSERT(!RunLoop::isMain());
580
581         Vector<WebsiteDataRecord> matchingDataRecords;
582         HashSet<String> domainsWithMatchingDataRecords;
583         for (auto&& dataRecord : existingDataRecords) {
584             for (auto& topPrivatelyControlledDomain : topPrivatelyControlledDomains) {
585                 if (dataRecord.matchesTopPrivatelyControlledDomain(topPrivatelyControlledDomain)) {
586                     matchingDataRecords.append(WTFMove(dataRecord));
587                     domainsWithMatchingDataRecords.add(topPrivatelyControlledDomain.isolatedCopy());
588                     break;
589                 }
590             }
591         }
592         RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler), matchingDataRecords = WTFMove(matchingDataRecords), domainsWithMatchingDataRecords = WTFMove(domainsWithMatchingDataRecords)] () mutable {
593             completionHandler(WTFMove(matchingDataRecords), WTFMove(domainsWithMatchingDataRecords));
594         });
595     });
596 }
597     
598 void WebsiteDataStore::topPrivatelyControlledDomainsWithWebsiteData(OptionSet<WebsiteDataType> dataTypes, OptionSet<WebsiteDataFetchOption> fetchOptions, Function<void(HashSet<String>&&)>&& completionHandler)
599 {
600     fetchData(dataTypes, fetchOptions, [completionHandler = WTFMove(completionHandler), protectedThis = makeRef(*this)](auto&& existingDataRecords) {
601         HashSet<String> domainsWithDataRecords;
602         for (auto&& dataRecord : existingDataRecords) {
603             String domain = dataRecord.topPrivatelyControlledDomain();
604             if (domain.isEmpty())
605                 continue;
606             domainsWithDataRecords.add(WTFMove(domain));
607         }
608         completionHandler(WTFMove(domainsWithDataRecords));
609     });
610 }
611
612 static ProcessAccessType computeNetworkProcessAccessTypeForDataRemoval(OptionSet<WebsiteDataType> dataTypes, bool isNonPersistentStore)
613 {
614     ProcessAccessType processAccessType = ProcessAccessType::None;
615
616     if (dataTypes.contains(WebsiteDataType::Cookies)) {
617         if (isNonPersistentStore)
618             processAccessType = std::max(processAccessType, ProcessAccessType::OnlyIfLaunched);
619         else
620             processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
621     }
622
623     if (dataTypes.contains(WebsiteDataType::DiskCache) && !isNonPersistentStore)
624         processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
625
626     if (dataTypes.contains(WebsiteDataType::HSTSCache))
627         processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
628
629     if (dataTypes.contains(WebsiteDataType::Credentials))
630         processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
631
632     if (dataTypes.contains(WebsiteDataType::DOMCache))
633         processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
634
635     return processAccessType;
636 }
637
638 static ProcessAccessType computeWebProcessAccessTypeForDataRemoval(OptionSet<WebsiteDataType> dataTypes, bool isNonPersistentStore)
639 {
640     UNUSED_PARAM(isNonPersistentStore);
641
642     ProcessAccessType processAccessType = ProcessAccessType::None;
643
644     if (dataTypes.contains(WebsiteDataType::MemoryCache))
645         processAccessType = std::max(processAccessType, ProcessAccessType::OnlyIfLaunched);
646
647     if (dataTypes.contains(WebsiteDataType::Credentials))
648         processAccessType = std::max(processAccessType, ProcessAccessType::OnlyIfLaunched);
649
650     return processAccessType;
651 }
652
653 void WebsiteDataStore::removeData(OptionSet<WebsiteDataType> dataTypes, WallTime modifiedSince, Function<void()>&& completionHandler)
654 {
655     struct CallbackAggregator : ThreadSafeRefCounted<CallbackAggregator> {
656         explicit CallbackAggregator(WebsiteDataStore& dataStore, Function<void()>&& completionHandler)
657             : completionHandler(WTFMove(completionHandler))
658             , protectedDataStore(dataStore)
659         {
660         }
661
662         void addPendingCallback()
663         {
664             pendingCallbacks++;
665         }
666
667         void removePendingCallback()
668         {
669             ASSERT(pendingCallbacks);
670             --pendingCallbacks;
671
672             callIfNeeded();
673         }
674
675         void callIfNeeded()
676         {
677             if (!pendingCallbacks)
678                 RunLoop::main().dispatch(WTFMove(completionHandler));
679         }
680
681         unsigned pendingCallbacks = 0;
682         Function<void()> completionHandler;
683         Ref<WebsiteDataStore> protectedDataStore;
684     };
685
686     RefPtr<CallbackAggregator> callbackAggregator = adoptRef(new CallbackAggregator(*this, WTFMove(completionHandler)));
687
688 #if ENABLE(VIDEO)
689     if (dataTypes.contains(WebsiteDataType::DiskCache)) {
690         callbackAggregator->addPendingCallback();
691         m_queue->dispatch([modifiedSince, mediaCacheDirectory = m_configuration.mediaCacheDirectory.isolatedCopy(), callbackAggregator] {
692             WebCore::HTMLMediaElement::clearMediaCache(mediaCacheDirectory, modifiedSince);
693             
694             WTF::RunLoop::main().dispatch([callbackAggregator] {
695                 callbackAggregator->removePendingCallback();
696             });
697         });
698     }
699 #endif
700
701     auto networkProcessAccessType = computeNetworkProcessAccessTypeForDataRemoval(dataTypes, !isPersistent());
702     if (networkProcessAccessType != ProcessAccessType::None) {
703         for (auto& processPool : processPools()) {
704             switch (networkProcessAccessType) {
705             case ProcessAccessType::OnlyIfLaunched:
706                 if (!processPool->networkProcess())
707                     continue;
708                 break;
709
710             case ProcessAccessType::Launch:
711                 processPool->ensureNetworkProcess(this);
712                 break;
713
714             case ProcessAccessType::None:
715                 ASSERT_NOT_REACHED();
716             }
717
718             callbackAggregator->addPendingCallback();
719             processPool->networkProcess()->deleteWebsiteData(m_sessionID, dataTypes, modifiedSince, [callbackAggregator, processPool] {
720                 callbackAggregator->removePendingCallback();
721             });
722         }
723     }
724
725     auto webProcessAccessType = computeWebProcessAccessTypeForDataRemoval(dataTypes, !isPersistent());
726     if (webProcessAccessType != ProcessAccessType::None) {
727         for (auto& process : processes()) {
728             switch (webProcessAccessType) {
729             case ProcessAccessType::OnlyIfLaunched:
730                 if (!process->canSendMessage())
731                     continue;
732                 break;
733
734             case ProcessAccessType::Launch:
735                 // FIXME: Handle this.
736                 ASSERT_NOT_REACHED();
737                 break;
738
739             case ProcessAccessType::None:
740                 ASSERT_NOT_REACHED();
741             }
742
743             callbackAggregator->addPendingCallback();
744             process->deleteWebsiteData(m_sessionID, dataTypes, modifiedSince, [callbackAggregator] {
745                 callbackAggregator->removePendingCallback();
746             });
747         }
748     }
749
750     if (dataTypes.contains(WebsiteDataType::SessionStorage) && m_storageManager) {
751         callbackAggregator->addPendingCallback();
752
753         m_storageManager->deleteSessionStorageOrigins([callbackAggregator] {
754             callbackAggregator->removePendingCallback();
755         });
756     }
757
758     if (dataTypes.contains(WebsiteDataType::LocalStorage) && m_storageManager) {
759         callbackAggregator->addPendingCallback();
760
761         m_storageManager->deleteLocalStorageOriginsModifiedSince(modifiedSince, [callbackAggregator] {
762             callbackAggregator->removePendingCallback();
763         });
764     }
765
766     if (dataTypes.contains(WebsiteDataType::OfflineWebApplicationCache) && isPersistent()) {
767         callbackAggregator->addPendingCallback();
768
769         m_queue->dispatch([applicationCacheDirectory = m_configuration.applicationCacheDirectory.isolatedCopy(), applicationCacheFlatFileSubdirectoryName = m_configuration.applicationCacheFlatFileSubdirectoryName.isolatedCopy(), callbackAggregator] {
770             auto storage = WebCore::ApplicationCacheStorage::create(applicationCacheDirectory, applicationCacheFlatFileSubdirectoryName);
771
772             storage->deleteAllCaches();
773
774             WTF::RunLoop::main().dispatch([callbackAggregator] {
775                 callbackAggregator->removePendingCallback();
776             });
777         });
778     }
779
780     if (dataTypes.contains(WebsiteDataType::WebSQLDatabases) && isPersistent()) {
781         callbackAggregator->addPendingCallback();
782
783         m_queue->dispatch([webSQLDatabaseDirectory = m_configuration.webSQLDatabaseDirectory.isolatedCopy(), callbackAggregator, modifiedSince] {
784             WebCore::DatabaseTracker::trackerWithDatabasePath(webSQLDatabaseDirectory)->deleteDatabasesModifiedSince(modifiedSince);
785
786             RunLoop::main().dispatch([callbackAggregator] {
787                 callbackAggregator->removePendingCallback();
788             });
789         });
790     }
791
792     if ((dataTypes.contains(WebsiteDataType::IndexedDBDatabases)
793 #if ENABLE(SERVICE_WORKER)
794         || dataTypes.contains(WebsiteDataType::ServiceWorkerRegistrations)
795 #endif
796         ) && isPersistent()) {
797         for (auto& processPool : processPools()) {
798             processPool->ensureStorageProcessAndWebsiteDataStore(this);
799
800             callbackAggregator->addPendingCallback();
801             processPool->storageProcess()->deleteWebsiteData(m_sessionID, dataTypes, modifiedSince, [callbackAggregator, processPool] {
802                 callbackAggregator->removePendingCallback();
803             });
804         }
805     }
806
807     if (dataTypes.contains(WebsiteDataType::MediaKeys) && isPersistent()) {
808         callbackAggregator->addPendingCallback();
809
810         m_queue->dispatch([mediaKeysStorageDirectory = m_configuration.mediaKeysStorageDirectory.isolatedCopy(), callbackAggregator, modifiedSince] {
811             removeMediaKeys(mediaKeysStorageDirectory, modifiedSince);
812
813             RunLoop::main().dispatch([callbackAggregator] {
814                 callbackAggregator->removePendingCallback();
815             });
816         });
817     }
818
819     if (dataTypes.contains(WebsiteDataType::SearchFieldRecentSearches) && isPersistent()) {
820         callbackAggregator->addPendingCallback();
821
822         m_queue->dispatch([modifiedSince, callbackAggregator] {
823             platformRemoveRecentSearches(modifiedSince);
824
825             RunLoop::main().dispatch([callbackAggregator] {
826                 callbackAggregator->removePendingCallback();
827             });
828         });
829     }
830
831 #if ENABLE(NETSCAPE_PLUGIN_API)
832     if (dataTypes.contains(WebsiteDataType::PlugInData) && isPersistent()) {
833         class State {
834         public:
835             static void deleteData(Ref<CallbackAggregator>&& callbackAggregator, Vector<PluginModuleInfo>&& plugins, WallTime modifiedSince)
836             {
837                 new State(WTFMove(callbackAggregator), WTFMove(plugins), modifiedSince);
838             }
839
840         private:
841             State(Ref<CallbackAggregator>&& callbackAggregator, Vector<PluginModuleInfo>&& plugins, WallTime modifiedSince)
842                 : m_callbackAggregator(WTFMove(callbackAggregator))
843                 , m_plugins(WTFMove(plugins))
844                 , m_modifiedSince(modifiedSince)
845             {
846                 m_callbackAggregator->addPendingCallback();
847
848                 deleteWebsiteDataForNextPlugin();
849             }
850
851             ~State()
852             {
853                 ASSERT(m_plugins.isEmpty());
854             }
855
856             void deleteWebsiteDataForNextPlugin()
857             {
858                 if (m_plugins.isEmpty()) {
859                     m_callbackAggregator->removePendingCallback();
860
861                     delete this;
862                     return;
863                 }
864
865                 auto plugin = m_plugins.takeLast();
866                 PluginProcessManager::singleton().deleteWebsiteData(plugin, m_modifiedSince, [this] {
867                     deleteWebsiteDataForNextPlugin();
868                 });
869             }
870
871             Ref<CallbackAggregator> m_callbackAggregator;
872             Vector<PluginModuleInfo> m_plugins;
873             WallTime m_modifiedSince;
874         };
875
876         State::deleteData(*callbackAggregator, plugins(), modifiedSince);
877     }
878 #endif
879
880     if (dataTypes.contains(WebsiteDataType::ResourceLoadStatistics) && m_resourceLoadStatistics) {
881         auto deletedTypesRaw = dataTypes.toRaw();
882         auto monitoredTypesRaw = WebResourceLoadStatisticsStore::monitoredDataTypes().toRaw();
883         
884         // If we are deleting all of the data types that the resource load statistics store monitors
885         // we do not need to re-grandfather old data.
886         callbackAggregator->addPendingCallback();
887         if ((monitoredTypesRaw & deletedTypesRaw) == monitoredTypesRaw)
888             m_resourceLoadStatistics->scheduleClearInMemoryAndPersistent(modifiedSince, WebResourceLoadStatisticsStore::ShouldGrandfather::No, [callbackAggregator] {
889                 callbackAggregator->removePendingCallback();
890             });
891         else
892             m_resourceLoadStatistics->scheduleClearInMemoryAndPersistent(modifiedSince, WebResourceLoadStatisticsStore::ShouldGrandfather::Yes, [callbackAggregator] {
893                 callbackAggregator->removePendingCallback();
894             });
895
896         callbackAggregator->addPendingCallback();
897         clearResourceLoadStatisticsInWebProcesses([callbackAggregator] {
898             callbackAggregator->removePendingCallback();
899         });
900     }
901
902     // There's a chance that we don't have any pending callbacks. If so, we want to dispatch the completion handler right away.
903     callbackAggregator->callIfNeeded();
904 }
905
906 void WebsiteDataStore::removeData(OptionSet<WebsiteDataType> dataTypes, const Vector<WebsiteDataRecord>& dataRecords, Function<void()>&& completionHandler)
907 {
908     Vector<WebCore::SecurityOriginData> origins;
909
910     for (const auto& dataRecord : dataRecords) {
911         for (auto& origin : dataRecord.origins)
912             origins.append(origin);
913     }
914
915     struct CallbackAggregator : ThreadSafeRefCounted<CallbackAggregator> {
916         explicit CallbackAggregator(WebsiteDataStore& dataStore, Function<void()>&& completionHandler)
917             : completionHandler(WTFMove(completionHandler))
918             , protectedDataStore(dataStore)
919         {
920         }
921
922         void addPendingCallback()
923         {
924             pendingCallbacks++;
925         }
926
927         void removePendingCallback()
928         {
929             ASSERT(pendingCallbacks);
930             --pendingCallbacks;
931
932             callIfNeeded();
933         }
934
935         void callIfNeeded()
936         {
937             if (!pendingCallbacks)
938                 RunLoop::main().dispatch(WTFMove(completionHandler));
939         }
940
941         unsigned pendingCallbacks = 0;
942         Function<void()> completionHandler;
943         Ref<WebsiteDataStore> protectedDataStore;
944     };
945
946     RefPtr<CallbackAggregator> callbackAggregator = adoptRef(new CallbackAggregator(*this, WTFMove(completionHandler)));
947     
948     if (dataTypes.contains(WebsiteDataType::DiskCache)) {
949         HashSet<WebCore::SecurityOriginData> origins;
950         for (const auto& dataRecord : dataRecords) {
951             for (const auto& origin : dataRecord.origins)
952                 origins.add(origin);
953         }
954         
955 #if ENABLE(VIDEO)
956         callbackAggregator->addPendingCallback();
957         m_queue->dispatch([origins = WTFMove(origins), mediaCacheDirectory = m_configuration.mediaCacheDirectory.isolatedCopy(), callbackAggregator] {
958
959             // FIXME: Move SecurityOrigin::toRawString to SecurityOriginData and
960             // make HTMLMediaElement::clearMediaCacheForOrigins take SecurityOriginData.
961             HashSet<RefPtr<WebCore::SecurityOrigin>> securityOrigins;
962             for (auto& origin : origins)
963                 securityOrigins.add(origin.securityOrigin());
964             WebCore::HTMLMediaElement::clearMediaCacheForOrigins(mediaCacheDirectory, securityOrigins);
965             
966             WTF::RunLoop::main().dispatch([callbackAggregator] {
967                 callbackAggregator->removePendingCallback();
968             });
969         });
970 #endif
971     }
972     
973     auto networkProcessAccessType = computeNetworkProcessAccessTypeForDataRemoval(dataTypes, !isPersistent());
974     if (networkProcessAccessType != ProcessAccessType::None) {
975         for (auto& processPool : processPools()) {
976             switch (networkProcessAccessType) {
977             case ProcessAccessType::OnlyIfLaunched:
978                 if (!processPool->networkProcess())
979                     continue;
980                 break;
981
982             case ProcessAccessType::Launch:
983                 processPool->ensureNetworkProcess(this);
984                 break;
985
986             case ProcessAccessType::None:
987                 ASSERT_NOT_REACHED();
988             }
989
990             Vector<String> cookieHostNames;
991             for (const auto& dataRecord : dataRecords) {
992                 for (auto& hostName : dataRecord.cookieHostNames)
993                     cookieHostNames.append(hostName);
994             }
995
996             callbackAggregator->addPendingCallback();
997             processPool->networkProcess()->deleteWebsiteDataForOrigins(m_sessionID, dataTypes, origins, cookieHostNames, [callbackAggregator, processPool] {
998                 callbackAggregator->removePendingCallback();
999             });
1000         }
1001     }
1002
1003     auto webProcessAccessType = computeWebProcessAccessTypeForDataRemoval(dataTypes, !isPersistent());
1004     if (webProcessAccessType != ProcessAccessType::None) {
1005         for (auto& process : processes()) {
1006             switch (webProcessAccessType) {
1007             case ProcessAccessType::OnlyIfLaunched:
1008                 if (!process->canSendMessage())
1009                     continue;
1010                 break;
1011
1012             case ProcessAccessType::Launch:
1013                 // FIXME: Handle this.
1014                 ASSERT_NOT_REACHED();
1015                 break;
1016
1017             case ProcessAccessType::None:
1018                 ASSERT_NOT_REACHED();
1019             }
1020
1021             callbackAggregator->addPendingCallback();
1022
1023             process->deleteWebsiteDataForOrigins(m_sessionID, dataTypes, origins, [callbackAggregator] {
1024                 callbackAggregator->removePendingCallback();
1025             });
1026         }
1027     }
1028
1029     if (dataTypes.contains(WebsiteDataType::SessionStorage) && m_storageManager) {
1030         callbackAggregator->addPendingCallback();
1031
1032         m_storageManager->deleteSessionStorageEntriesForOrigins(origins, [callbackAggregator] {
1033             callbackAggregator->removePendingCallback();
1034         });
1035     }
1036
1037     if (dataTypes.contains(WebsiteDataType::LocalStorage) && m_storageManager) {
1038         callbackAggregator->addPendingCallback();
1039
1040         m_storageManager->deleteLocalStorageEntriesForOrigins(origins, [callbackAggregator] {
1041             callbackAggregator->removePendingCallback();
1042         });
1043     }
1044
1045     if (dataTypes.contains(WebsiteDataType::OfflineWebApplicationCache) && isPersistent()) {
1046         HashSet<WebCore::SecurityOriginData> origins;
1047         for (const auto& dataRecord : dataRecords) {
1048             for (const auto& origin : dataRecord.origins)
1049                 origins.add(origin);
1050         }
1051
1052         callbackAggregator->addPendingCallback();
1053         m_queue->dispatch([origins = WTFMove(origins), applicationCacheDirectory = m_configuration.applicationCacheDirectory.isolatedCopy(), applicationCacheFlatFileSubdirectoryName = m_configuration.applicationCacheFlatFileSubdirectoryName.isolatedCopy(), callbackAggregator] {
1054             auto storage = WebCore::ApplicationCacheStorage::create(applicationCacheDirectory, applicationCacheFlatFileSubdirectoryName);
1055
1056             for (const auto& origin : origins)
1057                 storage->deleteCacheForOrigin(origin.securityOrigin());
1058
1059             WTF::RunLoop::main().dispatch([callbackAggregator] {
1060                 callbackAggregator->removePendingCallback();
1061             });
1062         });
1063     }
1064
1065     if (dataTypes.contains(WebsiteDataType::WebSQLDatabases) && isPersistent()) {
1066         HashSet<WebCore::SecurityOriginData> origins;
1067         for (const auto& dataRecord : dataRecords) {
1068             for (const auto& origin : dataRecord.origins)
1069                 origins.add(origin);
1070         }
1071
1072         callbackAggregator->addPendingCallback();
1073         m_queue->dispatch([origins = WTFMove(origins), callbackAggregator, webSQLDatabaseDirectory = m_configuration.webSQLDatabaseDirectory.isolatedCopy()] {
1074             auto databaseTracker = WebCore::DatabaseTracker::trackerWithDatabasePath(webSQLDatabaseDirectory);
1075             for (auto& origin : origins)
1076                 databaseTracker->deleteOrigin(origin);
1077             RunLoop::main().dispatch([callbackAggregator] {
1078                 callbackAggregator->removePendingCallback();
1079             });
1080         });
1081     }
1082
1083     if ((dataTypes.contains(WebsiteDataType::IndexedDBDatabases)
1084 #if ENABLE(SERVICE_WORKER)
1085         || dataTypes.contains(WebsiteDataType::ServiceWorkerRegistrations)
1086 #endif
1087         ) && isPersistent()) {
1088         for (auto& processPool : processPools()) {
1089             processPool->ensureStorageProcessAndWebsiteDataStore(this);
1090
1091             callbackAggregator->addPendingCallback();
1092             processPool->storageProcess()->deleteWebsiteDataForOrigins(m_sessionID, dataTypes, origins, [callbackAggregator, processPool] {
1093                 callbackAggregator->removePendingCallback();
1094             });
1095         }
1096     }
1097
1098     if (dataTypes.contains(WebsiteDataType::MediaKeys) && isPersistent()) {
1099         HashSet<WebCore::SecurityOriginData> origins;
1100         for (const auto& dataRecord : dataRecords) {
1101             for (const auto& origin : dataRecord.origins)
1102                 origins.add(origin);
1103         }
1104
1105         callbackAggregator->addPendingCallback();
1106         m_queue->dispatch([mediaKeysStorageDirectory = m_configuration.mediaKeysStorageDirectory.isolatedCopy(), callbackAggregator, origins = WTFMove(origins)] {
1107
1108             removeMediaKeys(mediaKeysStorageDirectory, origins);
1109
1110             RunLoop::main().dispatch([callbackAggregator] {
1111                 callbackAggregator->removePendingCallback();
1112             });
1113         });
1114     }
1115
1116 #if ENABLE(NETSCAPE_PLUGIN_API)
1117     if (dataTypes.contains(WebsiteDataType::PlugInData) && isPersistent()) {
1118         Vector<String> hostNames;
1119         for (const auto& dataRecord : dataRecords) {
1120             for (const auto& hostName : dataRecord.pluginDataHostNames)
1121                 hostNames.append(hostName);
1122         }
1123
1124
1125         class State {
1126         public:
1127             static void deleteData(Ref<CallbackAggregator>&& callbackAggregator, Vector<PluginModuleInfo>&& plugins, Vector<String>&& hostNames)
1128             {
1129                 new State(WTFMove(callbackAggregator), WTFMove(plugins), WTFMove(hostNames));
1130             }
1131
1132         private:
1133             State(Ref<CallbackAggregator>&& callbackAggregator, Vector<PluginModuleInfo>&& plugins, Vector<String>&& hostNames)
1134                 : m_callbackAggregator(WTFMove(callbackAggregator))
1135                 , m_plugins(WTFMove(plugins))
1136                 , m_hostNames(WTFMove(hostNames))
1137             {
1138                 m_callbackAggregator->addPendingCallback();
1139
1140                 deleteWebsiteDataForNextPlugin();
1141             }
1142
1143             ~State()
1144             {
1145                 ASSERT(m_plugins.isEmpty());
1146             }
1147
1148             void deleteWebsiteDataForNextPlugin()
1149             {
1150                 if (m_plugins.isEmpty()) {
1151                     m_callbackAggregator->removePendingCallback();
1152
1153                     delete this;
1154                     return;
1155                 }
1156
1157                 auto plugin = m_plugins.takeLast();
1158                 PluginProcessManager::singleton().deleteWebsiteDataForHostNames(plugin, m_hostNames, [this] {
1159                     deleteWebsiteDataForNextPlugin();
1160                 });
1161             }
1162
1163             Ref<CallbackAggregator> m_callbackAggregator;
1164             Vector<PluginModuleInfo> m_plugins;
1165             Vector<String> m_hostNames;
1166         };
1167
1168         if (!hostNames.isEmpty())
1169             State::deleteData(*callbackAggregator, plugins(), WTFMove(hostNames));
1170     }
1171 #endif
1172
1173     // FIXME <rdar://problem/33491222>; scheduleClearInMemoryAndPersistent does not have a completion handler,
1174     // so the completion handler for this removeData() call can be called prematurely.
1175     if (dataTypes.contains(WebsiteDataType::ResourceLoadStatistics) && m_resourceLoadStatistics) {
1176         auto deletedTypesRaw = dataTypes.toRaw();
1177         auto monitoredTypesRaw = WebResourceLoadStatisticsStore::monitoredDataTypes().toRaw();
1178
1179         // If we are deleting all of the data types that the resource load statistics store monitors
1180         // we do not need to re-grandfather old data.
1181         callbackAggregator->addPendingCallback();
1182         if ((monitoredTypesRaw & deletedTypesRaw) == monitoredTypesRaw)
1183             m_resourceLoadStatistics->scheduleClearInMemoryAndPersistent(WebResourceLoadStatisticsStore::ShouldGrandfather::No, [callbackAggregator] {
1184                 callbackAggregator->removePendingCallback();
1185             });
1186         else
1187             m_resourceLoadStatistics->scheduleClearInMemoryAndPersistent(WebResourceLoadStatisticsStore::ShouldGrandfather::Yes, [callbackAggregator] {
1188                 callbackAggregator->removePendingCallback();
1189             });
1190
1191         callbackAggregator->addPendingCallback();
1192         clearResourceLoadStatisticsInWebProcesses([callbackAggregator] {
1193             callbackAggregator->removePendingCallback();
1194         });
1195     }
1196
1197     // There's a chance that we don't have any pending callbacks. If so, we want to dispatch the completion handler right away.
1198     callbackAggregator->callIfNeeded();
1199 }
1200
1201 void WebsiteDataStore::removeDataForTopPrivatelyControlledDomains(OptionSet<WebsiteDataType> dataTypes, OptionSet<WebsiteDataFetchOption> fetchOptions, const Vector<String>& topPrivatelyControlledDomains, Function<void(HashSet<String>&&)>&& completionHandler)
1202 {
1203     fetchDataForTopPrivatelyControlledDomains(dataTypes, fetchOptions, topPrivatelyControlledDomains, [dataTypes, completionHandler = WTFMove(completionHandler), this, protectedThis = makeRef(*this)](Vector<WebsiteDataRecord>&& websiteDataRecords, HashSet<String>&& domainsWithDataRecords) mutable {
1204         this->removeData(dataTypes, websiteDataRecords, [domainsWithDataRecords = WTFMove(domainsWithDataRecords), completionHandler = WTFMove(completionHandler)]() mutable {
1205             completionHandler(WTFMove(domainsWithDataRecords));
1206         });
1207     });
1208 }
1209
1210 #if HAVE(CFNETWORK_STORAGE_PARTITIONING)
1211 void WebsiteDataStore::updatePrevalentDomainsToPartitionOrBlockCookies(const Vector<String>& domainsToPartition, const Vector<String>& domainsToBlock, const Vector<String>& domainsToNeitherPartitionNorBlock, ShouldClearFirst shouldClearFirst, CompletionHandler<void()>&& completionHandler)
1212 {
1213     auto callbackAggregator = CallbackAggregator::create(WTFMove(completionHandler));
1214
1215     for (auto& processPool : processPools()) {
1216         if (auto* process = processPool->networkProcess())
1217             process->updatePrevalentDomainsToPartitionOrBlockCookies(m_sessionID, domainsToPartition, domainsToBlock, domainsToNeitherPartitionNorBlock, shouldClearFirst, [callbackAggregator = callbackAggregator.copyRef()] { });
1218     }
1219 }
1220
1221 void WebsiteDataStore::hasStorageAccessForFrameHandler(const String& resourceDomain, const String& firstPartyDomain, uint64_t frameID, uint64_t pageID, CompletionHandler<void(bool hasAccess)>&& completionHandler)
1222 {
1223     auto* webPage = WebProcessProxy::webPage(pageID);
1224     if (!webPage) {
1225         completionHandler(false);
1226         return;
1227     }
1228
1229     auto& networkProcess = webPage->process().processPool().ensureNetworkProcess();
1230     networkProcess.hasStorageAccessForFrame(m_sessionID, resourceDomain, firstPartyDomain, frameID, pageID, WTFMove(completionHandler));
1231 }
1232
1233 void WebsiteDataStore::getAllStorageAccessEntries(CompletionHandler<void(Vector<String>&& domains)>&& completionHandler)
1234 {
1235     // FIXME: Although this is only used for testing, it should not iterate and WTFMove the completion handler.
1236     for (auto& processPool : processPools())
1237         processPool->networkProcess()->getAllStorageAccessEntries(m_sessionID, WTFMove(completionHandler));
1238 }
1239
1240 void WebsiteDataStore::grantStorageAccessHandler(const String& resourceDomain, const String& firstPartyDomain, std::optional<uint64_t> frameID, uint64_t pageID, CompletionHandler<void(bool wasGranted)>&& completionHandler)
1241 {
1242     auto* webPage = WebProcessProxy::webPage(pageID);
1243     if (!webPage) {
1244         completionHandler(false);
1245         return;
1246     }
1247
1248     auto& networkProcess = webPage->process().processPool().ensureNetworkProcess();
1249     networkProcess.grantStorageAccess(m_sessionID, resourceDomain, firstPartyDomain, frameID, pageID, WTFMove(completionHandler));
1250 }
1251
1252 void WebsiteDataStore::removeAllStorageAccessHandler()
1253 {
1254     for (auto& processPool : processPools()) {
1255         if (auto networkProcess = processPool->networkProcess())
1256             networkProcess->removeAllStorageAccess(m_sessionID);
1257     }
1258 }
1259
1260 void WebsiteDataStore::removePrevalentDomains(const Vector<String>& domains)
1261 {
1262     for (auto& processPool : processPools())
1263         processPool->sendToNetworkingProcess(Messages::NetworkProcess::RemovePrevalentDomains(m_sessionID, domains));
1264 }
1265
1266 void WebsiteDataStore::hasStorageAccess(String&& subFrameHost, String&& topFrameHost, uint64_t frameID, uint64_t pageID, CompletionHandler<void(bool)>&& completionHandler)
1267 {
1268     if (!resourceLoadStatisticsEnabled()) {
1269         completionHandler(false);
1270         return;
1271     }
1272     
1273     m_resourceLoadStatistics->hasStorageAccess(WTFMove(subFrameHost), WTFMove(topFrameHost), frameID, pageID, WTFMove(completionHandler));
1274 }
1275
1276 void WebsiteDataStore::requestStorageAccess(String&& subFrameHost, String&& topFrameHost, uint64_t frameID, uint64_t pageID, bool promptEnabled, CompletionHandler<void(StorageAccessStatus)>&& completionHandler)
1277 {
1278     if (!resourceLoadStatisticsEnabled()) {
1279         completionHandler(StorageAccessStatus::CannotRequestAccess);
1280         return;
1281     }
1282     
1283     m_resourceLoadStatistics->requestStorageAccess(WTFMove(subFrameHost), WTFMove(topFrameHost), frameID, pageID, promptEnabled, WTFMove(completionHandler));
1284 }
1285
1286 void WebsiteDataStore::grantStorageAccess(String&& subFrameHost, String&& topFrameHost, uint64_t frameID, uint64_t pageID, bool userWasPrompted, CompletionHandler<void(bool)>&& completionHandler)
1287 {
1288     if (!resourceLoadStatisticsEnabled()) {
1289         completionHandler(false);
1290         return;
1291     }
1292     
1293     m_resourceLoadStatistics->grantStorageAccess(WTFMove(subFrameHost), WTFMove(topFrameHost), frameID, pageID, userWasPrompted, WTFMove(completionHandler));
1294 }
1295 #endif
1296
1297 void WebsiteDataStore::networkProcessDidCrash()
1298 {
1299 #if HAVE(CFNETWORK_STORAGE_PARTITIONING)
1300     if (m_resourceLoadStatistics)
1301         m_resourceLoadStatistics->scheduleCookiePartitioningStateReset();
1302 #endif
1303 }
1304
1305 void WebsiteDataStore::webPageWasAdded(WebPageProxy& webPageProxy)
1306 {
1307     if (m_storageManager)
1308         m_storageManager->createSessionStorageNamespace(webPageProxy.pageID(), std::numeric_limits<unsigned>::max());
1309 }
1310
1311 void WebsiteDataStore::webPageWasInvalidated(WebPageProxy& webPageProxy)
1312 {
1313     if (m_storageManager)
1314         m_storageManager->destroySessionStorageNamespace(webPageProxy.pageID());
1315 }
1316
1317 void WebsiteDataStore::webProcessWillOpenConnection(WebProcessProxy& webProcessProxy, IPC::Connection& connection)
1318 {
1319     if (m_storageManager)
1320         m_storageManager->processWillOpenConnection(webProcessProxy, connection);
1321
1322     if (m_resourceLoadStatistics)
1323         m_resourceLoadStatistics->processWillOpenConnection(webProcessProxy, connection);
1324 }
1325
1326 void WebsiteDataStore::webPageWillOpenConnection(WebPageProxy& webPageProxy, IPC::Connection& connection)
1327 {
1328     if (m_storageManager)
1329         m_storageManager->setAllowedSessionStorageNamespaceConnection(webPageProxy.pageID(), &connection);
1330 }
1331
1332 void WebsiteDataStore::webPageDidCloseConnection(WebPageProxy& webPageProxy, IPC::Connection&)
1333 {
1334     if (m_storageManager)
1335         m_storageManager->setAllowedSessionStorageNamespaceConnection(webPageProxy.pageID(), nullptr);
1336 }
1337
1338 void WebsiteDataStore::webProcessDidCloseConnection(WebProcessProxy& webProcessProxy, IPC::Connection& connection)
1339 {
1340     if (m_resourceLoadStatistics)
1341         m_resourceLoadStatistics->processDidCloseConnection(webProcessProxy, connection);
1342
1343     if (m_storageManager)
1344         m_storageManager->processDidCloseConnection(webProcessProxy, connection);
1345 }
1346
1347 bool WebsiteDataStore::isAssociatedProcessPool(WebProcessPool& processPool) const
1348 {
1349     if (auto* processPoolDataStore = processPool.websiteDataStore())
1350         return &processPoolDataStore->websiteDataStore() == this;
1351     return false;
1352 }
1353
1354 HashSet<RefPtr<WebProcessPool>> WebsiteDataStore::processPools(size_t count, bool ensureAPoolExists) const
1355 {
1356     HashSet<RefPtr<WebProcessPool>> processPools;
1357     for (auto& process : processes())
1358         processPools.add(&process->processPool());
1359
1360     if (processPools.isEmpty()) {
1361         // Check if we're one of the legacy data stores.
1362         for (auto& processPool : WebProcessPool::allProcessPools()) {
1363             if (!isAssociatedProcessPool(*processPool))
1364                 continue;
1365
1366             processPools.add(processPool);
1367
1368             if (processPools.size() == count)
1369                 break;
1370         }
1371     }
1372
1373     if (processPools.isEmpty() && count && ensureAPoolExists) {
1374         auto processPool = WebProcessPool::create(API::ProcessPoolConfiguration::createWithWebsiteDataStoreConfiguration(m_configuration));
1375         processPools.add(processPool.ptr());
1376     }
1377
1378     return processPools;
1379 }
1380
1381 #if ENABLE(NETSCAPE_PLUGIN_API)
1382 Vector<PluginModuleInfo> WebsiteDataStore::plugins() const
1383 {
1384     Vector<PluginModuleInfo> plugins;
1385
1386     for (auto& processPool : processPools()) {
1387         for (auto& plugin : processPool->pluginInfoStore().plugins())
1388             plugins.append(plugin);
1389     }
1390
1391     return plugins;
1392 }
1393 #endif
1394
1395 static String computeMediaKeyFile(const String& mediaKeyDirectory)
1396 {
1397     return WebCore::FileSystem::pathByAppendingComponent(mediaKeyDirectory, "SecureStop.plist");
1398 }
1399
1400 Vector<WebCore::SecurityOriginData> WebsiteDataStore::mediaKeyOrigins(const String& mediaKeysStorageDirectory)
1401 {
1402     ASSERT(!mediaKeysStorageDirectory.isEmpty());
1403
1404     Vector<WebCore::SecurityOriginData> origins;
1405
1406     for (const auto& originPath : WebCore::FileSystem::listDirectory(mediaKeysStorageDirectory, "*")) {
1407         auto mediaKeyFile = computeMediaKeyFile(originPath);
1408         if (!WebCore::FileSystem::fileExists(mediaKeyFile))
1409             continue;
1410
1411         auto mediaKeyIdentifier = WebCore::FileSystem::pathGetFileName(originPath);
1412
1413         if (auto securityOrigin = WebCore::SecurityOriginData::fromDatabaseIdentifier(mediaKeyIdentifier))
1414             origins.append(*securityOrigin);
1415     }
1416
1417     return origins;
1418 }
1419
1420 void WebsiteDataStore::removeMediaKeys(const String& mediaKeysStorageDirectory, WallTime modifiedSince)
1421 {
1422     ASSERT(!mediaKeysStorageDirectory.isEmpty());
1423
1424     for (const auto& mediaKeyDirectory : WebCore::FileSystem::listDirectory(mediaKeysStorageDirectory, "*")) {
1425         auto mediaKeyFile = computeMediaKeyFile(mediaKeyDirectory);
1426
1427         auto modificationTime = WebCore::FileSystem::getFileModificationTime(mediaKeyFile);
1428         if (!modificationTime)
1429             continue;
1430
1431         if (modificationTime.value() < modifiedSince)
1432             continue;
1433
1434         WebCore::FileSystem::deleteFile(mediaKeyFile);
1435         WebCore::FileSystem::deleteEmptyDirectory(mediaKeyDirectory);
1436     }
1437 }
1438
1439 void WebsiteDataStore::removeMediaKeys(const String& mediaKeysStorageDirectory, const HashSet<WebCore::SecurityOriginData>& origins)
1440 {
1441     ASSERT(!mediaKeysStorageDirectory.isEmpty());
1442
1443     for (const auto& origin : origins) {
1444         auto mediaKeyDirectory = WebCore::FileSystem::pathByAppendingComponent(mediaKeysStorageDirectory, origin.databaseIdentifier());
1445         auto mediaKeyFile = computeMediaKeyFile(mediaKeyDirectory);
1446
1447         WebCore::FileSystem::deleteFile(mediaKeyFile);
1448         WebCore::FileSystem::deleteEmptyDirectory(mediaKeyDirectory);
1449     }
1450 }
1451
1452 bool WebsiteDataStore::resourceLoadStatisticsEnabled() const
1453 {
1454     return !!m_resourceLoadStatistics;
1455 }
1456
1457 void WebsiteDataStore::setResourceLoadStatisticsEnabled(bool enabled)
1458 {
1459     if (m_sessionID.isEphemeral() || enabled == resourceLoadStatisticsEnabled())
1460         return;
1461
1462     if (enabled) {
1463         ASSERT(!m_resourceLoadStatistics);
1464         enableResourceLoadStatisticsAndSetTestingCallback(nullptr);
1465         return;
1466     }
1467
1468     m_resourceLoadStatistics = nullptr;
1469
1470     auto existingProcessPools = processPools(std::numeric_limits<size_t>::max(), false);
1471     for (auto& processPool : existingProcessPools)
1472         processPool->setResourceLoadStatisticsEnabled(false);
1473 }
1474
1475 bool WebsiteDataStore::resourceLoadStatisticsDebugMode() const
1476 {
1477     return m_resourceLoadStatisticsDebugMode;
1478 }
1479
1480 void WebsiteDataStore::setResourceLoadStatisticsDebugMode(bool enabled)
1481 {
1482     m_resourceLoadStatisticsDebugMode = enabled;
1483     if (m_resourceLoadStatistics)
1484         m_resourceLoadStatistics->setResourceLoadStatisticsDebugMode(enabled);
1485 }
1486
1487 void WebsiteDataStore::enableResourceLoadStatisticsAndSetTestingCallback(Function<void (const String&)>&& callback)
1488 {
1489     ASSERT(!m_sessionID.isEphemeral());
1490
1491     if (m_resourceLoadStatistics) {
1492         m_resourceLoadStatistics->setStatisticsTestingCallback(WTFMove(callback));
1493         return;
1494     }
1495
1496     resolveDirectoriesIfNecessary();
1497     m_resourceLoadStatistics = WebResourceLoadStatisticsStore::create(*this);
1498     m_resourceLoadStatistics->setStatisticsTestingCallback(WTFMove(callback));
1499
1500     for (auto& processPool : processPools(std::numeric_limits<size_t>::max(), false))
1501         processPool->setResourceLoadStatisticsEnabled(true);
1502 }
1503
1504 void WebsiteDataStore::clearResourceLoadStatisticsInWebProcesses(CompletionHandler<void()>&& callback)
1505 {
1506     if (resourceLoadStatisticsEnabled()) {
1507         for (auto& processPool : processPools())
1508             processPool->clearResourceLoadStatistics();
1509     }
1510     callback();
1511 }
1512
1513 StorageProcessCreationParameters WebsiteDataStore::storageProcessParameters()
1514 {
1515     resolveDirectoriesIfNecessary();
1516
1517     StorageProcessCreationParameters parameters;
1518
1519     parameters.sessionID = m_sessionID;
1520
1521 #if ENABLE(INDEXED_DATABASE)
1522     parameters.indexedDatabaseDirectory = resolvedIndexedDatabaseDirectory();
1523     if (!parameters.indexedDatabaseDirectory.isEmpty())
1524         SandboxExtension::createHandleForReadWriteDirectory(parameters.indexedDatabaseDirectory, parameters.indexedDatabaseDirectoryExtensionHandle);
1525 #endif
1526 #if ENABLE(SERVICE_WORKER)
1527     parameters.serviceWorkerRegistrationDirectory = resolvedServiceWorkerRegistrationDirectory();
1528     if (!parameters.serviceWorkerRegistrationDirectory.isEmpty())
1529         SandboxExtension::createHandleForReadWriteDirectory(parameters.serviceWorkerRegistrationDirectory, parameters.serviceWorkerRegistrationDirectoryExtensionHandle);
1530 #endif
1531
1532     return parameters;
1533 }
1534
1535 Vector<WebCore::Cookie> WebsiteDataStore::pendingCookies() const
1536 {
1537     return copyToVector(m_pendingCookies);
1538 }
1539
1540 void WebsiteDataStore::addPendingCookie(const WebCore::Cookie& cookie)
1541 {
1542     m_pendingCookies.add(cookie);
1543 }
1544
1545 void WebsiteDataStore::removePendingCookie(const WebCore::Cookie& cookie)
1546 {
1547     m_pendingCookies.remove(cookie);
1548 }
1549     
1550 void WebsiteDataStore::clearPendingCookies()
1551 {
1552     m_pendingCookies.clear();
1553 }
1554
1555 #if !PLATFORM(COCOA)
1556 WebsiteDataStoreParameters WebsiteDataStore::parameters()
1557 {
1558     // FIXME: Implement cookies.
1559     WebsiteDataStoreParameters parameters;
1560     parameters.networkSessionParameters.sessionID = m_sessionID;
1561     return parameters;
1562 }
1563 #endif
1564
1565 #if HAVE(SEC_KEY_PROXY)
1566 void WebsiteDataStore::addSecKeyProxyStore(Ref<SecKeyProxyStore>&& store)
1567 {
1568     m_secKeyProxyStores.append(WTFMove(store));
1569 }
1570 #endif
1571
1572 }