75ee8bfc5711a851036379dd0f0c905928e1e3c2
[WebKit-https.git] / Source / WebKit / UIProcess / WebsiteData / WebsiteDataStore.cpp
1 /*
2  * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WebsiteDataStore.h"
28
29 #include "APIProcessPoolConfiguration.h"
30 #include "APIWebsiteDataRecord.h"
31 #include "NetworkProcessMessages.h"
32 #include "StorageManager.h"
33 #include "StorageProcessCreationParameters.h"
34 #include "WebCookieManagerProxy.h"
35 #include "WebProcessMessages.h"
36 #include "WebProcessPool.h"
37 #include "WebResourceLoadStatisticsStore.h"
38 #include "WebResourceLoadStatisticsStoreMessages.h"
39 #include "WebsiteData.h"
40 #include "WebsiteDataStoreParameters.h"
41 #include <WebCore/ApplicationCacheStorage.h>
42 #include <WebCore/DatabaseTracker.h>
43 #include <WebCore/FileSystem.h>
44 #include <WebCore/HTMLMediaElement.h>
45 #include <WebCore/OriginLock.h>
46 #include <WebCore/SecurityOrigin.h>
47 #include <WebCore/SecurityOriginData.h>
48 #include <wtf/CompletionHandler.h>
49 #include <wtf/CrossThreadCopier.h>
50 #include <wtf/RunLoop.h>
51
52 #if ENABLE(NETSCAPE_PLUGIN_API)
53 #include "PluginProcessManager.h"
54 #endif
55
56 namespace WebKit {
57
58 Ref<WebsiteDataStore> WebsiteDataStore::createNonPersistent()
59 {
60     return adoptRef(*new WebsiteDataStore(PAL::SessionID::generateEphemeralSessionID()));
61 }
62
63 Ref<WebsiteDataStore> WebsiteDataStore::create(Configuration configuration, PAL::SessionID sessionID)
64 {
65     return adoptRef(*new WebsiteDataStore(WTFMove(configuration), sessionID));
66 }
67
68 WebsiteDataStore::WebsiteDataStore(Configuration configuration, PAL::SessionID sessionID)
69     : m_sessionID(sessionID)
70     , m_configuration(WTFMove(configuration))
71     , m_storageManager(StorageManager::create(m_configuration.localStorageDirectory))
72     , m_queue(WorkQueue::create("com.apple.WebKit.WebsiteDataStore"))
73 {
74     platformInitialize();
75 }
76
77 WebsiteDataStore::WebsiteDataStore(PAL::SessionID sessionID)
78     : m_sessionID(sessionID)
79     , m_configuration()
80     , m_queue(WorkQueue::create("com.apple.WebKit.WebsiteDataStore"))
81 {
82     platformInitialize();
83 }
84
85 WebsiteDataStore::~WebsiteDataStore()
86 {
87     platformDestroy();
88
89     if (m_sessionID.isValid() && m_sessionID != PAL::SessionID::defaultSessionID()) {
90         for (auto& processPool : WebProcessPool::allProcessPools())
91             processPool->sendToNetworkingProcess(Messages::NetworkProcess::DestroySession(m_sessionID));
92     }
93 }
94
95 WebProcessPool* WebsiteDataStore::processPoolForCookieStorageOperations()
96 {
97     auto pools = processPools(1, false);
98     return pools.isEmpty() ? nullptr : pools.begin()->get();
99 }
100
101 void WebsiteDataStore::resolveDirectoriesIfNecessary()
102 {
103     if (m_hasResolvedDirectories)
104         return;
105     m_hasResolvedDirectories = true;
106
107     // Resolve directory paths.
108     if (!m_configuration.applicationCacheDirectory.isEmpty())
109         m_resolvedConfiguration.applicationCacheDirectory = resolveAndCreateReadWriteDirectoryForSandboxExtension(m_configuration.applicationCacheDirectory);
110     if (!m_configuration.mediaCacheDirectory.isEmpty())
111         m_resolvedConfiguration.mediaCacheDirectory = resolveAndCreateReadWriteDirectoryForSandboxExtension(m_configuration.mediaCacheDirectory);
112     if (!m_configuration.mediaKeysStorageDirectory.isEmpty())
113         m_resolvedConfiguration.mediaKeysStorageDirectory = resolveAndCreateReadWriteDirectoryForSandboxExtension(m_configuration.mediaKeysStorageDirectory);
114     if (!m_configuration.webSQLDatabaseDirectory.isEmpty())
115         m_resolvedConfiguration.webSQLDatabaseDirectory = resolveAndCreateReadWriteDirectoryForSandboxExtension(m_configuration.webSQLDatabaseDirectory);
116     if (!m_configuration.indexedDBDatabaseDirectory.isEmpty())
117         m_resolvedConfiguration.indexedDBDatabaseDirectory = resolveAndCreateReadWriteDirectoryForSandboxExtension(m_configuration.indexedDBDatabaseDirectory);
118     if (!m_configuration.serviceWorkerRegistrationDirectory.isEmpty())
119         m_resolvedConfiguration.serviceWorkerRegistrationDirectory = resolveAndCreateReadWriteDirectoryForSandboxExtension(m_configuration.serviceWorkerRegistrationDirectory);
120     if (!m_configuration.javaScriptConfigurationDirectory.isEmpty())
121         m_resolvedConfiguration.javaScriptConfigurationDirectory = resolvePathForSandboxExtension(m_configuration.javaScriptConfigurationDirectory);
122     if (!m_configuration.cacheStorageDirectory.isEmpty() && m_resolvedConfiguration.cacheStorageDirectory.isEmpty())
123         m_resolvedConfiguration.cacheStorageDirectory = resolvePathForSandboxExtension(m_configuration.cacheStorageDirectory);
124
125     // Resolve directories for file paths.
126     if (!m_configuration.cookieStorageFile.isEmpty()) {
127         m_resolvedConfiguration.cookieStorageFile = resolveAndCreateReadWriteDirectoryForSandboxExtension(WebCore::FileSystem::directoryName(m_configuration.cookieStorageFile));
128         m_resolvedConfiguration.cookieStorageFile = WebCore::FileSystem::pathByAppendingComponent(m_resolvedConfiguration.cookieStorageFile, WebCore::FileSystem::pathGetFileName(m_configuration.cookieStorageFile));
129     }
130 }
131
132 void WebsiteDataStore::cloneSessionData(WebPageProxy& sourcePage, WebPageProxy& newPage)
133 {
134     auto& sourceDataStore = sourcePage.websiteDataStore();
135     auto& newDataStore = newPage.websiteDataStore();
136
137     // FIXME: Handle this.
138     if (&sourceDataStore != &newDataStore)
139         return;
140
141     if (!sourceDataStore.m_storageManager)
142         return;
143
144     sourceDataStore.m_storageManager->cloneSessionStorageNamespace(sourcePage.pageID(), newPage.pageID());
145 }
146
147 enum class ProcessAccessType {
148     None,
149     OnlyIfLaunched,
150     Launch,
151 };
152
153 static ProcessAccessType computeNetworkProcessAccessTypeForDataFetch(OptionSet<WebsiteDataType> dataTypes, bool isNonPersistentStore)
154 {
155     ProcessAccessType processAccessType = ProcessAccessType::None;
156
157     if (dataTypes.contains(WebsiteDataType::Cookies)) {
158         if (isNonPersistentStore)
159             processAccessType = std::max(processAccessType, ProcessAccessType::OnlyIfLaunched);
160         else
161             processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
162     }
163
164     if (dataTypes.contains(WebsiteDataType::DiskCache) && !isNonPersistentStore)
165         processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
166
167     if (dataTypes.contains(WebsiteDataType::DOMCache))
168         processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
169
170     return processAccessType;
171 }
172
173 static ProcessAccessType computeWebProcessAccessTypeForDataFetch(OptionSet<WebsiteDataType> dataTypes, bool isNonPersistentStore)
174 {
175     UNUSED_PARAM(isNonPersistentStore);
176
177     ProcessAccessType processAccessType = ProcessAccessType::None;
178
179     if (dataTypes.contains(WebsiteDataType::MemoryCache))
180         return ProcessAccessType::OnlyIfLaunched;
181
182     return processAccessType;
183 }
184
185 void WebsiteDataStore::fetchData(OptionSet<WebsiteDataType> dataTypes, OptionSet<WebsiteDataFetchOption> fetchOptions, Function<void(Vector<WebsiteDataRecord>)>&& completionHandler)
186 {
187     fetchDataAndApply(dataTypes, fetchOptions, nullptr, WTFMove(completionHandler));
188 }
189
190 void WebsiteDataStore::fetchDataAndApply(OptionSet<WebsiteDataType> dataTypes, OptionSet<WebsiteDataFetchOption> fetchOptions, RefPtr<WorkQueue>&& queue, Function<void(Vector<WebsiteDataRecord>)>&& apply)
191 {
192     struct CallbackAggregator final : ThreadSafeRefCounted<CallbackAggregator> {
193         explicit CallbackAggregator(OptionSet<WebsiteDataFetchOption> fetchOptions, RefPtr<WorkQueue>&& queue, Function<void(Vector<WebsiteDataRecord>)>&& apply)
194             : fetchOptions(fetchOptions)
195             , queue(WTFMove(queue))
196             , apply(WTFMove(apply))
197         {
198         }
199
200         ~CallbackAggregator()
201         {
202             ASSERT(!pendingCallbacks);
203         }
204
205         void addPendingCallback()
206         {
207             pendingCallbacks++;
208         }
209
210         void removePendingCallback(WebsiteData websiteData)
211         {
212             ASSERT(pendingCallbacks);
213             --pendingCallbacks;
214
215             for (auto& entry : websiteData.entries) {
216                 auto displayName = WebsiteDataRecord::displayNameForOrigin(entry.origin);
217                 if (!displayName)
218                     continue;
219
220                 auto& record = m_websiteDataRecords.add(displayName, WebsiteDataRecord { }).iterator->value;
221                 if (!record.displayName)
222                     record.displayName = WTFMove(displayName);
223
224                 record.add(entry.type, entry.origin);
225
226                 if (fetchOptions.contains(WebsiteDataFetchOption::ComputeSizes)) {
227                     if (!record.size)
228                         record.size = WebsiteDataRecord::Size { 0, { } };
229
230                     record.size->totalSize += entry.size;
231                     record.size->typeSizes.add(static_cast<unsigned>(entry.type), 0).iterator->value += entry.size;
232                 }
233             }
234
235             for (auto& hostName : websiteData.hostNamesWithCookies) {
236                 auto displayName = WebsiteDataRecord::displayNameForCookieHostName(hostName);
237                 if (!displayName)
238                     continue;
239
240                 auto& record = m_websiteDataRecords.add(displayName, WebsiteDataRecord { }).iterator->value;
241                 if (!record.displayName)
242                     record.displayName = WTFMove(displayName);
243
244                 record.addCookieHostName(hostName);
245             }
246
247 #if ENABLE(NETSCAPE_PLUGIN_API)
248             for (auto& hostName : websiteData.hostNamesWithPluginData) {
249                 auto displayName = WebsiteDataRecord::displayNameForPluginDataHostName(hostName);
250                 if (!displayName)
251                     continue;
252
253                 auto& record = m_websiteDataRecords.add(displayName, WebsiteDataRecord { }).iterator->value;
254                 if (!record.displayName)
255                     record.displayName = WTFMove(displayName);
256
257                 record.addPluginDataHostName(hostName);
258             }
259 #endif
260
261             for (auto& origin : websiteData.originsWithCredentials) {
262                 auto& record = m_websiteDataRecords.add(origin, WebsiteDataRecord { }).iterator->value;
263                 
264                 record.addOriginWithCredential(origin);
265             }
266
267             callIfNeeded();
268         }
269
270         void callIfNeeded()
271         {
272             if (pendingCallbacks)
273                 return;
274
275             Vector<WebsiteDataRecord> records;
276             records.reserveInitialCapacity(m_websiteDataRecords.size());
277             for (auto& record : m_websiteDataRecords.values())
278                 records.uncheckedAppend(WTFMove(record));
279
280             auto processRecords = [apply = WTFMove(apply), records = WTFMove(records)] () mutable {
281                 apply(WTFMove(records));
282             };
283
284             if (queue)
285                 queue->dispatch(WTFMove(processRecords));
286             else
287                 RunLoop::main().dispatch(WTFMove(processRecords));
288         }
289
290         const OptionSet<WebsiteDataFetchOption> fetchOptions;
291
292         unsigned pendingCallbacks = 0;
293         RefPtr<WorkQueue> queue;
294         Function<void(Vector<WebsiteDataRecord>)> apply;
295
296         HashMap<String, WebsiteDataRecord> m_websiteDataRecords;
297     };
298
299     RefPtr<CallbackAggregator> callbackAggregator = adoptRef(new CallbackAggregator(fetchOptions, WTFMove(queue), WTFMove(apply)));
300
301 #if ENABLE(VIDEO)
302     if (dataTypes.contains(WebsiteDataType::DiskCache)) {
303         callbackAggregator->addPendingCallback();
304         m_queue->dispatch([mediaCacheDirectory = m_configuration.mediaCacheDirectory.isolatedCopy(), callbackAggregator] {
305             // FIXME: Make HTMLMediaElement::originsInMediaCache return a collection of SecurityOriginDatas.
306             HashSet<RefPtr<WebCore::SecurityOrigin>> origins = WebCore::HTMLMediaElement::originsInMediaCache(mediaCacheDirectory);
307             WebsiteData websiteData;
308             
309             for (auto& origin : origins) {
310                 WebsiteData::Entry entry { WebCore::SecurityOriginData::fromSecurityOrigin(*origin), WebsiteDataType::DiskCache, 0 };
311                 websiteData.entries.append(WTFMove(entry));
312             }
313             
314             RunLoop::main().dispatch([callbackAggregator, origins = WTFMove(origins), websiteData = WTFMove(websiteData)]() mutable {
315                 callbackAggregator->removePendingCallback(WTFMove(websiteData));
316             });
317         });
318     }
319 #endif
320
321     auto networkProcessAccessType = computeNetworkProcessAccessTypeForDataFetch(dataTypes, !isPersistent());
322     if (networkProcessAccessType != ProcessAccessType::None) {
323         for (auto& processPool : processPools()) {
324             switch (networkProcessAccessType) {
325             case ProcessAccessType::OnlyIfLaunched:
326                 if (!processPool->networkProcess())
327                     continue;
328                 break;
329
330             case ProcessAccessType::Launch:
331                 processPool->ensureNetworkProcess(this);
332                 break;
333
334             case ProcessAccessType::None:
335                 ASSERT_NOT_REACHED();
336             }
337
338             callbackAggregator->addPendingCallback();
339             processPool->networkProcess()->fetchWebsiteData(m_sessionID, dataTypes, fetchOptions, [callbackAggregator, processPool](WebsiteData websiteData) {
340                 callbackAggregator->removePendingCallback(WTFMove(websiteData));
341             });
342         }
343     }
344
345     auto webProcessAccessType = computeWebProcessAccessTypeForDataFetch(dataTypes, !isPersistent());
346     if (webProcessAccessType != ProcessAccessType::None) {
347         for (auto& process : processes()) {
348             switch (webProcessAccessType) {
349             case ProcessAccessType::OnlyIfLaunched:
350                 if (!process->canSendMessage())
351                     continue;
352                 break;
353
354             case ProcessAccessType::Launch:
355                 // FIXME: Handle this.
356                 ASSERT_NOT_REACHED();
357                 break;
358
359             case ProcessAccessType::None:
360                 ASSERT_NOT_REACHED();
361             }
362
363             callbackAggregator->addPendingCallback();
364             process->fetchWebsiteData(m_sessionID, dataTypes, [callbackAggregator](WebsiteData websiteData) {
365                 callbackAggregator->removePendingCallback(WTFMove(websiteData));
366             });
367         }
368     }
369
370     if (dataTypes.contains(WebsiteDataType::SessionStorage) && m_storageManager) {
371         callbackAggregator->addPendingCallback();
372
373         m_storageManager->getSessionStorageOrigins([callbackAggregator](HashSet<WebCore::SecurityOriginData>&& origins) {
374             WebsiteData websiteData;
375
376             while (!origins.isEmpty())
377                 websiteData.entries.append(WebsiteData::Entry { origins.takeAny(), WebsiteDataType::SessionStorage, 0 });
378
379             callbackAggregator->removePendingCallback(WTFMove(websiteData));
380         });
381     }
382
383     if (dataTypes.contains(WebsiteDataType::LocalStorage) && m_storageManager) {
384         callbackAggregator->addPendingCallback();
385
386         m_storageManager->getLocalStorageOrigins([callbackAggregator](HashSet<WebCore::SecurityOriginData>&& origins) {
387             WebsiteData websiteData;
388
389             while (!origins.isEmpty())
390                 websiteData.entries.append(WebsiteData::Entry { origins.takeAny(), WebsiteDataType::LocalStorage, 0 });
391
392             callbackAggregator->removePendingCallback(WTFMove(websiteData));
393         });
394     }
395
396     if (dataTypes.contains(WebsiteDataType::OfflineWebApplicationCache) && isPersistent()) {
397         callbackAggregator->addPendingCallback();
398
399         m_queue->dispatch([fetchOptions, applicationCacheDirectory = m_configuration.applicationCacheDirectory.isolatedCopy(), applicationCacheFlatFileSubdirectoryName = m_configuration.applicationCacheFlatFileSubdirectoryName.isolatedCopy(), callbackAggregator] {
400             auto storage = WebCore::ApplicationCacheStorage::create(applicationCacheDirectory, applicationCacheFlatFileSubdirectoryName);
401
402             WebsiteData websiteData;
403
404             HashSet<RefPtr<WebCore::SecurityOrigin>> origins;
405             // FIXME: getOriginsWithCache should return a collection of SecurityOriginDatas.
406             storage->getOriginsWithCache(origins);
407
408             for (auto& origin : origins) {
409                 uint64_t size = fetchOptions.contains(WebsiteDataFetchOption::ComputeSizes) ? storage->diskUsageForOrigin(*origin) : 0;
410                 WebsiteData::Entry entry { WebCore::SecurityOriginData::fromSecurityOrigin(*origin), WebsiteDataType::OfflineWebApplicationCache, size };
411
412                 websiteData.entries.append(WTFMove(entry));
413             }
414
415             RunLoop::main().dispatch([callbackAggregator, origins = WTFMove(origins), websiteData = WTFMove(websiteData)]() mutable {
416                 callbackAggregator->removePendingCallback(WTFMove(websiteData));
417             });
418         });
419     }
420
421     if (dataTypes.contains(WebsiteDataType::WebSQLDatabases) && isPersistent()) {
422         callbackAggregator->addPendingCallback();
423
424         m_queue->dispatch([webSQLDatabaseDirectory = m_configuration.webSQLDatabaseDirectory.isolatedCopy(), callbackAggregator] {
425             auto origins = WebCore::DatabaseTracker::trackerWithDatabasePath(webSQLDatabaseDirectory)->origins();
426             RunLoop::main().dispatch([callbackAggregator, origins = WTFMove(origins)]() mutable {
427                 WebsiteData websiteData;
428                 for (auto& origin : origins)
429                     websiteData.entries.append(WebsiteData::Entry { WTFMove(origin), WebsiteDataType::WebSQLDatabases, 0 });
430                 callbackAggregator->removePendingCallback(WTFMove(websiteData));
431             });
432         });
433     }
434
435     if ((dataTypes.contains(WebsiteDataType::IndexedDBDatabases)
436 #if ENABLE(SERVICE_WORKER)
437         || dataTypes.contains(WebsiteDataType::ServiceWorkerRegistrations)
438 #endif
439         ) && isPersistent()) {
440         for (auto& processPool : processPools()) {
441             processPool->ensureStorageProcessAndWebsiteDataStore(this);
442
443             callbackAggregator->addPendingCallback();
444             processPool->storageProcess()->fetchWebsiteData(m_sessionID, dataTypes, [callbackAggregator, processPool](WebsiteData websiteData) {
445                 callbackAggregator->removePendingCallback(WTFMove(websiteData));
446             });
447         }
448     }
449
450     if (dataTypes.contains(WebsiteDataType::MediaKeys) && isPersistent()) {
451         callbackAggregator->addPendingCallback();
452
453         m_queue->dispatch([mediaKeysStorageDirectory = m_configuration.mediaKeysStorageDirectory.isolatedCopy(), callbackAggregator] {
454             auto origins = mediaKeyOrigins(mediaKeysStorageDirectory);
455
456             RunLoop::main().dispatch([callbackAggregator, origins = WTFMove(origins)]() mutable {
457                 WebsiteData websiteData;
458                 for (auto& origin : origins)
459                     websiteData.entries.append(WebsiteData::Entry { origin, WebsiteDataType::MediaKeys, 0 });
460
461                 callbackAggregator->removePendingCallback(WTFMove(websiteData));
462             });
463         });
464     }
465
466 #if ENABLE(NETSCAPE_PLUGIN_API)
467     if (dataTypes.contains(WebsiteDataType::PlugInData) && isPersistent()) {
468         class State {
469         public:
470             static void fetchData(Ref<CallbackAggregator>&& callbackAggregator, Vector<PluginModuleInfo>&& plugins)
471             {
472                 new State(WTFMove(callbackAggregator), WTFMove(plugins));
473             }
474
475         private:
476             State(Ref<CallbackAggregator>&& callbackAggregator, Vector<PluginModuleInfo>&& plugins)
477                 : m_callbackAggregator(WTFMove(callbackAggregator))
478                 , m_plugins(WTFMove(plugins))
479             {
480                 m_callbackAggregator->addPendingCallback();
481
482                 fetchWebsiteDataForNextPlugin();
483             }
484
485             ~State()
486             {
487                 ASSERT(m_plugins.isEmpty());
488             }
489
490             void fetchWebsiteDataForNextPlugin()
491             {
492                 if (m_plugins.isEmpty()) {
493                     WebsiteData websiteData;
494                     websiteData.hostNamesWithPluginData = WTFMove(m_hostNames);
495
496                     m_callbackAggregator->removePendingCallback(WTFMove(websiteData));
497
498                     delete this;
499                     return;
500                 }
501
502                 auto plugin = m_plugins.takeLast();
503                 PluginProcessManager::singleton().fetchWebsiteData(plugin, [this](Vector<String> hostNames) {
504                     for (auto& hostName : hostNames)
505                         m_hostNames.add(WTFMove(hostName));
506                     fetchWebsiteDataForNextPlugin();
507                 });
508             }
509
510             Ref<CallbackAggregator> m_callbackAggregator;
511             Vector<PluginModuleInfo> m_plugins;
512             HashSet<String> m_hostNames;
513         };
514
515         State::fetchData(*callbackAggregator, plugins());
516     }
517 #endif
518
519     callbackAggregator->callIfNeeded();
520 }
521
522 void WebsiteDataStore::fetchDataForTopPrivatelyControlledDomains(OptionSet<WebsiteDataType> dataTypes, OptionSet<WebsiteDataFetchOption> fetchOptions, const Vector<String>& topPrivatelyControlledDomains, Function<void(Vector<WebsiteDataRecord>&&, HashSet<String>&&)>&& completionHandler)
523 {
524     fetchDataAndApply(dataTypes, fetchOptions, m_queue.copyRef(), [topPrivatelyControlledDomains = crossThreadCopy(topPrivatelyControlledDomains), completionHandler = WTFMove(completionHandler)] (auto&& existingDataRecords) mutable {
525         ASSERT(!RunLoop::isMain());
526
527         Vector<WebsiteDataRecord> matchingDataRecords;
528         HashSet<String> domainsWithMatchingDataRecords;
529         for (auto&& dataRecord : existingDataRecords) {
530             for (auto& topPrivatelyControlledDomain : topPrivatelyControlledDomains) {
531                 if (dataRecord.matchesTopPrivatelyControlledDomain(topPrivatelyControlledDomain)) {
532                     matchingDataRecords.append(WTFMove(dataRecord));
533                     domainsWithMatchingDataRecords.add(topPrivatelyControlledDomain.isolatedCopy());
534                     break;
535                 }
536             }
537         }
538         RunLoop::main().dispatch([completionHandler = WTFMove(completionHandler), matchingDataRecords = WTFMove(matchingDataRecords), domainsWithMatchingDataRecords = WTFMove(domainsWithMatchingDataRecords)] () mutable {
539             completionHandler(WTFMove(matchingDataRecords), WTFMove(domainsWithMatchingDataRecords));
540         });
541     });
542 }
543     
544 void WebsiteDataStore::topPrivatelyControlledDomainsWithWebsiteData(OptionSet<WebsiteDataType> dataTypes, OptionSet<WebsiteDataFetchOption> fetchOptions, Function<void(HashSet<String>&&)>&& completionHandler)
545 {
546     fetchData(dataTypes, fetchOptions, [completionHandler = WTFMove(completionHandler), protectedThis = makeRef(*this)](auto&& existingDataRecords) {
547         HashSet<String> domainsWithDataRecords;
548         for (auto&& dataRecord : existingDataRecords) {
549             String domain = dataRecord.topPrivatelyControlledDomain();
550             if (domain.isEmpty())
551                 continue;
552             domainsWithDataRecords.add(WTFMove(domain));
553         }
554         completionHandler(WTFMove(domainsWithDataRecords));
555     });
556 }
557
558 static ProcessAccessType computeNetworkProcessAccessTypeForDataRemoval(OptionSet<WebsiteDataType> dataTypes, bool isNonPersistentStore)
559 {
560     ProcessAccessType processAccessType = ProcessAccessType::None;
561
562     if (dataTypes.contains(WebsiteDataType::Cookies)) {
563         if (isNonPersistentStore)
564             processAccessType = std::max(processAccessType, ProcessAccessType::OnlyIfLaunched);
565         else
566             processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
567     }
568
569     if (dataTypes.contains(WebsiteDataType::DiskCache) && !isNonPersistentStore)
570         processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
571
572     if (dataTypes.contains(WebsiteDataType::HSTSCache))
573         processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
574
575     if (dataTypes.contains(WebsiteDataType::Credentials))
576         processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
577
578     if (dataTypes.contains(WebsiteDataType::DOMCache))
579         processAccessType = std::max(processAccessType, ProcessAccessType::Launch);
580
581     return processAccessType;
582 }
583
584 static ProcessAccessType computeWebProcessAccessTypeForDataRemoval(OptionSet<WebsiteDataType> dataTypes, bool isNonPersistentStore)
585 {
586     UNUSED_PARAM(isNonPersistentStore);
587
588     ProcessAccessType processAccessType = ProcessAccessType::None;
589
590     if (dataTypes.contains(WebsiteDataType::MemoryCache))
591         processAccessType = std::max(processAccessType, ProcessAccessType::OnlyIfLaunched);
592
593     if (dataTypes.contains(WebsiteDataType::Credentials))
594         processAccessType = std::max(processAccessType, ProcessAccessType::OnlyIfLaunched);
595
596     return processAccessType;
597 }
598
599 void WebsiteDataStore::removeData(OptionSet<WebsiteDataType> dataTypes, std::chrono::system_clock::time_point modifiedSince, Function<void()>&& completionHandler)
600 {
601     struct CallbackAggregator : ThreadSafeRefCounted<CallbackAggregator> {
602         explicit CallbackAggregator(Function<void()>&& completionHandler)
603             : completionHandler(WTFMove(completionHandler))
604         {
605         }
606
607         void addPendingCallback()
608         {
609             pendingCallbacks++;
610         }
611
612         void removePendingCallback()
613         {
614             ASSERT(pendingCallbacks);
615             --pendingCallbacks;
616
617             callIfNeeded();
618         }
619
620         void callIfNeeded()
621         {
622             if (!pendingCallbacks)
623                 RunLoop::main().dispatch(WTFMove(completionHandler));
624         }
625
626         unsigned pendingCallbacks = 0;
627         Function<void()> completionHandler;
628     };
629
630     RefPtr<CallbackAggregator> callbackAggregator = adoptRef(new CallbackAggregator(WTFMove(completionHandler)));
631
632 #if ENABLE(VIDEO)
633     if (dataTypes.contains(WebsiteDataType::DiskCache)) {
634         callbackAggregator->addPendingCallback();
635         m_queue->dispatch([modifiedSince, mediaCacheDirectory = m_configuration.mediaCacheDirectory.isolatedCopy(), callbackAggregator] {
636             WebCore::HTMLMediaElement::clearMediaCache(mediaCacheDirectory, modifiedSince);
637             
638             WTF::RunLoop::main().dispatch([callbackAggregator] {
639                 callbackAggregator->removePendingCallback();
640             });
641         });
642     }
643 #endif
644
645     auto networkProcessAccessType = computeNetworkProcessAccessTypeForDataRemoval(dataTypes, !isPersistent());
646     if (networkProcessAccessType != ProcessAccessType::None) {
647         for (auto& processPool : processPools()) {
648             switch (networkProcessAccessType) {
649             case ProcessAccessType::OnlyIfLaunched:
650                 if (!processPool->networkProcess())
651                     continue;
652                 break;
653
654             case ProcessAccessType::Launch:
655                 processPool->ensureNetworkProcess(this);
656                 break;
657
658             case ProcessAccessType::None:
659                 ASSERT_NOT_REACHED();
660             }
661
662             callbackAggregator->addPendingCallback();
663             processPool->networkProcess()->deleteWebsiteData(m_sessionID, dataTypes, modifiedSince, [callbackAggregator, processPool] {
664                 callbackAggregator->removePendingCallback();
665             });
666         }
667     }
668
669     auto webProcessAccessType = computeWebProcessAccessTypeForDataRemoval(dataTypes, !isPersistent());
670     if (webProcessAccessType != ProcessAccessType::None) {
671         for (auto& process : processes()) {
672             switch (webProcessAccessType) {
673             case ProcessAccessType::OnlyIfLaunched:
674                 if (!process->canSendMessage())
675                     continue;
676                 break;
677
678             case ProcessAccessType::Launch:
679                 // FIXME: Handle this.
680                 ASSERT_NOT_REACHED();
681                 break;
682
683             case ProcessAccessType::None:
684                 ASSERT_NOT_REACHED();
685             }
686
687             callbackAggregator->addPendingCallback();
688             process->deleteWebsiteData(m_sessionID, dataTypes, modifiedSince, [callbackAggregator] {
689                 callbackAggregator->removePendingCallback();
690             });
691         }
692     }
693
694     if (dataTypes.contains(WebsiteDataType::SessionStorage) && m_storageManager) {
695         callbackAggregator->addPendingCallback();
696
697         m_storageManager->deleteSessionStorageOrigins([callbackAggregator] {
698             callbackAggregator->removePendingCallback();
699         });
700     }
701
702     if (dataTypes.contains(WebsiteDataType::LocalStorage) && m_storageManager) {
703         callbackAggregator->addPendingCallback();
704
705         m_storageManager->deleteLocalStorageOriginsModifiedSince(modifiedSince, [callbackAggregator] {
706             callbackAggregator->removePendingCallback();
707         });
708     }
709
710     if (dataTypes.contains(WebsiteDataType::OfflineWebApplicationCache) && isPersistent()) {
711         callbackAggregator->addPendingCallback();
712
713         m_queue->dispatch([applicationCacheDirectory = m_configuration.applicationCacheDirectory.isolatedCopy(), applicationCacheFlatFileSubdirectoryName = m_configuration.applicationCacheFlatFileSubdirectoryName.isolatedCopy(), callbackAggregator] {
714             auto storage = WebCore::ApplicationCacheStorage::create(applicationCacheDirectory, applicationCacheFlatFileSubdirectoryName);
715
716             storage->deleteAllCaches();
717
718             WTF::RunLoop::main().dispatch([callbackAggregator] {
719                 callbackAggregator->removePendingCallback();
720             });
721         });
722     }
723
724     if (dataTypes.contains(WebsiteDataType::WebSQLDatabases) && isPersistent()) {
725         callbackAggregator->addPendingCallback();
726
727         m_queue->dispatch([webSQLDatabaseDirectory = m_configuration.webSQLDatabaseDirectory.isolatedCopy(), callbackAggregator, modifiedSince] {
728             WebCore::DatabaseTracker::trackerWithDatabasePath(webSQLDatabaseDirectory)->deleteDatabasesModifiedSince(modifiedSince);
729
730             RunLoop::main().dispatch([callbackAggregator] {
731                 callbackAggregator->removePendingCallback();
732             });
733         });
734     }
735
736     if ((dataTypes.contains(WebsiteDataType::IndexedDBDatabases)
737 #if ENABLE(SERVICE_WORKER)
738         || dataTypes.contains(WebsiteDataType::ServiceWorkerRegistrations)
739 #endif
740         ) && isPersistent()) {
741         for (auto& processPool : processPools()) {
742             processPool->ensureStorageProcessAndWebsiteDataStore(this);
743
744             callbackAggregator->addPendingCallback();
745             processPool->storageProcess()->deleteWebsiteData(m_sessionID, dataTypes, modifiedSince, [callbackAggregator, processPool] {
746                 callbackAggregator->removePendingCallback();
747             });
748         }
749     }
750
751     if (dataTypes.contains(WebsiteDataType::MediaKeys) && isPersistent()) {
752         callbackAggregator->addPendingCallback();
753
754         m_queue->dispatch([mediaKeysStorageDirectory = m_configuration.mediaKeysStorageDirectory.isolatedCopy(), callbackAggregator, modifiedSince] {
755             removeMediaKeys(mediaKeysStorageDirectory, modifiedSince);
756
757             RunLoop::main().dispatch([callbackAggregator] {
758                 callbackAggregator->removePendingCallback();
759             });
760         });
761     }
762
763     if (dataTypes.contains(WebsiteDataType::SearchFieldRecentSearches) && isPersistent()) {
764         callbackAggregator->addPendingCallback();
765
766         m_queue->dispatch([modifiedSince, callbackAggregator] {
767             platformRemoveRecentSearches(modifiedSince);
768
769             RunLoop::main().dispatch([callbackAggregator] {
770                 callbackAggregator->removePendingCallback();
771             });
772         });
773     }
774
775 #if ENABLE(NETSCAPE_PLUGIN_API)
776     if (dataTypes.contains(WebsiteDataType::PlugInData) && isPersistent()) {
777         class State {
778         public:
779             static void deleteData(Ref<CallbackAggregator>&& callbackAggregator, Vector<PluginModuleInfo>&& plugins, std::chrono::system_clock::time_point modifiedSince)
780             {
781                 new State(WTFMove(callbackAggregator), WTFMove(plugins), modifiedSince);
782             }
783
784         private:
785             State(Ref<CallbackAggregator>&& callbackAggregator, Vector<PluginModuleInfo>&& plugins, std::chrono::system_clock::time_point modifiedSince)
786                 : m_callbackAggregator(WTFMove(callbackAggregator))
787                 , m_plugins(WTFMove(plugins))
788                 , m_modifiedSince(modifiedSince)
789             {
790                 m_callbackAggregator->addPendingCallback();
791
792                 deleteWebsiteDataForNextPlugin();
793             }
794
795             ~State()
796             {
797                 ASSERT(m_plugins.isEmpty());
798             }
799
800             void deleteWebsiteDataForNextPlugin()
801             {
802                 if (m_plugins.isEmpty()) {
803                     m_callbackAggregator->removePendingCallback();
804
805                     delete this;
806                     return;
807                 }
808
809                 auto plugin = m_plugins.takeLast();
810                 PluginProcessManager::singleton().deleteWebsiteData(plugin, m_modifiedSince, [this] {
811                     deleteWebsiteDataForNextPlugin();
812                 });
813             }
814
815             Ref<CallbackAggregator> m_callbackAggregator;
816             Vector<PluginModuleInfo> m_plugins;
817             std::chrono::system_clock::time_point m_modifiedSince;
818         };
819
820         State::deleteData(*callbackAggregator, plugins(), modifiedSince);
821     }
822 #endif
823
824     // FIXME <rdar://problem/33491222>; scheduleClearInMemoryAndPersistent does not have a completion handler,
825     // so the completion handler for this removeData() call can be called prematurely.
826     if (dataTypes.contains(WebsiteDataType::ResourceLoadStatistics) && m_resourceLoadStatistics) {
827         auto deletedTypesRaw = dataTypes.toRaw();
828         auto monitoredTypesRaw = WebResourceLoadStatisticsStore::monitoredDataTypes().toRaw();
829
830         // If we are deleting all of the data types that the resource load statistics store monitors
831         // we do not need to re-grandfather old data.
832         if ((monitoredTypesRaw & deletedTypesRaw) == monitoredTypesRaw)
833             m_resourceLoadStatistics->scheduleClearInMemoryAndPersistent(modifiedSince, WebResourceLoadStatisticsStore::ShouldGrandfather::No);
834         else
835             m_resourceLoadStatistics->scheduleClearInMemoryAndPersistent(modifiedSince, WebResourceLoadStatisticsStore::ShouldGrandfather::Yes);
836
837         clearResourceLoadStatisticsInWebProcesses();
838     }
839
840     // There's a chance that we don't have any pending callbacks. If so, we want to dispatch the completion handler right away.
841     callbackAggregator->callIfNeeded();
842 }
843
844 void WebsiteDataStore::removeData(OptionSet<WebsiteDataType> dataTypes, const Vector<WebsiteDataRecord>& dataRecords, Function<void()>&& completionHandler)
845 {
846     Vector<WebCore::SecurityOriginData> origins;
847
848     for (const auto& dataRecord : dataRecords) {
849         for (auto& origin : dataRecord.origins)
850             origins.append(origin);
851     }
852
853     struct CallbackAggregator : ThreadSafeRefCounted<CallbackAggregator> {
854         explicit CallbackAggregator(Function<void()>&& completionHandler)
855             : completionHandler(WTFMove(completionHandler))
856         {
857         }
858
859         void addPendingCallback()
860         {
861             pendingCallbacks++;
862         }
863
864         void removePendingCallback()
865         {
866             ASSERT(pendingCallbacks);
867             --pendingCallbacks;
868
869             callIfNeeded();
870         }
871
872         void callIfNeeded()
873         {
874             if (!pendingCallbacks)
875                 RunLoop::main().dispatch(WTFMove(completionHandler));
876         }
877
878         unsigned pendingCallbacks = 0;
879         Function<void()> completionHandler;
880     };
881
882     RefPtr<CallbackAggregator> callbackAggregator = adoptRef(new CallbackAggregator(WTFMove(completionHandler)));
883     
884     if (dataTypes.contains(WebsiteDataType::DiskCache)) {
885         HashSet<WebCore::SecurityOriginData> origins;
886         for (const auto& dataRecord : dataRecords) {
887             for (const auto& origin : dataRecord.origins)
888                 origins.add(origin);
889         }
890         
891 #if ENABLE(VIDEO)
892         callbackAggregator->addPendingCallback();
893         m_queue->dispatch([origins = WTFMove(origins), mediaCacheDirectory = m_configuration.mediaCacheDirectory.isolatedCopy(), callbackAggregator] {
894
895             // FIXME: Move SecurityOrigin::toRawString to SecurityOriginData and
896             // make HTMLMediaElement::clearMediaCacheForOrigins take SecurityOriginData.
897             HashSet<RefPtr<WebCore::SecurityOrigin>> securityOrigins;
898             for (auto& origin : origins)
899                 securityOrigins.add(origin.securityOrigin());
900             WebCore::HTMLMediaElement::clearMediaCacheForOrigins(mediaCacheDirectory, securityOrigins);
901             
902             WTF::RunLoop::main().dispatch([callbackAggregator] {
903                 callbackAggregator->removePendingCallback();
904             });
905         });
906 #endif
907     }
908     
909     auto networkProcessAccessType = computeNetworkProcessAccessTypeForDataRemoval(dataTypes, !isPersistent());
910     if (networkProcessAccessType != ProcessAccessType::None) {
911         for (auto& processPool : processPools()) {
912             switch (networkProcessAccessType) {
913             case ProcessAccessType::OnlyIfLaunched:
914                 if (!processPool->networkProcess())
915                     continue;
916                 break;
917
918             case ProcessAccessType::Launch:
919                 processPool->ensureNetworkProcess(this);
920                 break;
921
922             case ProcessAccessType::None:
923                 ASSERT_NOT_REACHED();
924             }
925
926             Vector<String> cookieHostNames;
927             for (const auto& dataRecord : dataRecords) {
928                 for (auto& hostName : dataRecord.cookieHostNames)
929                     cookieHostNames.append(hostName);
930             }
931
932             callbackAggregator->addPendingCallback();
933             processPool->networkProcess()->deleteWebsiteDataForOrigins(m_sessionID, dataTypes, origins, cookieHostNames, [callbackAggregator, processPool] {
934                 callbackAggregator->removePendingCallback();
935             });
936         }
937     }
938
939     auto webProcessAccessType = computeWebProcessAccessTypeForDataRemoval(dataTypes, !isPersistent());
940     if (webProcessAccessType != ProcessAccessType::None) {
941         for (auto& process : processes()) {
942             switch (webProcessAccessType) {
943             case ProcessAccessType::OnlyIfLaunched:
944                 if (!process->canSendMessage())
945                     continue;
946                 break;
947
948             case ProcessAccessType::Launch:
949                 // FIXME: Handle this.
950                 ASSERT_NOT_REACHED();
951                 break;
952
953             case ProcessAccessType::None:
954                 ASSERT_NOT_REACHED();
955             }
956
957             callbackAggregator->addPendingCallback();
958
959             process->deleteWebsiteDataForOrigins(m_sessionID, dataTypes, origins, [callbackAggregator] {
960                 callbackAggregator->removePendingCallback();
961             });
962         }
963     }
964
965     if (dataTypes.contains(WebsiteDataType::SessionStorage) && m_storageManager) {
966         callbackAggregator->addPendingCallback();
967
968         m_storageManager->deleteSessionStorageEntriesForOrigins(origins, [callbackAggregator] {
969             callbackAggregator->removePendingCallback();
970         });
971     }
972
973     if (dataTypes.contains(WebsiteDataType::LocalStorage) && m_storageManager) {
974         callbackAggregator->addPendingCallback();
975
976         m_storageManager->deleteLocalStorageEntriesForOrigins(origins, [callbackAggregator] {
977             callbackAggregator->removePendingCallback();
978         });
979     }
980
981     if (dataTypes.contains(WebsiteDataType::OfflineWebApplicationCache) && isPersistent()) {
982         HashSet<WebCore::SecurityOriginData> origins;
983         for (const auto& dataRecord : dataRecords) {
984             for (const auto& origin : dataRecord.origins)
985                 origins.add(origin);
986         }
987
988         callbackAggregator->addPendingCallback();
989         m_queue->dispatch([origins = WTFMove(origins), applicationCacheDirectory = m_configuration.applicationCacheDirectory.isolatedCopy(), applicationCacheFlatFileSubdirectoryName = m_configuration.applicationCacheFlatFileSubdirectoryName.isolatedCopy(), callbackAggregator] {
990             auto storage = WebCore::ApplicationCacheStorage::create(applicationCacheDirectory, applicationCacheFlatFileSubdirectoryName);
991
992             for (const auto& origin : origins)
993                 storage->deleteCacheForOrigin(origin.securityOrigin());
994
995             WTF::RunLoop::main().dispatch([callbackAggregator] {
996                 callbackAggregator->removePendingCallback();
997             });
998         });
999     }
1000
1001     if (dataTypes.contains(WebsiteDataType::WebSQLDatabases) && isPersistent()) {
1002         HashSet<WebCore::SecurityOriginData> origins;
1003         for (const auto& dataRecord : dataRecords) {
1004             for (const auto& origin : dataRecord.origins)
1005                 origins.add(origin);
1006         }
1007
1008         callbackAggregator->addPendingCallback();
1009         m_queue->dispatch([origins = WTFMove(origins), callbackAggregator, webSQLDatabaseDirectory = m_configuration.webSQLDatabaseDirectory.isolatedCopy()] {
1010             auto databaseTracker = WebCore::DatabaseTracker::trackerWithDatabasePath(webSQLDatabaseDirectory);
1011             for (auto& origin : origins)
1012                 databaseTracker->deleteOrigin(origin);
1013             RunLoop::main().dispatch([callbackAggregator] {
1014                 callbackAggregator->removePendingCallback();
1015             });
1016         });
1017     }
1018
1019     if ((dataTypes.contains(WebsiteDataType::IndexedDBDatabases)
1020 #if ENABLE(SERVICE_WORKER)
1021         || dataTypes.contains(WebsiteDataType::ServiceWorkerRegistrations)
1022 #endif
1023         ) && isPersistent()) {
1024         for (auto& processPool : processPools()) {
1025             processPool->ensureStorageProcessAndWebsiteDataStore(this);
1026
1027             callbackAggregator->addPendingCallback();
1028             processPool->storageProcess()->deleteWebsiteDataForOrigins(m_sessionID, dataTypes, origins, [callbackAggregator, processPool] {
1029                 callbackAggregator->removePendingCallback();
1030             });
1031         }
1032     }
1033
1034     if (dataTypes.contains(WebsiteDataType::MediaKeys) && isPersistent()) {
1035         HashSet<WebCore::SecurityOriginData> origins;
1036         for (const auto& dataRecord : dataRecords) {
1037             for (const auto& origin : dataRecord.origins)
1038                 origins.add(origin);
1039         }
1040
1041         callbackAggregator->addPendingCallback();
1042         m_queue->dispatch([mediaKeysStorageDirectory = m_configuration.mediaKeysStorageDirectory.isolatedCopy(), callbackAggregator, origins = WTFMove(origins)] {
1043
1044             removeMediaKeys(mediaKeysStorageDirectory, origins);
1045
1046             RunLoop::main().dispatch([callbackAggregator] {
1047                 callbackAggregator->removePendingCallback();
1048             });
1049         });
1050     }
1051
1052 #if ENABLE(NETSCAPE_PLUGIN_API)
1053     if (dataTypes.contains(WebsiteDataType::PlugInData) && isPersistent()) {
1054         Vector<String> hostNames;
1055         for (const auto& dataRecord : dataRecords) {
1056             for (const auto& hostName : dataRecord.pluginDataHostNames)
1057                 hostNames.append(hostName);
1058         }
1059
1060
1061         class State {
1062         public:
1063             static void deleteData(Ref<CallbackAggregator>&& callbackAggregator, Vector<PluginModuleInfo>&& plugins, Vector<String>&& hostNames)
1064             {
1065                 new State(WTFMove(callbackAggregator), WTFMove(plugins), WTFMove(hostNames));
1066             }
1067
1068         private:
1069             State(Ref<CallbackAggregator>&& callbackAggregator, Vector<PluginModuleInfo>&& plugins, Vector<String>&& hostNames)
1070                 : m_callbackAggregator(WTFMove(callbackAggregator))
1071                 , m_plugins(WTFMove(plugins))
1072                 , m_hostNames(WTFMove(hostNames))
1073             {
1074                 m_callbackAggregator->addPendingCallback();
1075
1076                 deleteWebsiteDataForNextPlugin();
1077             }
1078
1079             ~State()
1080             {
1081                 ASSERT(m_plugins.isEmpty());
1082             }
1083
1084             void deleteWebsiteDataForNextPlugin()
1085             {
1086                 if (m_plugins.isEmpty()) {
1087                     m_callbackAggregator->removePendingCallback();
1088
1089                     delete this;
1090                     return;
1091                 }
1092
1093                 auto plugin = m_plugins.takeLast();
1094                 PluginProcessManager::singleton().deleteWebsiteDataForHostNames(plugin, m_hostNames, [this] {
1095                     deleteWebsiteDataForNextPlugin();
1096                 });
1097             }
1098
1099             Ref<CallbackAggregator> m_callbackAggregator;
1100             Vector<PluginModuleInfo> m_plugins;
1101             Vector<String> m_hostNames;
1102         };
1103
1104         State::deleteData(*callbackAggregator, plugins(), WTFMove(hostNames));
1105     }
1106 #endif
1107
1108     // FIXME <rdar://problem/33491222>; scheduleClearInMemoryAndPersistent does not have a completion handler,
1109     // so the completion handler for this removeData() call can be called prematurely.
1110     if (dataTypes.contains(WebsiteDataType::ResourceLoadStatistics) && m_resourceLoadStatistics) {
1111         auto deletedTypesRaw = dataTypes.toRaw();
1112         auto monitoredTypesRaw = WebResourceLoadStatisticsStore::monitoredDataTypes().toRaw();
1113
1114         // If we are deleting all of the data types that the resource load statistics store monitors
1115         // we do not need to re-grandfather old data.
1116         if ((monitoredTypesRaw & deletedTypesRaw) == monitoredTypesRaw)
1117             m_resourceLoadStatistics->scheduleClearInMemoryAndPersistent(WebResourceLoadStatisticsStore::ShouldGrandfather::No);
1118         else
1119             m_resourceLoadStatistics->scheduleClearInMemoryAndPersistent(WebResourceLoadStatisticsStore::ShouldGrandfather::Yes);
1120
1121         clearResourceLoadStatisticsInWebProcesses();
1122     }
1123
1124     // There's a chance that we don't have any pending callbacks. If so, we want to dispatch the completion handler right away.
1125     callbackAggregator->callIfNeeded();
1126 }
1127
1128 void WebsiteDataStore::removeDataForTopPrivatelyControlledDomains(OptionSet<WebsiteDataType> dataTypes, OptionSet<WebsiteDataFetchOption> fetchOptions, const Vector<String>& topPrivatelyControlledDomains, Function<void(HashSet<String>&&)>&& completionHandler)
1129 {
1130     fetchDataForTopPrivatelyControlledDomains(dataTypes, fetchOptions, topPrivatelyControlledDomains, [dataTypes, completionHandler = WTFMove(completionHandler), this, protectedThis = makeRef(*this)](Vector<WebsiteDataRecord>&& websiteDataRecords, HashSet<String>&& domainsWithDataRecords) mutable {
1131         this->removeData(dataTypes, websiteDataRecords, [domainsWithDataRecords = WTFMove(domainsWithDataRecords), completionHandler = WTFMove(completionHandler)]() mutable {
1132             completionHandler(WTFMove(domainsWithDataRecords));
1133         });
1134     });
1135 }
1136
1137 #if HAVE(CFNETWORK_STORAGE_PARTITIONING)
1138 void WebsiteDataStore::updatePrevalentDomainsToPartitionOrBlockCookies(const Vector<String>& domainsToPartition, const Vector<String>& domainsToBlock, const Vector<String>& domainsToNeitherPartitionNorBlock, ShouldClearFirst shouldClearFirst)
1139 {
1140     for (auto& processPool : processPools())
1141         processPool->sendToNetworkingProcess(Messages::NetworkProcess::UpdatePrevalentDomainsToPartitionOrBlockCookies(m_sessionID, domainsToPartition, domainsToBlock, domainsToNeitherPartitionNorBlock, shouldClearFirst == ShouldClearFirst::Yes));
1142 }
1143
1144 void WebsiteDataStore::updateStorageAccessForPrevalentDomainsHandler(const String& resourceDomain, const String& firstPartyDomain, uint64_t frameID, uint64_t pageID, bool value, WTF::CompletionHandler<void(bool wasGranted)>&& callback)
1145 {
1146     for (auto& processPool : processPools())
1147         processPool->networkProcess()->updateStorageAccessForPrevalentDomains(m_sessionID, resourceDomain, firstPartyDomain, frameID, pageID, value, WTFMove(callback));
1148 }
1149
1150 void WebsiteDataStore::removePrevalentDomains(const Vector<String>& domains)
1151 {
1152     for (auto& processPool : processPools())
1153         processPool->sendToNetworkingProcess(Messages::NetworkProcess::RemovePrevalentDomains(m_sessionID, domains));
1154 }
1155 #endif
1156
1157 void WebsiteDataStore::networkProcessDidCrash()
1158 {
1159 #if HAVE(CFNETWORK_STORAGE_PARTITIONING)
1160     if (m_resourceLoadStatistics)
1161         m_resourceLoadStatistics->scheduleCookiePartitioningStateReset();
1162 #endif
1163 }
1164
1165 void WebsiteDataStore::webPageWasAdded(WebPageProxy& webPageProxy)
1166 {
1167     if (m_storageManager)
1168         m_storageManager->createSessionStorageNamespace(webPageProxy.pageID(), std::numeric_limits<unsigned>::max());
1169 }
1170
1171 void WebsiteDataStore::webPageWasRemoved(WebPageProxy& webPageProxy)
1172 {
1173     if (m_storageManager)
1174         m_storageManager->destroySessionStorageNamespace(webPageProxy.pageID());
1175 }
1176
1177 void WebsiteDataStore::webProcessWillOpenConnection(WebProcessProxy& webProcessProxy, IPC::Connection& connection)
1178 {
1179     if (m_storageManager)
1180         m_storageManager->processWillOpenConnection(webProcessProxy, connection);
1181
1182     if (m_resourceLoadStatistics)
1183         m_resourceLoadStatistics->processWillOpenConnection(webProcessProxy, connection);
1184 }
1185
1186 void WebsiteDataStore::webPageWillOpenConnection(WebPageProxy& webPageProxy, IPC::Connection& connection)
1187 {
1188     if (m_storageManager)
1189         m_storageManager->setAllowedSessionStorageNamespaceConnection(webPageProxy.pageID(), &connection);
1190 }
1191
1192 void WebsiteDataStore::webPageDidCloseConnection(WebPageProxy& webPageProxy, IPC::Connection&)
1193 {
1194     if (m_storageManager)
1195         m_storageManager->setAllowedSessionStorageNamespaceConnection(webPageProxy.pageID(), nullptr);
1196 }
1197
1198 void WebsiteDataStore::webProcessDidCloseConnection(WebProcessProxy& webProcessProxy, IPC::Connection& connection)
1199 {
1200     if (m_resourceLoadStatistics)
1201         m_resourceLoadStatistics->processDidCloseConnection(webProcessProxy, connection);
1202
1203     if (m_storageManager)
1204         m_storageManager->processDidCloseConnection(webProcessProxy, connection);
1205 }
1206
1207 bool WebsiteDataStore::isAssociatedProcessPool(WebProcessPool& processPool) const
1208 {
1209     if (auto* processPoolDataStore = processPool.websiteDataStore())
1210         return &processPoolDataStore->websiteDataStore() == this;
1211     return false;
1212 }
1213
1214 HashSet<RefPtr<WebProcessPool>> WebsiteDataStore::processPools(size_t count, bool ensureAPoolExists) const
1215 {
1216     HashSet<RefPtr<WebProcessPool>> processPools;
1217     for (auto& process : processes())
1218         processPools.add(&process->processPool());
1219
1220     if (processPools.isEmpty()) {
1221         // Check if we're one of the legacy data stores.
1222         for (auto& processPool : WebProcessPool::allProcessPools()) {
1223             if (!isAssociatedProcessPool(*processPool))
1224                 continue;
1225
1226             processPools.add(processPool);
1227
1228             if (processPools.size() == count)
1229                 break;
1230         }
1231     }
1232
1233     if (processPools.isEmpty() && count && ensureAPoolExists) {
1234         auto processPool = WebProcessPool::create(API::ProcessPoolConfiguration::createWithWebsiteDataStoreConfiguration(m_configuration));
1235         processPools.add(processPool.ptr());
1236     }
1237
1238     return processPools;
1239 }
1240
1241 #if ENABLE(NETSCAPE_PLUGIN_API)
1242 Vector<PluginModuleInfo> WebsiteDataStore::plugins() const
1243 {
1244     Vector<PluginModuleInfo> plugins;
1245
1246     for (auto& processPool : processPools()) {
1247         for (auto& plugin : processPool->pluginInfoStore().plugins())
1248             plugins.append(plugin);
1249     }
1250
1251     return plugins;
1252 }
1253 #endif
1254
1255 static String computeMediaKeyFile(const String& mediaKeyDirectory)
1256 {
1257     return WebCore::FileSystem::pathByAppendingComponent(mediaKeyDirectory, "SecureStop.plist");
1258 }
1259
1260 Vector<WebCore::SecurityOriginData> WebsiteDataStore::mediaKeyOrigins(const String& mediaKeysStorageDirectory)
1261 {
1262     ASSERT(!mediaKeysStorageDirectory.isEmpty());
1263
1264     Vector<WebCore::SecurityOriginData> origins;
1265
1266     for (const auto& originPath : WebCore::FileSystem::listDirectory(mediaKeysStorageDirectory, "*")) {
1267         auto mediaKeyFile = computeMediaKeyFile(originPath);
1268         if (!WebCore::FileSystem::fileExists(mediaKeyFile))
1269             continue;
1270
1271         auto mediaKeyIdentifier = WebCore::FileSystem::pathGetFileName(originPath);
1272
1273         if (auto securityOrigin = WebCore::SecurityOriginData::fromDatabaseIdentifier(mediaKeyIdentifier))
1274             origins.append(*securityOrigin);
1275     }
1276
1277     return origins;
1278 }
1279
1280 void WebsiteDataStore::removeMediaKeys(const String& mediaKeysStorageDirectory, std::chrono::system_clock::time_point modifiedSince)
1281 {
1282     ASSERT(!mediaKeysStorageDirectory.isEmpty());
1283
1284     for (const auto& mediaKeyDirectory : WebCore::FileSystem::listDirectory(mediaKeysStorageDirectory, "*")) {
1285         auto mediaKeyFile = computeMediaKeyFile(mediaKeyDirectory);
1286
1287         time_t modificationTime;
1288         if (!WebCore::FileSystem::getFileModificationTime(mediaKeyFile, modificationTime))
1289             continue;
1290
1291         if (std::chrono::system_clock::from_time_t(modificationTime) < modifiedSince)
1292             continue;
1293
1294         WebCore::FileSystem::deleteFile(mediaKeyFile);
1295         WebCore::FileSystem::deleteEmptyDirectory(mediaKeyDirectory);
1296     }
1297 }
1298
1299 void WebsiteDataStore::removeMediaKeys(const String& mediaKeysStorageDirectory, const HashSet<WebCore::SecurityOriginData>& origins)
1300 {
1301     ASSERT(!mediaKeysStorageDirectory.isEmpty());
1302
1303     for (const auto& origin : origins) {
1304         auto mediaKeyDirectory = WebCore::FileSystem::pathByAppendingComponent(mediaKeysStorageDirectory, origin.databaseIdentifier());
1305         auto mediaKeyFile = computeMediaKeyFile(mediaKeyDirectory);
1306
1307         WebCore::FileSystem::deleteFile(mediaKeyFile);
1308         WebCore::FileSystem::deleteEmptyDirectory(mediaKeyDirectory);
1309     }
1310 }
1311
1312 bool WebsiteDataStore::resourceLoadStatisticsEnabled() const
1313 {
1314     return !!m_resourceLoadStatistics;
1315 }
1316
1317 void WebsiteDataStore::setResourceLoadStatisticsEnabled(bool enabled)
1318 {
1319     if (enabled == resourceLoadStatisticsEnabled())
1320         return;
1321
1322     if (enabled) {
1323         ASSERT(!m_resourceLoadStatistics);
1324         enableResourceLoadStatisticsAndSetTestingCallback(nullptr);
1325         return;
1326     }
1327
1328     m_resourceLoadStatistics = nullptr;
1329
1330     auto existingProcessPools = processPools(std::numeric_limits<size_t>::max(), false);
1331     for (auto& processPool : existingProcessPools)
1332         processPool->setResourceLoadStatisticsEnabled(false);
1333 }
1334
1335 void WebsiteDataStore::enableResourceLoadStatisticsAndSetTestingCallback(Function<void (const String&)>&& callback)
1336 {
1337     if (m_resourceLoadStatistics) {
1338         m_resourceLoadStatistics->setStatisticsTestingCallback(WTFMove(callback));
1339         return;
1340     }
1341
1342 #if HAVE(CFNETWORK_STORAGE_PARTITIONING)
1343     m_resourceLoadStatistics = WebResourceLoadStatisticsStore::create(m_configuration.resourceLoadStatisticsDirectory, WTFMove(callback), [this] (const Vector<String>& domainsToPartition, const Vector<String>& domainsToBlock, const Vector<String>& domainsToNeitherPartitionNorBlock, ShouldClearFirst shouldClearFirst) {
1344         updatePrevalentDomainsToPartitionOrBlockCookies(domainsToPartition, domainsToBlock, domainsToNeitherPartitionNorBlock, shouldClearFirst);
1345     }, [this, protectedThis = makeRef(*this)] (const String& resourceDomain, const String& firstPartyDomain, uint64_t frameID, uint64_t pageID, bool value, WTF::Function<void(bool wasGranted)>&& callback) {
1346         updateStorageAccessForPrevalentDomainsHandler(resourceDomain, firstPartyDomain, frameID, pageID, value, WTFMove(callback));
1347     }, [this, protectedThis = makeRef(*this)] (const Vector<String>& domainsToRemove) {
1348         removePrevalentDomains(domainsToRemove);
1349     });
1350 #else
1351     m_resourceLoadStatistics = WebResourceLoadStatisticsStore::create(m_configuration.resourceLoadStatisticsDirectory, WTFMove(callback));
1352 #endif
1353
1354     for (auto& processPool : processPools())
1355         processPool->setResourceLoadStatisticsEnabled(true);
1356 }
1357
1358 void WebsiteDataStore::clearResourceLoadStatisticsInWebProcesses()
1359 {
1360     if (!resourceLoadStatisticsEnabled())
1361         return;
1362
1363     for (auto& processPool : processPools())
1364         processPool->clearResourceLoadStatistics();
1365 }
1366
1367 StorageProcessCreationParameters WebsiteDataStore::storageProcessParameters()
1368 {
1369     resolveDirectoriesIfNecessary();
1370
1371     StorageProcessCreationParameters parameters;
1372
1373     parameters.sessionID = m_sessionID;
1374
1375 #if ENABLE(INDEXED_DATABASE)
1376     parameters.indexedDatabaseDirectory = resolvedIndexedDatabaseDirectory();
1377     if (!parameters.indexedDatabaseDirectory.isEmpty())
1378         SandboxExtension::createHandleForReadWriteDirectory(parameters.indexedDatabaseDirectory, parameters.indexedDatabaseDirectoryExtensionHandle);
1379 #endif
1380 #if ENABLE(SERVICE_WORKER)
1381     parameters.serviceWorkerRegistrationDirectory = resolvedServiceWorkerRegistrationDirectory();
1382     if (!parameters.serviceWorkerRegistrationDirectory.isEmpty())
1383         SandboxExtension::createHandleForReadWriteDirectory(parameters.serviceWorkerRegistrationDirectory, parameters.serviceWorkerRegistrationDirectoryExtensionHandle);
1384 #endif
1385
1386     return parameters;
1387 }
1388
1389 Vector<WebCore::Cookie> WebsiteDataStore::pendingCookies() const
1390 {
1391     return copyToVector(m_pendingCookies);
1392 }
1393
1394 void WebsiteDataStore::addPendingCookie(const WebCore::Cookie& cookie)
1395 {
1396     m_pendingCookies.add(cookie);
1397 }
1398
1399 void WebsiteDataStore::removePendingCookie(const WebCore::Cookie& cookie)
1400 {
1401     m_pendingCookies.remove(cookie);
1402 }
1403
1404 void WebsiteDataStore::hasStorageAccess(String&& subFrameHost, String&& topFrameHost, WTF::CompletionHandler<void (bool)>&& callback)
1405 {
1406     if (!resourceLoadStatisticsEnabled()) {
1407         callback(false);
1408         return;
1409     }
1410     
1411     m_resourceLoadStatistics->hasStorageAccess(WTFMove(subFrameHost), WTFMove(topFrameHost), WTFMove(callback));
1412 }
1413     
1414 void WebsiteDataStore::requestStorageAccess(String&& subFrameHost, String&& topFrameHost, uint64_t frameID, uint64_t pageID, WTF::CompletionHandler<void (bool)>&& callback)
1415 {
1416     if (!resourceLoadStatisticsEnabled()) {
1417         callback(false);
1418         return;
1419     }
1420
1421     m_resourceLoadStatistics->requestStorageAccess(WTFMove(subFrameHost), WTFMove(topFrameHost), frameID, pageID, WTFMove(callback));
1422 }
1423
1424 #if !PLATFORM(COCOA)
1425 WebsiteDataStoreParameters WebsiteDataStore::parameters()
1426 {
1427     // FIXME: Implement cookies.
1428     WebsiteDataStoreParameters parameters;
1429     parameters.networkSessionParameters.sessionID = m_sessionID;
1430     return parameters;
1431 }
1432 #endif
1433
1434 }