Resource Load Statistics: Classify resources as prevalent based on redirects to other...
[WebKit-https.git] / Source / WebKit / UIProcess / WebResourceLoadStatisticsStore.cpp
1 /*
2  * Copyright (C) 2016-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WebResourceLoadStatisticsStore.h"
28
29 #include "Logging.h"
30 #include "PluginProcessManager.h"
31 #include "PluginProcessProxy.h"
32 #include "WebProcessMessages.h"
33 #include "WebProcessProxy.h"
34 #include "WebResourceLoadStatisticsStoreMessages.h"
35 #include "WebResourceLoadStatisticsTelemetry.h"
36 #include "WebsiteDataFetchOption.h"
37 #include "WebsiteDataStore.h"
38 #include <WebCore/KeyedCoding.h>
39 #include <WebCore/ResourceLoadStatistics.h>
40 #include <wtf/CrossThreadCopier.h>
41 #include <wtf/DateMath.h>
42 #include <wtf/MathExtras.h>
43 #include <wtf/NeverDestroyed.h>
44
45 namespace WebKit {
46 using namespace WebCore;
47
48 constexpr unsigned operatingDatesWindow { 30 };
49 constexpr unsigned statisticsModelVersion { 11 };
50 constexpr unsigned maxImportance { 3 };
51 constexpr unsigned maxNumberOfRecursiveCallsInRedirectTraceBack { 50 };
52
53 template<typename T> static inline String isolatedPrimaryDomain(const T& value)
54 {
55     return ResourceLoadStatistics::primaryDomain(value).isolatedCopy();
56 }
57
58 const OptionSet<WebsiteDataType>& WebResourceLoadStatisticsStore::monitoredDataTypes()
59 {
60     static NeverDestroyed<OptionSet<WebsiteDataType>> dataTypes(std::initializer_list<WebsiteDataType>({
61         WebsiteDataType::Cookies,
62         WebsiteDataType::DOMCache,
63         WebsiteDataType::IndexedDBDatabases,
64         WebsiteDataType::LocalStorage,
65         WebsiteDataType::MediaKeys,
66         WebsiteDataType::OfflineWebApplicationCache,
67 #if ENABLE(NETSCAPE_PLUGIN_API)
68         WebsiteDataType::PlugInData,
69 #endif
70         WebsiteDataType::SearchFieldRecentSearches,
71         WebsiteDataType::SessionStorage,
72 #if ENABLE(SERVICE_WORKER)
73         WebsiteDataType::ServiceWorkerRegistrations,
74 #endif
75         WebsiteDataType::WebSQLDatabases,
76     }));
77
78     ASSERT(RunLoop::isMain());
79
80     return dataTypes;
81 }
82
83 class OperatingDate {
84 public:
85     OperatingDate() = default;
86
87     static OperatingDate fromWallTime(WallTime time)
88     {
89         double ms = time.secondsSinceEpoch().milliseconds();
90         int year = msToYear(ms);
91         int yearDay = dayInYear(ms, year);
92         int month = monthFromDayInYear(yearDay, isLeapYear(year));
93         int monthDay = dayInMonthFromDayInYear(yearDay, isLeapYear(year));
94
95         return OperatingDate { year, month, monthDay };
96     }
97
98     static OperatingDate today()
99     {
100         return OperatingDate::fromWallTime(WallTime::now());
101     }
102
103     Seconds secondsSinceEpoch() const
104     {
105         return Seconds { dateToDaysFrom1970(m_year, m_month, m_monthDay) * secondsPerDay };
106     }
107
108     bool operator==(const OperatingDate& other) const
109     {
110         return m_monthDay == other.m_monthDay && m_month == other.m_month && m_year == other.m_year;
111     }
112
113     bool operator<(const OperatingDate& other) const
114     {
115         return secondsSinceEpoch() < other.secondsSinceEpoch();
116     }
117
118     bool operator<=(const OperatingDate& other) const
119     {
120         return secondsSinceEpoch() <= other.secondsSinceEpoch();
121     }
122
123 private:
124     OperatingDate(int year, int month, int monthDay)
125         : m_year(year)
126         , m_month(month)
127         , m_monthDay(monthDay)
128     { }
129
130     int m_year { 0 };
131     int m_month { 0 }; // [0, 11].
132     int m_monthDay { 0 }; // [1, 31].
133 };
134
135 static Vector<OperatingDate> mergeOperatingDates(const Vector<OperatingDate>& existingDates, Vector<OperatingDate>&& newDates)
136 {
137     if (existingDates.isEmpty())
138         return WTFMove(newDates);
139
140     Vector<OperatingDate> mergedDates(existingDates.size() + newDates.size());
141
142     // Merge the two sorted vectors of dates.
143     std::merge(existingDates.begin(), existingDates.end(), newDates.begin(), newDates.end(), mergedDates.begin());
144     // Remove duplicate dates.
145     removeRepeatedElements(mergedDates);
146
147     // Drop old dates until the Vector size reaches operatingDatesWindow.
148     while (mergedDates.size() > operatingDatesWindow)
149         mergedDates.remove(0);
150
151     return mergedDates;
152 }
153
154 WebResourceLoadStatisticsStore::WebResourceLoadStatisticsStore(const String& resourceLoadStatisticsDirectory, Function<void(const String&)>&& testingCallback, bool isEphemeral, UpdatePrevalentDomainsToPartitionOrBlockCookiesHandler&& updatePrevalentDomainsToPartitionOrBlockCookiesHandler, HasStorageAccessForFrameHandler&& hasStorageAccessForFrameHandler, GrantStorageAccessForFrameHandler&& grantStorageAccessForFrameHandler, RemovePrevalentDomainsHandler&& removeDomainsHandler)
155     : m_statisticsQueue(WorkQueue::create("WebResourceLoadStatisticsStore Process Data Queue", WorkQueue::Type::Serial, WorkQueue::QOS::Utility))
156     , m_persistentStorage(*this, resourceLoadStatisticsDirectory, isEphemeral ? ResourceLoadStatisticsPersistentStorage::IsReadOnly::Yes : ResourceLoadStatisticsPersistentStorage::IsReadOnly::No)
157     , m_updatePrevalentDomainsToPartitionOrBlockCookiesHandler(WTFMove(updatePrevalentDomainsToPartitionOrBlockCookiesHandler))
158     , m_hasStorageAccessForFrameHandler(WTFMove(hasStorageAccessForFrameHandler))
159     , m_grantStorageAccessForFrameHandler(WTFMove(grantStorageAccessForFrameHandler))
160     , m_removeDomainsHandler(WTFMove(removeDomainsHandler))
161     , m_dailyTasksTimer(RunLoop::main(), this, &WebResourceLoadStatisticsStore::performDailyTasks)
162     , m_statisticsTestingCallback(WTFMove(testingCallback))
163 {
164     ASSERT(RunLoop::isMain());
165
166 #if PLATFORM(COCOA)
167     registerUserDefaultsIfNeeded();
168 #endif
169
170     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
171         m_persistentStorage.initialize();
172         includeTodayAsOperatingDateIfNecessary();
173     });
174
175     m_statisticsQueue->dispatchAfter(5_s, [this, protectedThis = makeRef(*this)] {
176         if (m_parameters.shouldSubmitTelemetry)
177             WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
178     });
179
180     m_dailyTasksTimer.startRepeating(24_h);
181 }
182
183 WebResourceLoadStatisticsStore::~WebResourceLoadStatisticsStore()
184 {
185     m_persistentStorage.finishAllPendingWorkSynchronously();
186 }
187     
188 void WebResourceLoadStatisticsStore::removeDataRecords()
189 {
190     ASSERT(!RunLoop::isMain());
191     
192     if (!shouldRemoveDataRecords())
193         return;
194
195 #if ENABLE(NETSCAPE_PLUGIN_API)
196     m_activePluginTokens.clear();
197     for (auto plugin : PluginProcessManager::singleton().pluginProcesses())
198         m_activePluginTokens.add(plugin->pluginProcessToken());
199 #endif
200
201     auto prevalentResourceDomains = topPrivatelyControlledDomainsToRemoveWebsiteDataFor();
202     if (prevalentResourceDomains.isEmpty())
203         return;
204     
205     setDataRecordsBeingRemoved(true);
206
207     RunLoop::main().dispatch([prevalentResourceDomains = crossThreadCopy(prevalentResourceDomains), this, protectedThis = makeRef(*this)] () mutable {
208         WebProcessProxy::deleteWebsiteDataForTopPrivatelyControlledDomainsInAllPersistentDataStores(WebResourceLoadStatisticsStore::monitoredDataTypes(), WTFMove(prevalentResourceDomains), m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, [this, protectedThis = WTFMove(protectedThis)](const HashSet<String>& domainsWithDeletedWebsiteData) mutable {
209             m_statisticsQueue->dispatch([this, protectedThis = WTFMove(protectedThis), topDomains = crossThreadCopy(domainsWithDeletedWebsiteData)] () mutable {
210                 for (auto& prevalentResourceDomain : topDomains) {
211                     auto& statistic = ensureResourceStatisticsForPrimaryDomain(prevalentResourceDomain);
212                     ++statistic.dataRecordsRemoved;
213                 }
214                 setDataRecordsBeingRemoved(false);
215             });
216         });
217     });
218 }
219
220 void WebResourceLoadStatisticsStore::scheduleStatisticsAndDataRecordsProcessing()
221 {
222     ASSERT(RunLoop::isMain());
223     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
224         processStatisticsAndDataRecords();
225     });
226 }
227
228
229 unsigned WebResourceLoadStatisticsStore::recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(const WebCore::ResourceLoadStatistics& resourceStatistic, HashSet<String>& domainsThatHaveRedirectedTo, unsigned numberOfRecursiveCalls)
230 {
231     if (numberOfRecursiveCalls >= maxNumberOfRecursiveCallsInRedirectTraceBack) {
232         ASSERT_NOT_REACHED();
233         WTFLogAlways("Hit %u recursive calls in redirect backtrace. Returning early.", maxNumberOfRecursiveCallsInRedirectTraceBack);
234         return numberOfRecursiveCalls;
235     }
236
237     numberOfRecursiveCalls++;
238     
239     for (auto& subresourceUniqueRedirectFromDomain : resourceStatistic.subresourceUniqueRedirectsFrom.values()) {
240         auto mapEntry = m_resourceStatisticsMap.find(subresourceUniqueRedirectFromDomain);
241         if (mapEntry == m_resourceStatisticsMap.end() || mapEntry->value.isPrevalentResource)
242             continue;
243         if (domainsThatHaveRedirectedTo.add(mapEntry->value.highLevelDomain).isNewEntry)
244             numberOfRecursiveCalls = recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(mapEntry->value, domainsThatHaveRedirectedTo, numberOfRecursiveCalls);
245     }
246     for (auto& topFrameUniqueRedirectFromDomain : resourceStatistic.topFrameUniqueRedirectsFrom.values()) {
247         auto mapEntry = m_resourceStatisticsMap.find(topFrameUniqueRedirectFromDomain);
248         if (mapEntry == m_resourceStatisticsMap.end() || mapEntry->value.isPrevalentResource)
249             continue;
250         if (domainsThatHaveRedirectedTo.add(mapEntry->value.highLevelDomain).isNewEntry)
251             numberOfRecursiveCalls = recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(mapEntry->value, domainsThatHaveRedirectedTo, numberOfRecursiveCalls);
252     }
253     
254     return numberOfRecursiveCalls;
255 }
256
257 void WebResourceLoadStatisticsStore::processStatisticsAndDataRecords()
258 {
259     ASSERT(!RunLoop::isMain());
260
261     if (m_parameters.shouldClassifyResourcesBeforeDataRecordsRemoval) {
262         for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
263             if (!resourceStatistic.isPrevalentResource && m_resourceLoadStatisticsClassifier.hasPrevalentResourceCharacteristics(resourceStatistic))
264                 setPrevalentResource(resourceStatistic);
265         }
266     }
267     removeDataRecords();
268
269     pruneStatisticsIfNeeded();
270
271     if (m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned) {
272         RunLoop::main().dispatch([] {
273             WebProcessProxy::notifyPageStatisticsAndDataRecordsProcessed();
274         });
275     }
276
277     m_persistentStorage.scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::No);
278 }
279
280 void WebResourceLoadStatisticsStore::resourceLoadStatisticsUpdated(Vector<WebCore::ResourceLoadStatistics>&& origins)
281 {
282     ASSERT(!RunLoop::isMain());
283
284     mergeStatistics(WTFMove(origins));
285     // Fire before processing statistics to propagate user interaction as fast as possible to the network process.
286     updateCookiePartitioning([]() { });
287     processStatisticsAndDataRecords();
288 }
289
290 void WebResourceLoadStatisticsStore::hasStorageAccess(String&& subFrameHost, String&& topFrameHost, uint64_t frameID, uint64_t pageID, WTF::CompletionHandler<void (bool)>&& callback)
291 {
292     ASSERT(subFrameHost != topFrameHost);
293     ASSERT(RunLoop::isMain());
294
295     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), subFramePrimaryDomain = isolatedPrimaryDomain(subFrameHost), topFramePrimaryDomain = isolatedPrimaryDomain(topFrameHost), frameID, pageID, callback = WTFMove(callback)] () mutable {
296         
297         auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
298         if (shouldBlockCookies(subFrameStatistic)) {
299             callback(false);
300             return;
301         }
302
303         if (!shouldPartitionCookies(subFrameStatistic)) {
304             callback(true);
305             return;
306         }
307
308         m_hasStorageAccessForFrameHandler(subFramePrimaryDomain, topFramePrimaryDomain, frameID, pageID, WTFMove(callback));
309     });
310 }
311
312 void WebResourceLoadStatisticsStore::requestStorageAccess(String&& subFrameHost, String&& topFrameHost, uint64_t frameID, uint64_t pageID, WTF::CompletionHandler<void (bool)>&& callback)
313 {
314     ASSERT(subFrameHost != topFrameHost);
315     ASSERT(RunLoop::isMain());
316
317     auto subFramePrimaryDomain = isolatedPrimaryDomain(subFrameHost);
318     auto topFramePrimaryDomain = isolatedPrimaryDomain(topFrameHost);
319     if (subFramePrimaryDomain == topFramePrimaryDomain) {
320         callback(true);
321         return;
322     }
323
324     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), subFramePrimaryDomain = crossThreadCopy(subFramePrimaryDomain), topFramePrimaryDomain = crossThreadCopy(topFramePrimaryDomain), frameID, pageID, callback = WTFMove(callback)] () mutable {
325
326         auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
327         if (shouldBlockCookies(subFrameStatistic)) {
328             callback(false);
329             return;
330         }
331         
332         if (!shouldPartitionCookies(subFrameStatistic)) {
333             callback(true);
334             return;
335         }
336
337         subFrameStatistic.timesAccessedAsFirstPartyDueToStorageAccessAPI++;
338
339         m_grantStorageAccessForFrameHandler(subFramePrimaryDomain, topFramePrimaryDomain, frameID, pageID, WTFMove(callback));
340     });
341 }
342     
343 void WebResourceLoadStatisticsStore::grandfatherExistingWebsiteData(CompletionHandler<void()>&& callback)
344 {
345     ASSERT(!RunLoop::isMain());
346
347     RunLoop::main().dispatch([this, protectedThis = makeRef(*this), callback = WTFMove(callback)] () mutable {
348         // FIXME: This method being a static call on WebProcessProxy is wrong.
349         // It should be on the data store that this object belongs to.
350         WebProcessProxy::topPrivatelyControlledDomainsWithWebsiteData(WebResourceLoadStatisticsStore::monitoredDataTypes(), m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, [this, protectedThis = WTFMove(protectedThis), callback = WTFMove(callback)] (HashSet<String>&& topPrivatelyControlledDomainsWithWebsiteData) mutable {
351             m_statisticsQueue->dispatch([this, protectedThis = WTFMove(protectedThis), topDomains = crossThreadCopy(topPrivatelyControlledDomainsWithWebsiteData), callback = WTFMove(callback)] () mutable {
352                 for (auto& topPrivatelyControlledDomain : topDomains) {
353                     auto& statistic = ensureResourceStatisticsForPrimaryDomain(topPrivatelyControlledDomain);
354                     statistic.grandfathered = true;
355                 }
356                 m_endOfGrandfatheringTimestamp = WallTime::now() + m_parameters.grandfatheringTime;
357                 m_persistentStorage.scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::Yes);
358                 callback();
359                 logTestingEvent(ASCIILiteral("Grandfathered"));
360             });
361         });
362     });
363 }
364     
365 void WebResourceLoadStatisticsStore::processWillOpenConnection(WebProcessProxy&, IPC::Connection& connection)
366 {
367     connection.addWorkQueueMessageReceiver(Messages::WebResourceLoadStatisticsStore::messageReceiverName(), m_statisticsQueue.get(), this);
368 }
369
370 void WebResourceLoadStatisticsStore::processDidCloseConnection(WebProcessProxy&, IPC::Connection& connection)
371 {
372     connection.removeWorkQueueMessageReceiver(Messages::WebResourceLoadStatisticsStore::messageReceiverName());
373 }
374
375 void WebResourceLoadStatisticsStore::applicationWillTerminate()
376 {
377     m_persistentStorage.finishAllPendingWorkSynchronously();
378 }
379
380 void WebResourceLoadStatisticsStore::performDailyTasks()
381 {
382     ASSERT(RunLoop::isMain());
383
384     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
385         includeTodayAsOperatingDateIfNecessary();
386     });
387     if (m_parameters.shouldSubmitTelemetry)
388         submitTelemetry();
389 }
390
391 void WebResourceLoadStatisticsStore::submitTelemetry()
392 {
393     ASSERT(RunLoop::isMain());
394     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
395         WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
396     });
397 }
398
399 void WebResourceLoadStatisticsStore::setResourceLoadStatisticsDebugMode(bool enable)
400 {
401     if (enable)
402         setTimeToLiveCookiePartitionFree(30_s);
403     else
404         resetParametersToDefaultValues();
405 }
406
407 void WebResourceLoadStatisticsStore::logUserInteraction(const URL& url)
408 {
409     if (url.isBlankURL() || url.isEmpty())
410         return;
411
412     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
413         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
414         statistics.hadUserInteraction = true;
415         statistics.mostRecentUserInteractionTime = WallTime::now();
416
417         if (statistics.isMarkedForCookiePartitioning || statistics.isMarkedForCookieBlocking)
418             updateCookiePartitioningForDomains({ }, { }, { primaryDomain }, ShouldClearFirst::No, []() { });
419     });
420 }
421
422 void WebResourceLoadStatisticsStore::logNonRecentUserInteraction(const URL& url)
423 {
424     if (url.isBlankURL() || url.isEmpty())
425         return;
426     
427     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
428         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
429         statistics.hadUserInteraction = true;
430         statistics.mostRecentUserInteractionTime = WallTime::now() - (m_parameters.timeToLiveCookiePartitionFree + Seconds::fromHours(1));
431
432         updateCookiePartitioningForDomains({ primaryDomain }, { }, { }, ShouldClearFirst::No, []() { });
433     });
434 }
435
436 void WebResourceLoadStatisticsStore::clearUserInteraction(const URL& url)
437 {
438     if (url.isBlankURL() || url.isEmpty())
439         return;
440
441     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
442         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
443         statistics.hadUserInteraction = false;
444         statistics.mostRecentUserInteractionTime = { };
445     });
446 }
447
448 void WebResourceLoadStatisticsStore::hasHadUserInteraction(const URL& url, WTF::Function<void (bool)>&& completionHandler)
449 {
450     if (url.isBlankURL() || url.isEmpty()) {
451         completionHandler(false);
452         return;
453     }
454
455     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), completionHandler = WTFMove(completionHandler)] () mutable {
456         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
457         bool hadUserInteraction = mapEntry == m_resourceStatisticsMap.end() ? false: hasHadUnexpiredRecentUserInteraction(mapEntry->value);
458         RunLoop::main().dispatch([hadUserInteraction, completionHandler = WTFMove(completionHandler)] {
459             completionHandler(hadUserInteraction);
460         });
461     });
462 }
463
464 void WebResourceLoadStatisticsStore::setLastSeen(const URL& url, Seconds seconds)
465 {
466     if (url.isBlankURL() || url.isEmpty())
467         return;
468     
469     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), seconds] {
470         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
471         statistics.lastSeen = WallTime::fromRawSeconds(seconds.seconds());
472     });
473 }
474     
475 void WebResourceLoadStatisticsStore::setPrevalentResource(const URL& url)
476 {
477     if (url.isBlankURL() || url.isEmpty())
478         return;
479
480     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
481         auto& resourceStatistic = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
482         setPrevalentResource(resourceStatistic);
483     });
484 }
485
486 void WebResourceLoadStatisticsStore::setPrevalentResource(WebCore::ResourceLoadStatistics& resourceStatistic)
487 {
488     ASSERT(!RunLoop::isMain());
489     resourceStatistic.isPrevalentResource = true;
490     HashSet<String> domainsThatHaveRedirectedTo;
491     recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(resourceStatistic, domainsThatHaveRedirectedTo, 0);
492     for (auto& domain : domainsThatHaveRedirectedTo) {
493         auto mapEntry = m_resourceStatisticsMap.find(domain);
494         if (mapEntry == m_resourceStatisticsMap.end())
495             continue;
496         ASSERT(!mapEntry->value.isPrevalentResource);
497         mapEntry->value.isPrevalentResource = true;
498     }
499 }
500
501 void WebResourceLoadStatisticsStore::isPrevalentResource(const URL& url, WTF::Function<void (bool)>&& completionHandler)
502 {
503     if (url.isBlankURL() || url.isEmpty()) {
504         completionHandler(false);
505         return;
506     }
507
508     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), completionHandler = WTFMove(completionHandler)] () mutable {
509         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
510         bool isPrevalentResource = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource;
511         RunLoop::main().dispatch([isPrevalentResource, completionHandler = WTFMove(completionHandler)] {
512             completionHandler(isPrevalentResource);
513         });
514     });
515 }
516
517 void WebResourceLoadStatisticsStore::isRegisteredAsSubFrameUnder(const URL& subFrame, const URL& topFrame, WTF::Function<void (bool)>&& completionHandler)
518 {
519     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), subFramePrimaryDomain = isolatedPrimaryDomain(subFrame), topFramePrimaryDomain = isolatedPrimaryDomain(topFrame), completionHandler = WTFMove(completionHandler)] () mutable {
520         auto mapEntry = m_resourceStatisticsMap.find(subFramePrimaryDomain);
521         bool isRegisteredAsSubFrameUnder = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subframeUnderTopFrameOrigins.contains(topFramePrimaryDomain);
522         RunLoop::main().dispatch([isRegisteredAsSubFrameUnder, completionHandler = WTFMove(completionHandler)] {
523             completionHandler(isRegisteredAsSubFrameUnder);
524         });
525     });
526 }
527
528 void WebResourceLoadStatisticsStore::isRegisteredAsRedirectingTo(const URL& hostRedirectedFrom, const URL& hostRedirectedTo, WTF::Function<void (bool)>&& completionHandler)
529 {
530     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), hostRedirectedFromPrimaryDomain = isolatedPrimaryDomain(hostRedirectedFrom), hostRedirectedToPrimaryDomain = isolatedPrimaryDomain(hostRedirectedTo), completionHandler = WTFMove(completionHandler)] () mutable {
531         auto mapEntry = m_resourceStatisticsMap.find(hostRedirectedFromPrimaryDomain);
532         bool isRegisteredAsRedirectingTo = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subresourceUniqueRedirectsTo.contains(hostRedirectedToPrimaryDomain);
533         RunLoop::main().dispatch([isRegisteredAsRedirectingTo, completionHandler = WTFMove(completionHandler)] {
534             completionHandler(isRegisteredAsRedirectingTo);
535         });
536     });
537 }
538
539 void WebResourceLoadStatisticsStore::clearPrevalentResource(const URL& url)
540 {
541     if (url.isBlankURL() || url.isEmpty())
542         return;
543
544     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
545         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
546         statistics.isPrevalentResource = false;
547     });
548 }
549
550 void WebResourceLoadStatisticsStore::setGrandfathered(const URL& url, bool value)
551 {
552     if (url.isBlankURL() || url.isEmpty())
553         return;
554
555     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), value] {
556         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
557         statistics.grandfathered = value;
558     });
559 }
560
561 void WebResourceLoadStatisticsStore::isGrandfathered(const URL& url, WTF::Function<void (bool)>&& completionHandler)
562 {
563     if (url.isBlankURL() || url.isEmpty()) {
564         completionHandler(false);
565         return;
566     }
567
568     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler), primaryDomain = isolatedPrimaryDomain(url)] () mutable {
569         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
570         bool isGrandFathered = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.grandfathered;
571         RunLoop::main().dispatch([isGrandFathered, completionHandler = WTFMove(completionHandler)] {
572             completionHandler(isGrandFathered);
573         });
574     });
575 }
576
577 void WebResourceLoadStatisticsStore::setSubframeUnderTopFrameOrigin(const URL& subframe, const URL& topFrame)
578 {
579     if (subframe.isBlankURL() || subframe.isEmpty() || topFrame.isBlankURL() || topFrame.isEmpty())
580         return;
581
582     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryTopFrameDomain = isolatedPrimaryDomain(topFrame), primarySubFrameDomain = isolatedPrimaryDomain(subframe)] {
583         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubFrameDomain);
584         statistics.subframeUnderTopFrameOrigins.add(primaryTopFrameDomain);
585         // For consistency, make sure we also have a statistics entry for the top frame domain.
586         ensureResourceStatisticsForPrimaryDomain(primaryTopFrameDomain);
587     });
588 }
589
590 void WebResourceLoadStatisticsStore::setSubresourceUnderTopFrameOrigin(const URL& subresource, const URL& topFrame)
591 {
592     if (subresource.isBlankURL() || subresource.isEmpty() || topFrame.isBlankURL() || topFrame.isEmpty())
593         return;
594
595     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryTopFrameDomain = isolatedPrimaryDomain(topFrame), primarySubresourceDomain = isolatedPrimaryDomain(subresource)] {
596         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
597         statistics.subresourceUnderTopFrameOrigins.add(primaryTopFrameDomain);
598         // For consistency, make sure we also have a statistics entry for the top frame domain.
599         ensureResourceStatisticsForPrimaryDomain(primaryTopFrameDomain);
600     });
601 }
602
603 void WebResourceLoadStatisticsStore::setSubresourceUniqueRedirectTo(const URL& subresource, const URL& hostNameRedirectedTo)
604 {
605     if (subresource.isBlankURL() || subresource.isEmpty() || hostNameRedirectedTo.isBlankURL() || hostNameRedirectedTo.isEmpty())
606         return;
607
608     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryRedirectDomain = isolatedPrimaryDomain(hostNameRedirectedTo), primarySubresourceDomain = isolatedPrimaryDomain(subresource)] {
609         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
610         statistics.subresourceUniqueRedirectsTo.add(primaryRedirectDomain);
611         // For consistency, make sure we also have a statistics entry for the redirect domain.
612         ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
613     });
614 }
615
616 void WebResourceLoadStatisticsStore::setSubresourceUniqueRedirectFrom(const URL& subresource, const URL& hostNameRedirectedFrom)
617 {
618     if (subresource.isBlankURL() || subresource.isEmpty() || hostNameRedirectedFrom.isBlankURL() || hostNameRedirectedFrom.isEmpty())
619         return;
620     
621     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryRedirectDomain = isolatedPrimaryDomain(hostNameRedirectedFrom), primarySubresourceDomain = isolatedPrimaryDomain(subresource)] {
622         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
623         statistics.subresourceUniqueRedirectsFrom.add(primaryRedirectDomain);
624         // For consistency, make sure we also have a statistics entry for the redirect domain.
625         ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
626     });
627 }
628
629 void WebResourceLoadStatisticsStore::setTopFrameUniqueRedirectTo(const URL& topFrameHostName, const URL& hostNameRedirectedTo)
630 {
631     if (topFrameHostName.isBlankURL() || topFrameHostName.isEmpty() || hostNameRedirectedTo.isBlankURL() || hostNameRedirectedTo.isEmpty())
632         return;
633     
634     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryRedirectDomain = isolatedPrimaryDomain(hostNameRedirectedTo), topFramePrimaryDomain = isolatedPrimaryDomain(topFrameHostName)] {
635         auto& statistics = ensureResourceStatisticsForPrimaryDomain(topFramePrimaryDomain);
636         statistics.topFrameUniqueRedirectsTo.add(primaryRedirectDomain);
637         // For consistency, make sure we also have a statistics entry for the redirect domain.
638         ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
639     });
640 }
641
642 void WebResourceLoadStatisticsStore::setTopFrameUniqueRedirectFrom(const URL& topFrameHostName, const URL& hostNameRedirectedFrom)
643 {
644     if (topFrameHostName.isBlankURL() || topFrameHostName.isEmpty() || hostNameRedirectedFrom.isBlankURL() || hostNameRedirectedFrom.isEmpty())
645         return;
646     
647     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryRedirectDomain = isolatedPrimaryDomain(hostNameRedirectedFrom), topFramePrimaryDomain = isolatedPrimaryDomain(topFrameHostName)] {
648         auto& statistics = ensureResourceStatisticsForPrimaryDomain(topFramePrimaryDomain);
649         statistics.topFrameUniqueRedirectsFrom.add(primaryRedirectDomain);
650         // For consistency, make sure we also have a statistics entry for the redirect domain.
651         ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
652     });
653 }
654
655 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningUpdate(CompletionHandler<void()>&& callback)
656 {
657     // Helper function used by testing system. Should only be called from the main thread.
658     ASSERT(RunLoop::isMain());
659
660     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), callback = WTFMove(callback)] () mutable {
661         updateCookiePartitioning(WTFMove(callback));
662     });
663 }
664
665 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningUpdateForDomains(const Vector<String>& domainsToPartition, const Vector<String>& domainsToBlock, const Vector<String>& domainsToNeitherPartitionNorBlock, ShouldClearFirst shouldClearFirst, CompletionHandler<void()>&& callback)
666 {
667     // Helper function used by testing system. Should only be called from the main thread.
668     ASSERT(RunLoop::isMain());
669     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), domainsToPartition = crossThreadCopy(domainsToPartition), domainsToBlock = crossThreadCopy(domainsToBlock), domainsToNeitherPartitionNorBlock = crossThreadCopy(domainsToNeitherPartitionNorBlock), shouldClearFirst, callback = WTFMove(callback)] () mutable {
670         updateCookiePartitioningForDomains(domainsToPartition, domainsToBlock, domainsToNeitherPartitionNorBlock, shouldClearFirst, WTFMove(callback));
671     });
672 }
673
674 void WebResourceLoadStatisticsStore::scheduleClearPartitioningStateForDomains(const Vector<String>& domains, CompletionHandler<void()>&& callback)
675 {
676     // Helper function used by testing system. Should only be called from the main thread.
677     ASSERT(RunLoop::isMain());
678     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), domains = crossThreadCopy(domains), callback = WTFMove(callback)] () mutable {
679         clearPartitioningStateForDomains(domains, WTFMove(callback));
680     });
681 }
682
683 #if HAVE(CFNETWORK_STORAGE_PARTITIONING)
684 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningStateReset()
685 {
686     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
687         resetCookiePartitioningState();
688     });
689 }
690 #endif
691
692 void WebResourceLoadStatisticsStore::scheduleClearInMemory()
693 {
694     ASSERT(RunLoop::isMain());
695     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
696         clearInMemory();
697     });
698 }
699
700 void WebResourceLoadStatisticsStore::scheduleClearInMemoryAndPersistent(ShouldGrandfather shouldGrandfather, CompletionHandler<void()>&& callback)
701 {
702     ASSERT(RunLoop::isMain());
703     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), shouldGrandfather, callback = WTFMove(callback)] () mutable {
704         clearInMemory();
705         m_persistentStorage.clear();
706         
707         if (shouldGrandfather == ShouldGrandfather::Yes)
708             grandfatherExistingWebsiteData([protectedThis = makeRef(*this), callback = WTFMove(callback)]() {
709                 callback();
710             });
711         else {
712             callback();
713         }
714     });
715 }
716
717 void WebResourceLoadStatisticsStore::scheduleClearInMemoryAndPersistent(WallTime modifiedSince, ShouldGrandfather shouldGrandfather, CompletionHandler<void()>&& callback)
718 {
719     // For now, be conservative and clear everything regardless of modifiedSince.
720     UNUSED_PARAM(modifiedSince);
721     scheduleClearInMemoryAndPersistent(shouldGrandfather, WTFMove(callback));
722 }
723
724 void WebResourceLoadStatisticsStore::setTimeToLiveUserInteraction(Seconds seconds)
725 {
726     ASSERT(seconds >= 0_s);
727     m_parameters.timeToLiveUserInteraction = seconds;
728 }
729
730 void WebResourceLoadStatisticsStore::setTimeToLiveCookiePartitionFree(Seconds seconds)
731 {
732     ASSERT(seconds >= 0_s);
733     m_parameters.timeToLiveCookiePartitionFree = seconds;
734 }
735
736 void WebResourceLoadStatisticsStore::setMinimumTimeBetweenDataRecordsRemoval(Seconds seconds)
737 {
738     ASSERT(seconds >= 0_s);
739     m_parameters.minimumTimeBetweenDataRecordsRemoval = seconds;
740 }
741
742 void WebResourceLoadStatisticsStore::setGrandfatheringTime(Seconds seconds)
743 {
744     ASSERT(seconds >= 0_s);
745     m_parameters.grandfatheringTime = seconds;
746 }
747
748 bool WebResourceLoadStatisticsStore::shouldRemoveDataRecords() const
749 {
750     ASSERT(!RunLoop::isMain());
751     if (m_dataRecordsBeingRemoved)
752         return false;
753
754 #if ENABLE(NETSCAPE_PLUGIN_API)
755     for (auto plugin : PluginProcessManager::singleton().pluginProcesses()) {
756         if (!m_activePluginTokens.contains(plugin->pluginProcessToken()))
757             return true;
758     }
759 #endif
760
761     return !m_lastTimeDataRecordsWereRemoved || MonotonicTime::now() >= (m_lastTimeDataRecordsWereRemoved + m_parameters.minimumTimeBetweenDataRecordsRemoval);
762 }
763
764 void WebResourceLoadStatisticsStore::setDataRecordsBeingRemoved(bool value)
765 {
766     ASSERT(!RunLoop::isMain());
767     m_dataRecordsBeingRemoved = value;
768     if (m_dataRecordsBeingRemoved)
769         m_lastTimeDataRecordsWereRemoved = MonotonicTime::now();
770 }
771
772 ResourceLoadStatistics& WebResourceLoadStatisticsStore::ensureResourceStatisticsForPrimaryDomain(const String& primaryDomain)
773 {
774     ASSERT(!RunLoop::isMain());
775     return m_resourceStatisticsMap.ensure(primaryDomain, [&primaryDomain] {
776         return ResourceLoadStatistics(primaryDomain);
777     }).iterator->value;
778 }
779
780 std::unique_ptr<KeyedEncoder> WebResourceLoadStatisticsStore::createEncoderFromData() const
781 {
782     ASSERT(!RunLoop::isMain());
783     auto encoder = KeyedEncoder::encoder();
784     encoder->encodeUInt32("version", statisticsModelVersion);
785     encoder->encodeDouble("endOfGrandfatheringTimestamp", m_endOfGrandfatheringTimestamp.secondsSinceEpoch().value());
786
787     encoder->encodeObjects("browsingStatistics", m_resourceStatisticsMap.begin(), m_resourceStatisticsMap.end(), [](KeyedEncoder& encoderInner, const auto& origin) {
788         origin.value.encode(encoderInner);
789     });
790
791     encoder->encodeObjects("operatingDates", m_operatingDates.begin(), m_operatingDates.end(), [](KeyedEncoder& encoderInner, OperatingDate date) {
792         encoderInner.encodeDouble("date", date.secondsSinceEpoch().value());
793     });
794
795     return encoder;
796 }
797
798 void WebResourceLoadStatisticsStore::mergeWithDataFromDecoder(KeyedDecoder& decoder)
799 {
800     ASSERT(!RunLoop::isMain());
801
802     unsigned versionOnDisk;
803     if (!decoder.decodeUInt32("version", versionOnDisk))
804         return;
805
806     if (versionOnDisk != statisticsModelVersion)
807         return;
808
809     double endOfGrandfatheringTimestamp;
810     if (decoder.decodeDouble("endOfGrandfatheringTimestamp", endOfGrandfatheringTimestamp))
811         m_endOfGrandfatheringTimestamp = WallTime::fromRawSeconds(endOfGrandfatheringTimestamp);
812     else
813         m_endOfGrandfatheringTimestamp = { };
814
815     Vector<ResourceLoadStatistics> loadedStatistics;
816     bool succeeded = decoder.decodeObjects("browsingStatistics", loadedStatistics, [](KeyedDecoder& decoderInner, ResourceLoadStatistics& statistics) {
817         return statistics.decode(decoderInner);
818     });
819
820     if (!succeeded)
821         return;
822
823     mergeStatistics(WTFMove(loadedStatistics));
824     updateCookiePartitioning([]() { });
825
826     Vector<OperatingDate> operatingDates;
827     succeeded = decoder.decodeObjects("operatingDates", operatingDates, [](KeyedDecoder& decoder, OperatingDate& date) {
828         double value;
829         if (!decoder.decodeDouble("date", value))
830             return false;
831
832         date = OperatingDate::fromWallTime(WallTime::fromRawSeconds(value));
833         return true;
834     });
835
836     if (!succeeded)
837         return;
838
839     m_operatingDates = mergeOperatingDates(m_operatingDates, WTFMove(operatingDates));
840 }
841
842 void WebResourceLoadStatisticsStore::clearInMemory()
843 {
844     ASSERT(!RunLoop::isMain());
845     m_resourceStatisticsMap.clear();
846     m_operatingDates.clear();
847
848     updateCookiePartitioningForDomains({ }, { }, { }, ShouldClearFirst::Yes, []() { });
849 }
850
851 bool WebResourceLoadStatisticsStore::wasAccessedAsFirstPartyDueToUserInteraction(const ResourceLoadStatistics& current, const ResourceLoadStatistics& updated)
852 {
853     if (!current.hadUserInteraction && !updated.hadUserInteraction)
854         return false;
855
856     auto mostRecentUserInteractionTime = std::max(current.mostRecentUserInteractionTime, updated.mostRecentUserInteractionTime);
857
858     return updated.lastSeen <= mostRecentUserInteractionTime + m_parameters.timeToLiveCookiePartitionFree;
859 }
860
861 void WebResourceLoadStatisticsStore::mergeStatistics(Vector<ResourceLoadStatistics>&& statistics)
862 {
863     ASSERT(!RunLoop::isMain());
864     for (auto& statistic : statistics) {
865         auto result = m_resourceStatisticsMap.ensure(statistic.highLevelDomain, [&statistic] {
866             return WTFMove(statistic);
867         });
868         if (!result.isNewEntry) {
869             if (wasAccessedAsFirstPartyDueToUserInteraction(result.iterator->value, statistic))
870                 result.iterator->value.timesAccessedAsFirstPartyDueToUserInteraction++;
871             result.iterator->value.merge(statistic);
872         }
873     }
874 }
875
876 bool WebResourceLoadStatisticsStore::shouldPartitionCookies(const ResourceLoadStatistics& statistic) const
877 {
878     return statistic.isPrevalentResource && statistic.hadUserInteraction && WallTime::now() > statistic.mostRecentUserInteractionTime + m_parameters.timeToLiveCookiePartitionFree;
879 }
880
881 bool WebResourceLoadStatisticsStore::shouldBlockCookies(const ResourceLoadStatistics& statistic) const
882 {
883     return statistic.isPrevalentResource && !statistic.hadUserInteraction;
884 }
885
886 void WebResourceLoadStatisticsStore::updateCookiePartitioning(CompletionHandler<void()>&& callback)
887 {
888     ASSERT(!RunLoop::isMain());
889
890     Vector<String> domainsToPartition;
891     Vector<String> domainsToBlock;
892     Vector<String> domainsToNeitherPartitionNorBlock;
893     for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
894
895         bool shouldPartition = shouldPartitionCookies(resourceStatistic);
896         bool shouldBlock = shouldBlockCookies(resourceStatistic);
897
898         if (shouldPartition && !resourceStatistic.isMarkedForCookiePartitioning) {
899             domainsToPartition.append(resourceStatistic.highLevelDomain);
900             resourceStatistic.isMarkedForCookiePartitioning = true;
901         } else if (shouldBlock && !resourceStatistic.isMarkedForCookieBlocking) {
902             domainsToBlock.append(resourceStatistic.highLevelDomain);
903             resourceStatistic.isMarkedForCookieBlocking = true;
904         } else if (!shouldPartition && !shouldBlock && resourceStatistic.isPrevalentResource) {
905             domainsToNeitherPartitionNorBlock.append(resourceStatistic.highLevelDomain);
906             resourceStatistic.isMarkedForCookiePartitioning = false;
907             resourceStatistic.isMarkedForCookieBlocking = false;
908         }
909     }
910
911     if (domainsToPartition.isEmpty() && domainsToBlock.isEmpty() && domainsToNeitherPartitionNorBlock.isEmpty()) {
912         callback();
913         return;
914     }
915
916     RunLoop::main().dispatch([this, protectedThis = makeRef(*this), domainsToPartition = crossThreadCopy(domainsToPartition), domainsToBlock = crossThreadCopy(domainsToBlock), domainsToNeitherPartitionNorBlock = crossThreadCopy(domainsToNeitherPartitionNorBlock), callback = WTFMove(callback)] () {
917         m_updatePrevalentDomainsToPartitionOrBlockCookiesHandler(domainsToPartition, domainsToBlock, domainsToNeitherPartitionNorBlock, ShouldClearFirst::No);
918         callback();
919     });
920 }
921
922 void WebResourceLoadStatisticsStore::updateCookiePartitioningForDomains(const Vector<String>& domainsToPartition, const Vector<String>& domainsToBlock, const Vector<String>& domainsToNeitherPartitionNorBlock, ShouldClearFirst shouldClearFirst, CompletionHandler<void()>&& callback)
923 {
924     ASSERT(!RunLoop::isMain());
925     if (domainsToPartition.isEmpty() && domainsToBlock.isEmpty() && domainsToNeitherPartitionNorBlock.isEmpty() && shouldClearFirst == ShouldClearFirst::No) {
926         callback();
927         return;
928     }
929     
930     RunLoop::main().dispatch([this, shouldClearFirst, protectedThis = makeRef(*this), domainsToPartition = crossThreadCopy(domainsToPartition), domainsToBlock = crossThreadCopy(domainsToBlock), domainsToNeitherPartitionNorBlock = crossThreadCopy(domainsToNeitherPartitionNorBlock)] () {
931         m_updatePrevalentDomainsToPartitionOrBlockCookiesHandler(domainsToPartition, domainsToBlock, domainsToNeitherPartitionNorBlock, shouldClearFirst);
932     });
933
934     if (shouldClearFirst == ShouldClearFirst::Yes)
935         resetCookiePartitioningState();
936     else {
937         for (auto& domain : domainsToNeitherPartitionNorBlock) {
938             auto& statistic = ensureResourceStatisticsForPrimaryDomain(domain);
939             statistic.isMarkedForCookiePartitioning = false;
940             statistic.isMarkedForCookieBlocking = false;
941         }
942     }
943
944     for (auto& domain : domainsToPartition)
945         ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookiePartitioning = true;
946
947     for (auto& domain : domainsToBlock)
948         ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookieBlocking = true;
949
950     callback();
951 }
952
953 void WebResourceLoadStatisticsStore::clearPartitioningStateForDomains(const Vector<String>& domains, CompletionHandler<void()>&& callback)
954 {
955     ASSERT(!RunLoop::isMain());
956     if (domains.isEmpty()) {
957         callback();
958         return;
959     }
960
961     RunLoop::main().dispatch([this, protectedThis = makeRef(*this), domains = crossThreadCopy(domains)] () {
962         m_removeDomainsHandler(domains);
963     });
964
965     for (auto& domain : domains) {
966         auto& statistic = ensureResourceStatisticsForPrimaryDomain(domain);
967         statistic.isMarkedForCookiePartitioning = false;
968         statistic.isMarkedForCookieBlocking = false;
969     }
970
971     callback();
972 }
973
974 void WebResourceLoadStatisticsStore::resetCookiePartitioningState()
975 {
976     ASSERT(!RunLoop::isMain());
977     for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
978         resourceStatistic.isMarkedForCookiePartitioning = false;
979         resourceStatistic.isMarkedForCookieBlocking = false;
980     }
981 }
982
983 void WebResourceLoadStatisticsStore::processStatistics(const WTF::Function<void (const ResourceLoadStatistics&)>& processFunction) const
984 {
985     ASSERT(!RunLoop::isMain());
986     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
987         processFunction(resourceStatistic);
988 }
989
990 bool WebResourceLoadStatisticsStore::hasHadUnexpiredRecentUserInteraction(ResourceLoadStatistics& resourceStatistic) const
991 {
992     if (resourceStatistic.hadUserInteraction && hasStatisticsExpired(resourceStatistic)) {
993         // Drop privacy sensitive data because we no longer need it.
994         // Set timestamp to 0 so that statistics merge will know
995         // it has been reset as opposed to its default -1.
996         resourceStatistic.mostRecentUserInteractionTime = { };
997         resourceStatistic.hadUserInteraction = false;
998     }
999
1000     return resourceStatistic.hadUserInteraction;
1001 }
1002
1003 Vector<String> WebResourceLoadStatisticsStore::topPrivatelyControlledDomainsToRemoveWebsiteDataFor()
1004 {
1005     ASSERT(!RunLoop::isMain());
1006
1007     bool shouldCheckForGrandfathering = m_endOfGrandfatheringTimestamp > WallTime::now();
1008     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && m_endOfGrandfatheringTimestamp;
1009
1010     if (shouldClearGrandfathering)
1011         m_endOfGrandfatheringTimestamp = { };
1012
1013     Vector<String> prevalentResources;
1014     for (auto& statistic : m_resourceStatisticsMap.values()) {
1015         if (statistic.isPrevalentResource && !hasHadUnexpiredRecentUserInteraction(statistic) && (!shouldCheckForGrandfathering || !statistic.grandfathered))
1016             prevalentResources.append(statistic.highLevelDomain);
1017
1018         if (shouldClearGrandfathering && statistic.grandfathered)
1019             statistic.grandfathered = false;
1020     }
1021
1022     return prevalentResources;
1023 }
1024
1025 void WebResourceLoadStatisticsStore::includeTodayAsOperatingDateIfNecessary()
1026 {
1027     ASSERT(!RunLoop::isMain());
1028
1029     auto today = OperatingDate::today();
1030     if (!m_operatingDates.isEmpty() && today <= m_operatingDates.last())
1031         return;
1032
1033     while (m_operatingDates.size() >= operatingDatesWindow)
1034         m_operatingDates.remove(0);
1035
1036     m_operatingDates.append(today);
1037 }
1038
1039 bool WebResourceLoadStatisticsStore::hasStatisticsExpired(const ResourceLoadStatistics& resourceStatistic) const
1040 {
1041     if (m_operatingDates.size() >= operatingDatesWindow) {
1042         if (OperatingDate::fromWallTime(resourceStatistic.mostRecentUserInteractionTime) < m_operatingDates.first())
1043             return true;
1044     }
1045
1046     // If we don't meet the real criteria for an expired statistic, check the user setting for a tighter restriction (mainly for testing).
1047     if (m_parameters.timeToLiveUserInteraction) {
1048         if (WallTime::now() > resourceStatistic.mostRecentUserInteractionTime + m_parameters.timeToLiveUserInteraction.value())
1049             return true;
1050     }
1051
1052     return false;
1053 }
1054     
1055 void WebResourceLoadStatisticsStore::setMaxStatisticsEntries(size_t maximumEntryCount)
1056 {
1057     m_parameters.maxStatisticsEntries = maximumEntryCount;
1058 }
1059     
1060 void WebResourceLoadStatisticsStore::setPruneEntriesDownTo(size_t pruneTargetCount)
1061 {
1062     m_parameters.pruneEntriesDownTo = pruneTargetCount;
1063 }
1064     
1065 struct StatisticsLastSeen {
1066     String topPrivatelyOwnedDomain;
1067     WallTime lastSeen;
1068 };
1069     
1070 static void pruneResources(HashMap<String, WebCore::ResourceLoadStatistics>& statisticsMap, Vector<StatisticsLastSeen>& statisticsToPrune, size_t& numberOfEntriesToPrune)
1071 {
1072     if (statisticsToPrune.size() > numberOfEntriesToPrune) {
1073         std::sort(statisticsToPrune.begin(), statisticsToPrune.end(), [](const StatisticsLastSeen& a, const StatisticsLastSeen& b) {
1074             return a.lastSeen < b.lastSeen;
1075         });
1076     }
1077
1078     for (size_t i = 0, end = std::min(numberOfEntriesToPrune, statisticsToPrune.size()); i != end; ++i, --numberOfEntriesToPrune)
1079         statisticsMap.remove(statisticsToPrune[i].topPrivatelyOwnedDomain);
1080 }
1081     
1082 static unsigned computeImportance(const ResourceLoadStatistics& resourceStatistic)
1083 {
1084     unsigned importance = maxImportance;
1085     if (!resourceStatistic.isPrevalentResource)
1086         importance -= 1;
1087     if (!resourceStatistic.hadUserInteraction)
1088         importance -= 2;
1089     return importance;
1090 }
1091     
1092 void WebResourceLoadStatisticsStore::pruneStatisticsIfNeeded()
1093 {
1094     ASSERT(!RunLoop::isMain());
1095     if (m_resourceStatisticsMap.size() <= m_parameters.maxStatisticsEntries)
1096         return;
1097
1098     ASSERT(m_parameters.pruneEntriesDownTo <= m_parameters.maxStatisticsEntries);
1099
1100     size_t numberOfEntriesLeftToPrune = m_resourceStatisticsMap.size() - m_parameters.pruneEntriesDownTo;
1101     ASSERT(numberOfEntriesLeftToPrune);
1102     
1103     Vector<StatisticsLastSeen> resourcesToPrunePerImportance[maxImportance + 1];
1104     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
1105         resourcesToPrunePerImportance[computeImportance(resourceStatistic)].append({ resourceStatistic.highLevelDomain, resourceStatistic.lastSeen });
1106     
1107     for (unsigned importance = 0; numberOfEntriesLeftToPrune && importance <= maxImportance; ++importance)
1108         pruneResources(m_resourceStatisticsMap, resourcesToPrunePerImportance[importance], numberOfEntriesLeftToPrune);
1109
1110     ASSERT(!numberOfEntriesLeftToPrune);
1111 }
1112
1113 void WebResourceLoadStatisticsStore::resetParametersToDefaultValues()
1114 {
1115     m_parameters = { };
1116 }
1117
1118 void WebResourceLoadStatisticsStore::logTestingEvent(const String& event)
1119 {
1120     if (!m_statisticsTestingCallback)
1121         return;
1122
1123     if (RunLoop::isMain())
1124         m_statisticsTestingCallback(event);
1125     else {
1126         RunLoop::main().dispatch([this, protectedThis = makeRef(*this), event = event.isolatedCopy()] {
1127             if (m_statisticsTestingCallback)
1128                 m_statisticsTestingCallback(event);
1129         });
1130     }
1131 }
1132
1133 } // namespace WebKit