7d17afe2a6194eccabd9f14fc2cb4a67282fda66
[WebKit-https.git] / Source / WebKit / UIProcess / ResourceLoadStatisticsMemoryStore.cpp
1 /*
2  * Copyright (C) 2017-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ResourceLoadStatisticsMemoryStore.h"
28
29 #include "Logging.h"
30 #include "PluginProcessManager.h"
31 #include "PluginProcessProxy.h"
32 #include "ResourceLoadStatisticsPersistentStorage.h"
33 #include "WebProcessProxy.h"
34 #include "WebResourceLoadStatisticsTelemetry.h"
35 #include "WebsiteDataStore.h"
36 #include <WebCore/KeyedCoding.h>
37 #include <WebCore/ResourceLoadStatistics.h>
38 #include <wtf/CallbackAggregator.h>
39 #include <wtf/DateMath.h>
40 #include <wtf/MathExtras.h>
41
42 #if !RELEASE_LOG_DISABLED
43 #include <wtf/text/StringBuilder.h>
44 #endif
45
46 namespace WebKit {
47 using namespace WebCore;
48
49 constexpr unsigned statisticsModelVersion { 14 };
50 constexpr unsigned maxNumberOfRecursiveCallsInRedirectTraceBack { 50 };
51 constexpr Seconds minimumStatisticsProcessingInterval { 5_s };
52 constexpr unsigned operatingDatesWindow { 30 };
53 constexpr unsigned maxImportance { 3 };
54 static const char* debugStaticPrevalentResource { "3rdpartytestwebkit.org" };
55
56 #if !RELEASE_LOG_DISABLED
57 static String domainsToString(const Vector<String>& domains)
58 {
59     StringBuilder builder;
60     for (auto& domain : domains) {
61         if (!builder.isEmpty())
62             builder.appendLiteral(", ");
63         builder.append(domain);
64     }
65     return builder.toString();
66 }
67 #endif
68
69 class OperatingDate {
70 public:
71     OperatingDate() = default;
72
73     static OperatingDate fromWallTime(WallTime time)
74     {
75         double ms = time.secondsSinceEpoch().milliseconds();
76         int year = msToYear(ms);
77         int yearDay = dayInYear(ms, year);
78         int month = monthFromDayInYear(yearDay, isLeapYear(year));
79         int monthDay = dayInMonthFromDayInYear(yearDay, isLeapYear(year));
80
81         return OperatingDate { year, month, monthDay };
82     }
83
84     static OperatingDate today()
85     {
86         return OperatingDate::fromWallTime(WallTime::now());
87     }
88
89     Seconds secondsSinceEpoch() const
90     {
91         return Seconds { dateToDaysFrom1970(m_year, m_month, m_monthDay) * secondsPerDay };
92     }
93
94     bool operator==(const OperatingDate& other) const
95     {
96         return m_monthDay == other.m_monthDay && m_month == other.m_month && m_year == other.m_year;
97     }
98
99     bool operator<(const OperatingDate& other) const
100     {
101         return secondsSinceEpoch() < other.secondsSinceEpoch();
102     }
103
104     bool operator<=(const OperatingDate& other) const
105     {
106         return secondsSinceEpoch() <= other.secondsSinceEpoch();
107     }
108
109 private:
110     OperatingDate(int year, int month, int monthDay)
111         : m_year(year)
112         , m_month(month)
113         , m_monthDay(monthDay)
114     { }
115
116     int m_year { 0 };
117     int m_month { 0 }; // [0, 11].
118     int m_monthDay { 0 }; // [1, 31].
119 };
120
121 static Vector<OperatingDate> mergeOperatingDates(const Vector<OperatingDate>& existingDates, Vector<OperatingDate>&& newDates)
122 {
123     if (existingDates.isEmpty())
124         return WTFMove(newDates);
125
126     Vector<OperatingDate> mergedDates(existingDates.size() + newDates.size());
127
128     // Merge the two sorted vectors of dates.
129     std::merge(existingDates.begin(), existingDates.end(), newDates.begin(), newDates.end(), mergedDates.begin());
130     // Remove duplicate dates.
131     removeRepeatedElements(mergedDates);
132
133     // Drop old dates until the Vector size reaches operatingDatesWindow.
134     while (mergedDates.size() > operatingDatesWindow)
135         mergedDates.remove(0);
136
137     return mergedDates;
138 }
139
140 struct StatisticsLastSeen {
141     String topPrivatelyOwnedDomain;
142     WallTime lastSeen;
143 };
144
145 static void pruneResources(HashMap<String, WebCore::ResourceLoadStatistics>& statisticsMap, Vector<StatisticsLastSeen>& statisticsToPrune, size_t& numberOfEntriesToPrune)
146 {
147     if (statisticsToPrune.size() > numberOfEntriesToPrune) {
148         std::sort(statisticsToPrune.begin(), statisticsToPrune.end(), [](const StatisticsLastSeen& a, const StatisticsLastSeen& b) {
149             return a.lastSeen < b.lastSeen;
150         });
151     }
152
153     for (size_t i = 0, end = std::min(numberOfEntriesToPrune, statisticsToPrune.size()); i != end; ++i, --numberOfEntriesToPrune)
154         statisticsMap.remove(statisticsToPrune[i].topPrivatelyOwnedDomain);
155 }
156
157 static unsigned computeImportance(const ResourceLoadStatistics& resourceStatistic)
158 {
159     unsigned importance = maxImportance;
160     if (!resourceStatistic.isPrevalentResource)
161         importance -= 1;
162     if (!resourceStatistic.hadUserInteraction)
163         importance -= 2;
164     return importance;
165 }
166
167 ResourceLoadStatisticsMemoryStore::ResourceLoadStatisticsMemoryStore(WebResourceLoadStatisticsStore& store, WorkQueue& workQueue)
168     : m_store(store)
169     , m_workQueue(workQueue)
170 {
171     ASSERT(!RunLoop::isMain());
172
173 #if PLATFORM(COCOA)
174     registerUserDefaultsIfNeeded();
175 #endif
176     includeTodayAsOperatingDateIfNecessary();
177
178     m_workQueue->dispatchAfter(5_s, [weakThis = makeWeakPtr(*this)] {
179         if (weakThis)
180             weakThis->calculateAndSubmitTelemetry();
181     });
182 }
183
184 ResourceLoadStatisticsMemoryStore::~ResourceLoadStatisticsMemoryStore()
185 {
186     ASSERT(!RunLoop::isMain());
187 }
188
189 void ResourceLoadStatisticsMemoryStore::setPersistentStorage(ResourceLoadStatisticsPersistentStorage& persistentStorage)
190 {
191     m_persistentStorage = makeWeakPtr(persistentStorage);
192 }
193
194 void ResourceLoadStatisticsMemoryStore::calculateAndSubmitTelemetry() const
195 {
196     ASSERT(!RunLoop::isMain());
197
198     if (m_parameters.shouldSubmitTelemetry)
199         WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
200 }
201
202 void ResourceLoadStatisticsMemoryStore::setNotifyPagesWhenDataRecordsWereScanned(bool value)
203 {
204     ASSERT(!RunLoop::isMain());
205     m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned = value;
206 }
207
208 void ResourceLoadStatisticsMemoryStore::setShouldClassifyResourcesBeforeDataRecordsRemoval(bool value)
209 {
210     ASSERT(!RunLoop::isMain());
211     m_parameters.shouldClassifyResourcesBeforeDataRecordsRemoval = value;
212 }
213
214 void ResourceLoadStatisticsMemoryStore::setShouldSubmitTelemetry(bool value)
215 {
216     ASSERT(!RunLoop::isMain());
217     m_parameters.shouldSubmitTelemetry = value;
218 }
219
220 void ResourceLoadStatisticsMemoryStore::removeDataRecords(CompletionHandler<void()>&& callback)
221 {
222     ASSERT(!RunLoop::isMain());
223
224     if (!shouldRemoveDataRecords()) {
225         callback();
226         return;
227     }
228
229 #if ENABLE(NETSCAPE_PLUGIN_API)
230     m_activePluginTokens.clear();
231     for (auto plugin : PluginProcessManager::singleton().pluginProcesses())
232         m_activePluginTokens.add(plugin->pluginProcessToken());
233 #endif
234
235     auto prevalentResourceDomains = topPrivatelyControlledDomainsToRemoveWebsiteDataFor();
236     if (prevalentResourceDomains.isEmpty()) {
237         callback();
238         return;
239     }
240
241 #if !RELEASE_LOG_DISABLED
242     RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "About to remove data records for %{public}s.", domainsToString(prevalentResourceDomains).utf8().data());
243 #endif
244
245     setDataRecordsBeingRemoved(true);
246
247     RunLoop::main().dispatch([prevalentResourceDomains = crossThreadCopy(prevalentResourceDomains), callback = WTFMove(callback), weakThis = makeWeakPtr(*this), shouldNotifyPagesWhenDataRecordsWereScanned = m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, workQueue = m_workQueue.copyRef()] () mutable {
248         WebProcessProxy::deleteWebsiteDataForTopPrivatelyControlledDomainsInAllPersistentDataStores(WebResourceLoadStatisticsStore::monitoredDataTypes(), WTFMove(prevalentResourceDomains), shouldNotifyPagesWhenDataRecordsWereScanned, [callback = WTFMove(callback), weakThis = WTFMove(weakThis), workQueue = workQueue.copyRef()](const HashSet<String>& domainsWithDeletedWebsiteData) mutable {
249             workQueue->dispatch([topDomains = crossThreadCopy(domainsWithDeletedWebsiteData), callback = WTFMove(callback), weakThis = WTFMove(weakThis)] () mutable {
250                 if (!weakThis) {
251                     callback();
252                     return;
253                 }
254                 for (auto& prevalentResourceDomain : topDomains) {
255                     auto& statistic = weakThis->ensureResourceStatisticsForPrimaryDomain(prevalentResourceDomain);
256                     ++statistic.dataRecordsRemoved;
257                 }
258                 weakThis->setDataRecordsBeingRemoved(false);
259                 callback();
260 #if !RELEASE_LOG_DISABLED
261                 RELEASE_LOG_INFO_IF(weakThis->m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Done removing data records.");
262 #endif
263             });
264         });
265     });
266 }
267
268 unsigned ResourceLoadStatisticsMemoryStore::recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(const WebCore::ResourceLoadStatistics& resourceStatistic, HashSet<String>& domainsThatHaveRedirectedTo, unsigned numberOfRecursiveCalls) const
269 {
270     ASSERT(!RunLoop::isMain());
271
272     if (numberOfRecursiveCalls >= maxNumberOfRecursiveCallsInRedirectTraceBack) {
273         // Model version 14 invokes a deliberate re-classification of the whole set.
274         if (statisticsModelVersion != 14)
275             ASSERT_NOT_REACHED();
276         RELEASE_LOG(ResourceLoadStatistics, "Hit %u recursive calls in redirect backtrace. Returning early.", maxNumberOfRecursiveCallsInRedirectTraceBack);
277         return numberOfRecursiveCalls;
278     }
279
280     numberOfRecursiveCalls++;
281
282     for (auto& subresourceUniqueRedirectFromDomain : resourceStatistic.subresourceUniqueRedirectsFrom.values()) {
283         auto mapEntry = m_resourceStatisticsMap.find(subresourceUniqueRedirectFromDomain);
284         if (mapEntry == m_resourceStatisticsMap.end() || mapEntry->value.isPrevalentResource)
285             continue;
286         if (domainsThatHaveRedirectedTo.add(mapEntry->value.highLevelDomain).isNewEntry)
287             numberOfRecursiveCalls = recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(mapEntry->value, domainsThatHaveRedirectedTo, numberOfRecursiveCalls);
288     }
289     for (auto& topFrameUniqueRedirectFromDomain : resourceStatistic.topFrameUniqueRedirectsFrom.values()) {
290         auto mapEntry = m_resourceStatisticsMap.find(topFrameUniqueRedirectFromDomain);
291         if (mapEntry == m_resourceStatisticsMap.end() || mapEntry->value.isPrevalentResource)
292             continue;
293         if (domainsThatHaveRedirectedTo.add(mapEntry->value.highLevelDomain).isNewEntry)
294             numberOfRecursiveCalls = recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(mapEntry->value, domainsThatHaveRedirectedTo, numberOfRecursiveCalls);
295     }
296
297     return numberOfRecursiveCalls;
298 }
299
300 void ResourceLoadStatisticsMemoryStore::markAsPrevalentIfHasRedirectedToPrevalent(WebCore::ResourceLoadStatistics& resourceStatistic)
301 {
302     ASSERT(!RunLoop::isMain());
303
304     if (resourceStatistic.isPrevalentResource)
305         return;
306
307     for (auto& subresourceDomainRedirectedTo : resourceStatistic.subresourceUniqueRedirectsTo.values()) {
308         auto mapEntry = m_resourceStatisticsMap.find(subresourceDomainRedirectedTo);
309         if (mapEntry != m_resourceStatisticsMap.end() && mapEntry->value.isPrevalentResource) {
310             setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
311             return;
312         }
313     }
314
315     for (auto& topFrameDomainRedirectedTo : resourceStatistic.topFrameUniqueRedirectsTo.values()) {
316         auto mapEntry = m_resourceStatisticsMap.find(topFrameDomainRedirectedTo);
317         if (mapEntry != m_resourceStatisticsMap.end() && mapEntry->value.isPrevalentResource) {
318             setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
319             return;
320         }
321     }
322 }
323
324 bool ResourceLoadStatisticsMemoryStore::isPrevalentDueToDebugMode(ResourceLoadStatistics& resourceStatistic)
325 {
326     if (!m_debugModeEnabled)
327         return false;
328
329     return resourceStatistic.highLevelDomain == debugStaticPrevalentResource || resourceStatistic.highLevelDomain == m_debugManualPrevalentResource;
330 }
331
332 void ResourceLoadStatisticsMemoryStore::processStatisticsAndDataRecords()
333 {
334     ASSERT(!RunLoop::isMain());
335
336     if (m_parameters.shouldClassifyResourcesBeforeDataRecordsRemoval) {
337         for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
338             if (isPrevalentDueToDebugMode(resourceStatistic))
339                 setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
340             else if (!resourceStatistic.isVeryPrevalentResource) {
341                 markAsPrevalentIfHasRedirectedToPrevalent(resourceStatistic);
342                 auto currentPrevalence = resourceStatistic.isPrevalentResource ? ResourceLoadPrevalence::High : ResourceLoadPrevalence::Low;
343                 auto newPrevalence = m_resourceLoadStatisticsClassifier.calculateResourcePrevalence(resourceStatistic, currentPrevalence);
344                 if (newPrevalence != currentPrevalence)
345                     setPrevalentResource(resourceStatistic, newPrevalence);
346             }
347         }
348     }
349
350     removeDataRecords([this, weakThis = makeWeakPtr(*this)] {
351         ASSERT(!RunLoop::isMain());
352         if (!weakThis)
353             return;
354
355         pruneStatisticsIfNeeded();
356         if (m_persistentStorage)
357             m_persistentStorage->scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::No);
358
359         if (!m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned)
360             return;
361
362         RunLoop::main().dispatch([] {
363             WebProcessProxy::notifyPageStatisticsAndDataRecordsProcessed();
364         });
365     });
366 }
367
368 void ResourceLoadStatisticsMemoryStore::hasStorageAccess(const String& subFramePrimaryDomain, const String& topFramePrimaryDomain, uint64_t frameID, uint64_t pageID, CompletionHandler<void(bool)>&& completionHandler)
369 {
370     ASSERT(!RunLoop::isMain());
371
372     auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
373     if (shouldBlockAndPurgeCookies(subFrameStatistic)) {
374         completionHandler(false);
375         return;
376     }
377
378     if (!shouldBlockAndKeepCookies(subFrameStatistic)) {
379         completionHandler(true);
380         return;
381     }
382
383     RunLoop::main().dispatch([store = makeRef(m_store), subFramePrimaryDomain = subFramePrimaryDomain.isolatedCopy(), topFramePrimaryDomain = topFramePrimaryDomain.isolatedCopy(), frameID, pageID, completionHandler = WTFMove(completionHandler)]() mutable {
384         store->callHasStorageAccessForFrameHandler(subFramePrimaryDomain, topFramePrimaryDomain, frameID, pageID, [store = store.copyRef(), completionHandler = WTFMove(completionHandler)](bool result) mutable {
385             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler), result] () mutable {
386                 completionHandler(result);
387             });
388         });
389     });
390 }
391
392 void ResourceLoadStatisticsMemoryStore::requestStorageAccess(String&& subFramePrimaryDomain, String&& topFramePrimaryDomain, uint64_t frameID, uint64_t pageID, bool promptEnabled, CompletionHandler<void(StorageAccessStatus)>&& completionHandler)
393 {
394     ASSERT(!RunLoop::isMain());
395
396     auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
397     if (shouldBlockAndPurgeCookies(subFrameStatistic)) {
398 #if !RELEASE_LOG_DISABLED
399         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Cannot grant storage access to %{public}s since its cookies are blocked in third-party contexts and it has not received user interaction as first-party.", subFramePrimaryDomain.utf8().data());
400 #endif
401         completionHandler(StorageAccessStatus::CannotRequestAccess);
402         return;
403     }
404
405     if (!shouldBlockAndKeepCookies(subFrameStatistic)) {
406 #if !RELEASE_LOG_DISABLED
407         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "No need to grant storage access to %{public}s since its cookies are not blocked in third-party contexts.", subFramePrimaryDomain.utf8().data());
408 #endif
409         completionHandler(StorageAccessStatus::HasAccess);
410         return;
411     }
412
413     auto userWasPromptedEarlier = promptEnabled && hasUserGrantedStorageAccessThroughPrompt(subFrameStatistic, topFramePrimaryDomain);
414     if (promptEnabled && !userWasPromptedEarlier) {
415 #if !RELEASE_LOG_DISABLED
416         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "About to ask the user whether they want to grant storage access to %{public}s under %{public}s or not.", subFramePrimaryDomain.utf8().data(), topFramePrimaryDomain.utf8().data());
417 #endif
418         completionHandler(StorageAccessStatus::RequiresUserPrompt);
419         return;
420     } else if (userWasPromptedEarlier) {
421 #if !RELEASE_LOG_DISABLED
422         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Storage access was granted to %{public}s under %{public}s.", subFramePrimaryDomain.utf8().data(), topFramePrimaryDomain.utf8().data());
423 #endif
424     }
425
426     subFrameStatistic.timesAccessedAsFirstPartyDueToStorageAccessAPI++;
427
428     grantStorageAccessInternal(WTFMove(subFramePrimaryDomain), WTFMove(topFramePrimaryDomain), frameID, pageID, userWasPromptedEarlier, [completionHandler = WTFMove(completionHandler)] (bool wasGrantedAccess) mutable {
429         completionHandler(wasGrantedAccess ? StorageAccessStatus::HasAccess : StorageAccessStatus::CannotRequestAccess);
430     });
431 }
432
433 void ResourceLoadStatisticsMemoryStore::requestStorageAccessUnderOpener(String&& primaryDomainInNeedOfStorageAccess, uint64_t openerPageID, String&& openerPrimaryDomain)
434 {
435     ASSERT(primaryDomainInNeedOfStorageAccess != openerPrimaryDomain);
436     ASSERT(!RunLoop::isMain());
437
438     if (primaryDomainInNeedOfStorageAccess == openerPrimaryDomain)
439         return;
440
441     auto& domainInNeedOfStorageAccessStatistic = ensureResourceStatisticsForPrimaryDomain(primaryDomainInNeedOfStorageAccess);
442     auto cookiesBlockedAndPurged = shouldBlockAndPurgeCookies(domainInNeedOfStorageAccessStatistic);
443
444     // The domain already has access if its cookies are not blocked.
445     if (!cookiesBlockedAndPurged && !shouldBlockAndKeepCookies(domainInNeedOfStorageAccessStatistic))
446         return;
447
448 #if !RELEASE_LOG_DISABLED
449     RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "[Temporary combatibility fix] Storage access was granted for %{public}s under opener page from %{public}s, with user interaction in the opened window.", primaryDomainInNeedOfStorageAccess.utf8().data(), openerPrimaryDomain.utf8().data());
450 #endif
451     grantStorageAccessInternal(WTFMove(primaryDomainInNeedOfStorageAccess), WTFMove(openerPrimaryDomain), std::nullopt, openerPageID, false, [](bool) { });
452 }
453
454 void ResourceLoadStatisticsMemoryStore::grantStorageAccess(String&& subFrameHost, String&& topFrameHost, uint64_t frameID, uint64_t pageID, bool userWasPromptedNow, CompletionHandler<void(bool)>&& completionHandler)
455 {
456     ASSERT(!RunLoop::isMain());
457
458     auto subFramePrimaryDomain = ResourceLoadStatistics::primaryDomain(subFrameHost);
459     auto topFramePrimaryDomain = ResourceLoadStatistics::primaryDomain(topFrameHost);
460     if (userWasPromptedNow) {
461         auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
462         ASSERT(subFrameStatistic.hadUserInteraction);
463         subFrameStatistic.storageAccessUnderTopFrameOrigins.add(topFramePrimaryDomain);
464     }
465     grantStorageAccessInternal(WTFMove(subFramePrimaryDomain), WTFMove(topFramePrimaryDomain), frameID, pageID, userWasPromptedNow, WTFMove(completionHandler));
466 }
467
468 void ResourceLoadStatisticsMemoryStore::grantStorageAccessInternal(String&& subFramePrimaryDomain, String&& topFramePrimaryDomain, std::optional<uint64_t> frameID, uint64_t pageID, bool userWasPromptedNowOrEarlier, CompletionHandler<void(bool)>&& callback)
469 {
470     ASSERT(!RunLoop::isMain());
471
472     if (subFramePrimaryDomain == topFramePrimaryDomain) {
473         callback(true);
474         return;
475     }
476
477     // FIXME: Remove m_storageAccessPromptsEnabled check if prompting is no longer experimental.
478     if (userWasPromptedNowOrEarlier && m_storageAccessPromptsEnabled) {
479         auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
480         ASSERT(subFrameStatistic.hadUserInteraction);
481         ASSERT(subFrameStatistic.storageAccessUnderTopFrameOrigins.contains(topFramePrimaryDomain));
482         subFrameStatistic.mostRecentUserInteractionTime = WallTime::now();
483     }
484
485     RunLoop::main().dispatch([subFramePrimaryDomain = subFramePrimaryDomain.isolatedCopy(), topFramePrimaryDomain = topFramePrimaryDomain.isolatedCopy(), frameID, pageID, store = makeRef(m_store), callback = WTFMove(callback)]() mutable {
486         store->callGrantStorageAccessHandler(subFramePrimaryDomain, topFramePrimaryDomain, frameID, pageID, [callback = WTFMove(callback), store = store.copyRef()](bool value) mutable {
487             store->statisticsQueue().dispatch([callback = WTFMove(callback), value] () mutable {
488                 callback(value);
489             });
490         });
491     });
492 }
493
494 void ResourceLoadStatisticsMemoryStore::grandfatherExistingWebsiteData(CompletionHandler<void()>&& callback)
495 {
496     ASSERT(!RunLoop::isMain());
497
498     RunLoop::main().dispatch([weakThis = makeWeakPtr(*this), callback = WTFMove(callback), shouldNotifyPagesWhenDataRecordsWereScanned = m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, workQueue = m_workQueue.copyRef()] () mutable {
499         // FIXME: This method being a static call on WebProcessProxy is wrong.
500         // It should be on the data store that this object belongs to.
501         WebProcessProxy::topPrivatelyControlledDomainsWithWebsiteData(WebResourceLoadStatisticsStore::monitoredDataTypes(), shouldNotifyPagesWhenDataRecordsWereScanned, [weakThis = WTFMove(weakThis), callback = WTFMove(callback), workQueue = workQueue.copyRef()] (HashSet<String>&& topPrivatelyControlledDomainsWithWebsiteData) mutable {
502             workQueue->dispatch([weakThis = WTFMove(weakThis), topDomains = crossThreadCopy(topPrivatelyControlledDomainsWithWebsiteData), callback = WTFMove(callback)] () mutable {
503                 if (!weakThis) {
504                     callback();
505                     return;
506                 }
507
508                 for (auto& topPrivatelyControlledDomain : topDomains) {
509                     auto& statistic = weakThis->ensureResourceStatisticsForPrimaryDomain(topPrivatelyControlledDomain);
510                     statistic.grandfathered = true;
511                 }
512                 weakThis->m_endOfGrandfatheringTimestamp = WallTime::now() + weakThis->m_parameters.grandfatheringTime;
513                 if (weakThis->m_persistentStorage)
514                     weakThis->m_persistentStorage->scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::Yes);
515                 callback();
516                 weakThis->logTestingEvent("Grandfathered"_s);
517             });
518         });
519     });
520 }
521
522 Vector<String> ResourceLoadStatisticsMemoryStore::ensurePrevalentResourcesForDebugMode()
523 {
524     if (!m_debugModeEnabled)
525         return { };
526
527     Vector<String> primaryDomainsToBlock;
528     primaryDomainsToBlock.reserveInitialCapacity(2);
529
530     auto& staticSesourceStatistic = ensureResourceStatisticsForPrimaryDomain(debugStaticPrevalentResource);
531     setPrevalentResource(staticSesourceStatistic, ResourceLoadPrevalence::High);
532     primaryDomainsToBlock.uncheckedAppend(debugStaticPrevalentResource);
533
534     if (!m_debugManualPrevalentResource.isEmpty()) {
535         auto& manualResourceStatistic = ensureResourceStatisticsForPrimaryDomain(m_debugManualPrevalentResource);
536         setPrevalentResource(manualResourceStatistic, ResourceLoadPrevalence::High);
537         primaryDomainsToBlock.uncheckedAppend(m_debugManualPrevalentResource);
538     }
539     
540     return primaryDomainsToBlock;
541 }
542
543 void ResourceLoadStatisticsMemoryStore::setResourceLoadStatisticsDebugMode(bool enable)
544 {
545     ASSERT(!RunLoop::isMain());
546
547     m_debugModeEnabled = enable;
548     m_debugLoggingEnabled = enable;
549
550     ensurePrevalentResourcesForDebugMode();
551 }
552
553 void ResourceLoadStatisticsMemoryStore::setPrevalentResourceForDebugMode(const String& domain)
554 {
555     if (!m_debugModeEnabled)
556         return;
557
558     m_debugManualPrevalentResource = domain;
559     auto& resourceStatistic = ensureResourceStatisticsForPrimaryDomain(domain);
560     setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
561
562 #if !RELEASE_LOG_DISABLED
563     RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "Did set %{public}s as prevalent resource for the purposes of ITP Debug Mode.", domain.utf8().data());
564 #endif
565 }
566
567 void ResourceLoadStatisticsMemoryStore::scheduleStatisticsProcessingRequestIfNecessary()
568 {
569     ASSERT(!RunLoop::isMain());
570
571     m_pendingStatisticsProcessingRequestIdentifier = ++m_lastStatisticsProcessingRequestIdentifier;
572     m_workQueue->dispatchAfter(minimumStatisticsProcessingInterval, [this, weakThis = makeWeakPtr(*this), statisticsProcessingRequestIdentifier = *m_pendingStatisticsProcessingRequestIdentifier] {
573         if (!weakThis)
574             return;
575
576         if (!m_pendingStatisticsProcessingRequestIdentifier || *m_pendingStatisticsProcessingRequestIdentifier != statisticsProcessingRequestIdentifier) {
577             // This request has been canceled.
578             return;
579         }
580
581         updateCookieBlocking([]() { });
582         processStatisticsAndDataRecords();
583     });
584 }
585
586 void ResourceLoadStatisticsMemoryStore::cancelPendingStatisticsProcessingRequest()
587 {
588     ASSERT(!RunLoop::isMain());
589
590     m_pendingStatisticsProcessingRequestIdentifier = std::nullopt;
591 }
592
593 void ResourceLoadStatisticsMemoryStore::logFrameNavigation(const String& targetPrimaryDomain, const String& mainFramePrimaryDomain, const String& sourcePrimaryDomain, const String& targetHost, const String& mainFrameHost, bool isRedirect, bool isMainFrame)
594 {
595     ASSERT(!RunLoop::isMain());
596
597     bool areTargetAndMainFrameDomainsAssociated = targetPrimaryDomain == mainFramePrimaryDomain;
598     bool areTargetAndSourceDomainsAssociated = targetPrimaryDomain == sourcePrimaryDomain;
599
600     bool statisticsWereUpdated = false;
601     if (!isMainFrame && targetHost != mainFrameHost && !(areTargetAndMainFrameDomainsAssociated || areTargetAndSourceDomainsAssociated)) {
602         auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
603         targetStatistics.lastSeen = ResourceLoadStatistics::reduceTimeResolution(WallTime::now());
604         if (targetStatistics.subframeUnderTopFrameOrigins.add(mainFramePrimaryDomain).isNewEntry)
605             statisticsWereUpdated = true;
606     }
607
608     if (isRedirect && !areTargetAndSourceDomainsAssociated) {
609         if (isMainFrame) {
610             auto& redirectingOriginStatistics = ensureResourceStatisticsForPrimaryDomain(sourcePrimaryDomain);
611             if (redirectingOriginStatistics.topFrameUniqueRedirectsTo.add(targetPrimaryDomain).isNewEntry)
612                 statisticsWereUpdated = true;
613             auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
614             if (targetStatistics.topFrameUniqueRedirectsFrom.add(sourcePrimaryDomain).isNewEntry)
615                 statisticsWereUpdated = true;
616         } else {
617             auto& redirectingOriginStatistics = ensureResourceStatisticsForPrimaryDomain(sourcePrimaryDomain);
618             if (redirectingOriginStatistics.subresourceUniqueRedirectsTo.add(targetPrimaryDomain).isNewEntry)
619                 statisticsWereUpdated = true;
620             auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
621             if (targetStatistics.subresourceUniqueRedirectsFrom.add(sourcePrimaryDomain).isNewEntry)
622                 statisticsWereUpdated = true;
623         }
624     }
625
626     if (statisticsWereUpdated)
627         scheduleStatisticsProcessingRequestIfNecessary();
628 }
629
630 void ResourceLoadStatisticsMemoryStore::logUserInteraction(const String& primaryDomain)
631 {
632     ASSERT(!RunLoop::isMain());
633
634     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
635     statistics.hadUserInteraction = true;
636     statistics.mostRecentUserInteractionTime = WallTime::now();
637 }
638
639 void ResourceLoadStatisticsMemoryStore::clearUserInteraction(const String& primaryDomain)
640 {
641     ASSERT(!RunLoop::isMain());
642
643     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
644     statistics.hadUserInteraction = false;
645     statistics.mostRecentUserInteractionTime = { };
646 }
647
648 bool ResourceLoadStatisticsMemoryStore::hasHadUserInteraction(const String& primaryDomain)
649 {
650     ASSERT(!RunLoop::isMain());
651
652     auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
653     return mapEntry == m_resourceStatisticsMap.end() ? false: hasHadUnexpiredRecentUserInteraction(mapEntry->value);
654 }
655
656 void ResourceLoadStatisticsMemoryStore::setPrevalentResource(WebCore::ResourceLoadStatistics& resourceStatistic, ResourceLoadPrevalence newPrevalence)
657 {
658     ASSERT(!RunLoop::isMain());
659
660     resourceStatistic.isPrevalentResource = true;
661     resourceStatistic.isVeryPrevalentResource = newPrevalence == ResourceLoadPrevalence::VeryHigh;
662     HashSet<String> domainsThatHaveRedirectedTo;
663     recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(resourceStatistic, domainsThatHaveRedirectedTo, 0);
664     for (auto& domain : domainsThatHaveRedirectedTo) {
665         auto mapEntry = m_resourceStatisticsMap.find(domain);
666         if (mapEntry == m_resourceStatisticsMap.end())
667             continue;
668         ASSERT(!mapEntry->value.isPrevalentResource);
669         mapEntry->value.isPrevalentResource = true;
670     }
671 }
672     
673 String ResourceLoadStatisticsMemoryStore::dumpResourceLoadStatistics() const
674 {
675     ASSERT(!RunLoop::isMain());
676
677     StringBuilder result;
678     result.appendLiteral("Resource load statistics:\n\n");
679     for (auto& mapEntry : m_resourceStatisticsMap.values())
680         result.append(mapEntry.toString());
681     return result.toString();
682 }
683
684 bool ResourceLoadStatisticsMemoryStore::isPrevalentResource(const String& primaryDomain) const
685 {
686     ASSERT(!RunLoop::isMain());
687
688     auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
689     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource;
690 }
691
692 bool ResourceLoadStatisticsMemoryStore::isVeryPrevalentResource(const String& primaryDomain) const
693 {
694     ASSERT(!RunLoop::isMain());
695
696     auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
697     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource && mapEntry->value.isVeryPrevalentResource;
698 }
699
700 bool ResourceLoadStatisticsMemoryStore::isRegisteredAsSubresourceUnder(const String& subresourcePrimaryDomain, const String& topFramePrimaryDomain) const
701 {
702     ASSERT(!RunLoop::isMain());
703
704     auto mapEntry = m_resourceStatisticsMap.find(subresourcePrimaryDomain);
705     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subresourceUnderTopFrameOrigins.contains(topFramePrimaryDomain);
706 }
707
708 bool ResourceLoadStatisticsMemoryStore::isRegisteredAsSubFrameUnder(const String& subFramePrimaryDomain, const String& topFramePrimaryDomain) const
709 {
710     ASSERT(!RunLoop::isMain());
711
712     auto mapEntry = m_resourceStatisticsMap.find(subFramePrimaryDomain);
713     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subframeUnderTopFrameOrigins.contains(topFramePrimaryDomain);
714 }
715
716 bool ResourceLoadStatisticsMemoryStore::isRegisteredAsRedirectingTo(const String& hostRedirectedFromPrimaryDomain, const String& hostRedirectedToPrimaryDomain) const
717 {
718     ASSERT(!RunLoop::isMain());
719
720     auto mapEntry = m_resourceStatisticsMap.find(hostRedirectedFromPrimaryDomain);
721     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subresourceUniqueRedirectsTo.contains(hostRedirectedToPrimaryDomain);
722 }
723
724 void ResourceLoadStatisticsMemoryStore::clearPrevalentResource(const String& primaryDomain)
725 {
726     ASSERT(!RunLoop::isMain());
727
728     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
729     statistics.isPrevalentResource = false;
730     statistics.isVeryPrevalentResource = false;
731 }
732
733 void ResourceLoadStatisticsMemoryStore::setGrandfathered(const String& primaryDomain, bool value)
734 {
735     ASSERT(!RunLoop::isMain());
736
737     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
738     statistics.grandfathered = value;
739 }
740
741 bool ResourceLoadStatisticsMemoryStore::isGrandfathered(const String& primaryDomain) const
742 {
743     ASSERT(!RunLoop::isMain());
744
745     auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
746     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.grandfathered;
747 }
748
749 void ResourceLoadStatisticsMemoryStore::setSubframeUnderTopFrameOrigin(const String& primarySubFrameDomain, const String& primaryTopFrameDomain)
750 {
751     ASSERT(!RunLoop::isMain());
752
753     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubFrameDomain);
754     statistics.subframeUnderTopFrameOrigins.add(primaryTopFrameDomain);
755     // For consistency, make sure we also have a statistics entry for the top frame domain.
756     ensureResourceStatisticsForPrimaryDomain(primaryTopFrameDomain);
757 }
758
759 void ResourceLoadStatisticsMemoryStore::setSubresourceUnderTopFrameOrigin(const String& primarySubresourceDomain, const String& primaryTopFrameDomain)
760 {
761     ASSERT(!RunLoop::isMain());
762
763     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
764     statistics.subresourceUnderTopFrameOrigins.add(primaryTopFrameDomain);
765     // For consistency, make sure we also have a statistics entry for the top frame domain.
766     ensureResourceStatisticsForPrimaryDomain(primaryTopFrameDomain);
767 }
768
769 void ResourceLoadStatisticsMemoryStore::setSubresourceUniqueRedirectTo(const String& primarySubresourceDomain, const String& primaryRedirectDomain)
770 {
771     ASSERT(!RunLoop::isMain());
772
773     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
774     statistics.subresourceUniqueRedirectsTo.add(primaryRedirectDomain);
775     // For consistency, make sure we also have a statistics entry for the redirect domain.
776     ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
777 }
778
779 void ResourceLoadStatisticsMemoryStore::setSubresourceUniqueRedirectFrom(const String& primarySubresourceDomain, const String& primaryRedirectDomain)
780 {
781     ASSERT(!RunLoop::isMain());
782
783     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
784     statistics.subresourceUniqueRedirectsFrom.add(primaryRedirectDomain);
785     // For consistency, make sure we also have a statistics entry for the redirect domain.
786     ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
787 }
788
789 void ResourceLoadStatisticsMemoryStore::setTopFrameUniqueRedirectTo(const String& topFramePrimaryDomain, const String& primaryRedirectDomain)
790 {
791     ASSERT(!RunLoop::isMain());
792
793     auto& statistics = ensureResourceStatisticsForPrimaryDomain(topFramePrimaryDomain);
794     statistics.topFrameUniqueRedirectsTo.add(primaryRedirectDomain);
795     // For consistency, make sure we also have a statistics entry for the redirect domain.
796     ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
797 }
798
799 void ResourceLoadStatisticsMemoryStore::setTopFrameUniqueRedirectFrom(const String& topFramePrimaryDomain, const String& primaryRedirectDomain)
800 {
801     ASSERT(!RunLoop::isMain());
802
803     auto& statistics = ensureResourceStatisticsForPrimaryDomain(topFramePrimaryDomain);
804     statistics.topFrameUniqueRedirectsFrom.add(primaryRedirectDomain);
805     // For consistency, make sure we also have a statistics entry for the redirect domain.
806     ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
807 }
808
809 void ResourceLoadStatisticsMemoryStore::setTimeToLiveUserInteraction(Seconds seconds)
810 {
811     ASSERT(!RunLoop::isMain());
812     ASSERT(seconds >= 0_s);
813
814     m_parameters.timeToLiveUserInteraction = seconds;
815 }
816
817 void ResourceLoadStatisticsMemoryStore::setMinimumTimeBetweenDataRecordsRemoval(Seconds seconds)
818 {
819     ASSERT(!RunLoop::isMain());
820     ASSERT(seconds >= 0_s);
821
822     m_parameters.minimumTimeBetweenDataRecordsRemoval = seconds;
823 }
824
825 void ResourceLoadStatisticsMemoryStore::setGrandfatheringTime(Seconds seconds)
826 {
827     ASSERT(!RunLoop::isMain());
828     ASSERT(seconds >= 0_s);
829
830     m_parameters.grandfatheringTime = seconds;
831 }
832
833 void ResourceLoadStatisticsMemoryStore::setCacheMaxAgeCap(Seconds seconds)
834 {
835     ASSERT(!RunLoop::isMain());
836     ASSERT(seconds >= 0_s);
837
838     RunLoop::main().dispatch([store = makeRef(m_store), seconds] () {
839         store->setCacheMaxAgeCap(seconds, [] { });
840     });
841 }
842
843 bool ResourceLoadStatisticsMemoryStore::shouldRemoveDataRecords() const
844 {
845     ASSERT(!RunLoop::isMain());
846
847     if (m_dataRecordsBeingRemoved)
848         return false;
849
850 #if ENABLE(NETSCAPE_PLUGIN_API)
851     for (auto plugin : PluginProcessManager::singleton().pluginProcesses()) {
852         if (!m_activePluginTokens.contains(plugin->pluginProcessToken()))
853             return true;
854     }
855 #endif
856
857     return !m_lastTimeDataRecordsWereRemoved || MonotonicTime::now() >= (m_lastTimeDataRecordsWereRemoved + m_parameters.minimumTimeBetweenDataRecordsRemoval);
858 }
859
860 void ResourceLoadStatisticsMemoryStore::setDataRecordsBeingRemoved(bool value)
861 {
862     ASSERT(!RunLoop::isMain());
863
864     m_dataRecordsBeingRemoved = value;
865     if (m_dataRecordsBeingRemoved)
866         m_lastTimeDataRecordsWereRemoved = MonotonicTime::now();
867 }
868
869 ResourceLoadStatistics& ResourceLoadStatisticsMemoryStore::ensureResourceStatisticsForPrimaryDomain(const String& primaryDomain)
870 {
871     ASSERT(!RunLoop::isMain());
872
873     return m_resourceStatisticsMap.ensure(primaryDomain, [&primaryDomain] {
874         return ResourceLoadStatistics(primaryDomain);
875     }).iterator->value;
876 }
877
878 std::unique_ptr<KeyedEncoder> ResourceLoadStatisticsMemoryStore::createEncoderFromData() const
879 {
880     ASSERT(!RunLoop::isMain());
881
882     auto encoder = KeyedEncoder::encoder();
883     encoder->encodeUInt32("version", statisticsModelVersion);
884     encoder->encodeDouble("endOfGrandfatheringTimestamp", m_endOfGrandfatheringTimestamp.secondsSinceEpoch().value());
885
886     encoder->encodeObjects("browsingStatistics", m_resourceStatisticsMap.begin(), m_resourceStatisticsMap.end(), [](KeyedEncoder& encoderInner, const auto& origin) {
887         origin.value.encode(encoderInner);
888     });
889
890     encoder->encodeObjects("operatingDates", m_operatingDates.begin(), m_operatingDates.end(), [](KeyedEncoder& encoderInner, OperatingDate date) {
891         encoderInner.encodeDouble("date", date.secondsSinceEpoch().value());
892     });
893
894     return encoder;
895 }
896
897 void ResourceLoadStatisticsMemoryStore::mergeWithDataFromDecoder(KeyedDecoder& decoder)
898 {
899     ASSERT(!RunLoop::isMain());
900
901     unsigned versionOnDisk;
902     if (!decoder.decodeUInt32("version", versionOnDisk))
903         return;
904
905     if (versionOnDisk > statisticsModelVersion) {
906         WTFLogAlways("Found resource load statistics on disk with model version %u whereas the highest supported version is %u. Resetting.", versionOnDisk, statisticsModelVersion);
907         return;
908     }
909
910     double endOfGrandfatheringTimestamp;
911     if (decoder.decodeDouble("endOfGrandfatheringTimestamp", endOfGrandfatheringTimestamp))
912         m_endOfGrandfatheringTimestamp = WallTime::fromRawSeconds(endOfGrandfatheringTimestamp);
913     else
914         m_endOfGrandfatheringTimestamp = { };
915
916     Vector<ResourceLoadStatistics> loadedStatistics;
917     bool succeeded = decoder.decodeObjects("browsingStatistics", loadedStatistics, [versionOnDisk](KeyedDecoder& decoderInner, ResourceLoadStatistics& statistics) {
918         return statistics.decode(decoderInner, versionOnDisk);
919     });
920
921     if (!succeeded)
922         return;
923
924     mergeStatistics(WTFMove(loadedStatistics));
925     updateCookieBlocking([]() { });
926
927     Vector<OperatingDate> operatingDates;
928     succeeded = decoder.decodeObjects("operatingDates", operatingDates, [](KeyedDecoder& decoder, OperatingDate& date) {
929         double value;
930         if (!decoder.decodeDouble("date", value))
931             return false;
932
933         date = OperatingDate::fromWallTime(WallTime::fromRawSeconds(value));
934         return true;
935     });
936
937     if (!succeeded)
938         return;
939
940     m_operatingDates = mergeOperatingDates(m_operatingDates, WTFMove(operatingDates));
941 }
942
943 void ResourceLoadStatisticsMemoryStore::clear(CompletionHandler<void()>&& completionHandler)
944 {
945     ASSERT(!RunLoop::isMain());
946
947     m_resourceStatisticsMap.clear();
948     m_operatingDates.clear();
949
950     auto callbackAggregator = CallbackAggregator::create(WTFMove(completionHandler));
951
952     removeAllStorageAccess([callbackAggregator = callbackAggregator.copyRef()] { });
953
954     auto primaryDomainsToBlock = ensurePrevalentResourcesForDebugMode();
955     updateCookieBlockingForDomains(primaryDomainsToBlock, ShouldClearFirst::Yes, [callbackAggregator = callbackAggregator.copyRef()] { });
956 }
957
958 bool ResourceLoadStatisticsMemoryStore::wasAccessedAsFirstPartyDueToUserInteraction(const ResourceLoadStatistics& current, const ResourceLoadStatistics& updated) const
959 {
960     if (!current.hadUserInteraction && !updated.hadUserInteraction)
961         return false;
962
963     auto mostRecentUserInteractionTime = std::max(current.mostRecentUserInteractionTime, updated.mostRecentUserInteractionTime);
964
965     return updated.lastSeen <= mostRecentUserInteractionTime + 24_h;
966 }
967
968 void ResourceLoadStatisticsMemoryStore::mergeStatistics(Vector<ResourceLoadStatistics>&& statistics)
969 {
970     ASSERT(!RunLoop::isMain());
971
972     for (auto& statistic : statistics) {
973         auto result = m_resourceStatisticsMap.ensure(statistic.highLevelDomain, [&statistic] {
974             return WTFMove(statistic);
975         });
976         if (!result.isNewEntry) {
977             if (wasAccessedAsFirstPartyDueToUserInteraction(result.iterator->value, statistic))
978                 result.iterator->value.timesAccessedAsFirstPartyDueToUserInteraction++;
979             result.iterator->value.merge(statistic);
980         }
981     }
982 }
983
984 bool ResourceLoadStatisticsMemoryStore::shouldBlockAndKeepCookies(const ResourceLoadStatistics& statistic)
985 {
986     return statistic.isPrevalentResource && statistic.hadUserInteraction;
987 }
988
989 bool ResourceLoadStatisticsMemoryStore::shouldBlockAndPurgeCookies(const ResourceLoadStatistics& statistic)
990 {
991     return statistic.isPrevalentResource && !statistic.hadUserInteraction;
992 }
993
994 bool ResourceLoadStatisticsMemoryStore::hasUserGrantedStorageAccessThroughPrompt(const ResourceLoadStatistics& statistic, const String& firstPartyPrimaryDomain)
995 {
996     return statistic.storageAccessUnderTopFrameOrigins.contains(firstPartyPrimaryDomain);
997 }
998
999 static void debugLogDomainsInBatches(const char* action, const Vector<String>& domains)
1000 {
1001 #if !RELEASE_LOG_DISABLED
1002     static const auto maxNumberOfDomainsInOneLogStatement = 50;
1003     if (domains.isEmpty())
1004         return;
1005
1006     if (domains.size() <= maxNumberOfDomainsInOneLogStatement) {
1007         RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "About to %{public}s cookies in third-party contexts for: %{public}s.", action, domainsToString(domains).utf8().data());
1008         return;
1009     }
1010     
1011     Vector<String> batch;
1012     batch.reserveInitialCapacity(maxNumberOfDomainsInOneLogStatement);
1013     auto batchNumber = 1;
1014     unsigned numberOfBatches = std::ceil(domains.size() / static_cast<float>(maxNumberOfDomainsInOneLogStatement));
1015
1016     for (auto& domain : domains) {
1017         if (batch.size() == maxNumberOfDomainsInOneLogStatement) {
1018             RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "About to %{public}s cookies in third-party contexts for (%{public}d of %u): %{public}s.", action, batchNumber, numberOfBatches, domainsToString(batch).utf8().data());
1019             batch.shrink(0);
1020             ++batchNumber;
1021         }
1022         batch.append(domain);
1023     }
1024     if (!batch.isEmpty())
1025         RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "About to %{public}s cookies in third-party contexts for (%{public}d of %u): %{public}s.", action, batchNumber, numberOfBatches, domainsToString(batch).utf8().data());
1026 #else
1027     UNUSED_PARAM(action);
1028     UNUSED_PARAM(domains);
1029 #endif
1030 }
1031
1032 void ResourceLoadStatisticsMemoryStore::updateCookieBlocking(CompletionHandler<void()>&& completionHandler)
1033 {
1034     ASSERT(!RunLoop::isMain());
1035
1036     Vector<String> domainsToBlock;
1037     for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
1038         if (resourceStatistic.isPrevalentResource && !resourceStatistic.isMarkedForCookieBlocking) {
1039             domainsToBlock.append(resourceStatistic.highLevelDomain);
1040             resourceStatistic.isMarkedForCookieBlocking = true;
1041         }
1042     }
1043
1044     if (domainsToBlock.isEmpty()) {
1045         completionHandler();
1046         return;
1047     }
1048
1049     if (m_debugLoggingEnabled && !domainsToBlock.isEmpty())
1050             debugLogDomainsInBatches("block", domainsToBlock);
1051
1052     RunLoop::main().dispatch([weakThis = makeWeakPtr(*this), store = makeRef(m_store), domainsToBlock = crossThreadCopy(domainsToBlock), completionHandler = WTFMove(completionHandler)] () mutable {
1053         store->callUpdatePrevalentDomainsToBlockCookiesForHandler(domainsToBlock, ShouldClearFirst::No, [weakThis = WTFMove(weakThis), store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1054             store->statisticsQueue().dispatch([weakThis = WTFMove(weakThis), completionHandler = WTFMove(completionHandler)]() mutable {
1055                 completionHandler();
1056                 if (!weakThis)
1057                     return;
1058 #if !RELEASE_LOG_DISABLED
1059                 RELEASE_LOG_INFO_IF(weakThis->m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Done updating cookie blocking.");
1060 #endif
1061             });
1062         });
1063     });
1064 }
1065
1066 void ResourceLoadStatisticsMemoryStore::updateCookieBlockingForDomains(const Vector<String>& domainsToBlock, ShouldClearFirst shouldClearFirst, CompletionHandler<void()>&& completionHandler)
1067 {
1068     ASSERT(!RunLoop::isMain());
1069
1070     if (domainsToBlock.isEmpty() && shouldClearFirst == ShouldClearFirst::No) {
1071         completionHandler();
1072         return;
1073     }
1074
1075     if (shouldClearFirst == ShouldClearFirst::Yes)
1076         resetCookieBlockingState();
1077
1078     for (auto& domain : domainsToBlock)
1079         ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookieBlocking = true;
1080
1081     RunLoop::main().dispatch([store = makeRef(m_store), shouldClearFirst, domainsToBlock = crossThreadCopy(domainsToBlock), completionHandler = WTFMove(completionHandler)] () mutable {
1082         store->callUpdatePrevalentDomainsToBlockCookiesForHandler(domainsToBlock, shouldClearFirst, [store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1083             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler)]() mutable {
1084                 completionHandler();
1085             });
1086         });
1087     });
1088 }
1089
1090 void ResourceLoadStatisticsMemoryStore::clearBlockingStateForDomains(const Vector<String>& domains, CompletionHandler<void()>&& completionHandler)
1091 {
1092     ASSERT(!RunLoop::isMain());
1093
1094     if (domains.isEmpty()) {
1095         completionHandler();
1096         return;
1097     }
1098
1099     RunLoop::main().dispatch([store = makeRef(m_store), domains = crossThreadCopy(domains)] {
1100         store->callRemoveDomainsHandler(domains);
1101     });
1102
1103     for (auto& domain : domains) {
1104         auto& statistic = ensureResourceStatisticsForPrimaryDomain(domain);
1105         statistic.isMarkedForCookieBlocking = false;
1106     }
1107
1108     completionHandler();
1109 }
1110
1111 void ResourceLoadStatisticsMemoryStore::resetCookieBlockingState()
1112 {
1113     ASSERT(!RunLoop::isMain());
1114
1115     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
1116         resourceStatistic.isMarkedForCookieBlocking = false;
1117 }
1118
1119 void ResourceLoadStatisticsMemoryStore::processStatistics(const Function<void(const ResourceLoadStatistics&)>& processFunction) const
1120 {
1121     ASSERT(!RunLoop::isMain());
1122
1123     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
1124         processFunction(resourceStatistic);
1125 }
1126
1127 bool ResourceLoadStatisticsMemoryStore::hasHadUnexpiredRecentUserInteraction(ResourceLoadStatistics& resourceStatistic) const
1128 {
1129     ASSERT(!RunLoop::isMain());
1130
1131     if (resourceStatistic.hadUserInteraction && hasStatisticsExpired(resourceStatistic)) {
1132         // Drop privacy sensitive data because we no longer need it.
1133         // Set timestamp to 0 so that statistics merge will know
1134         // it has been reset as opposed to its default -1.
1135         resourceStatistic.mostRecentUserInteractionTime = { };
1136         resourceStatistic.storageAccessUnderTopFrameOrigins.clear();
1137         resourceStatistic.hadUserInteraction = false;
1138     }
1139
1140     return resourceStatistic.hadUserInteraction;
1141 }
1142
1143 Vector<String> ResourceLoadStatisticsMemoryStore::topPrivatelyControlledDomainsToRemoveWebsiteDataFor()
1144 {
1145     ASSERT(!RunLoop::isMain());
1146
1147     bool shouldCheckForGrandfathering = m_endOfGrandfatheringTimestamp > WallTime::now();
1148     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && m_endOfGrandfatheringTimestamp;
1149
1150     if (shouldClearGrandfathering)
1151         m_endOfGrandfatheringTimestamp = { };
1152
1153     Vector<String> prevalentResources;
1154     for (auto& statistic : m_resourceStatisticsMap.values()) {
1155         if (statistic.isPrevalentResource && !hasHadUnexpiredRecentUserInteraction(statistic) && (!shouldCheckForGrandfathering || !statistic.grandfathered))
1156             prevalentResources.append(statistic.highLevelDomain);
1157
1158         if (shouldClearGrandfathering && statistic.grandfathered)
1159             statistic.grandfathered = false;
1160     }
1161
1162     return prevalentResources;
1163 }
1164
1165 void ResourceLoadStatisticsMemoryStore::includeTodayAsOperatingDateIfNecessary()
1166 {
1167     ASSERT(!RunLoop::isMain());
1168
1169     auto today = OperatingDate::today();
1170     if (!m_operatingDates.isEmpty() && today <= m_operatingDates.last())
1171         return;
1172
1173     while (m_operatingDates.size() >= operatingDatesWindow)
1174         m_operatingDates.remove(0);
1175
1176     m_operatingDates.append(today);
1177 }
1178
1179 bool ResourceLoadStatisticsMemoryStore::hasStatisticsExpired(const ResourceLoadStatistics& resourceStatistic) const
1180 {
1181     ASSERT(!RunLoop::isMain());
1182
1183     if (m_operatingDates.size() >= operatingDatesWindow) {
1184         if (OperatingDate::fromWallTime(resourceStatistic.mostRecentUserInteractionTime) < m_operatingDates.first())
1185             return true;
1186     }
1187
1188     // If we don't meet the real criteria for an expired statistic, check the user setting for a tighter restriction (mainly for testing).
1189     if (m_parameters.timeToLiveUserInteraction) {
1190         if (WallTime::now() > resourceStatistic.mostRecentUserInteractionTime + m_parameters.timeToLiveUserInteraction.value())
1191             return true;
1192     }
1193
1194     return false;
1195 }
1196
1197 void ResourceLoadStatisticsMemoryStore::setMaxStatisticsEntries(size_t maximumEntryCount)
1198 {
1199     ASSERT(!RunLoop::isMain());
1200
1201     m_parameters.maxStatisticsEntries = maximumEntryCount;
1202 }
1203
1204 void ResourceLoadStatisticsMemoryStore::setPruneEntriesDownTo(size_t pruneTargetCount)
1205 {
1206     ASSERT(!RunLoop::isMain());
1207
1208     m_parameters.pruneEntriesDownTo = pruneTargetCount;
1209 }
1210
1211 void ResourceLoadStatisticsMemoryStore::pruneStatisticsIfNeeded()
1212 {
1213     ASSERT(!RunLoop::isMain());
1214
1215     if (m_resourceStatisticsMap.size() <= m_parameters.maxStatisticsEntries)
1216         return;
1217
1218     ASSERT(m_parameters.pruneEntriesDownTo <= m_parameters.maxStatisticsEntries);
1219
1220     size_t numberOfEntriesLeftToPrune = m_resourceStatisticsMap.size() - m_parameters.pruneEntriesDownTo;
1221     ASSERT(numberOfEntriesLeftToPrune);
1222
1223     Vector<StatisticsLastSeen> resourcesToPrunePerImportance[maxImportance + 1];
1224     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
1225         resourcesToPrunePerImportance[computeImportance(resourceStatistic)].append({ resourceStatistic.highLevelDomain, resourceStatistic.lastSeen });
1226
1227     for (unsigned importance = 0; numberOfEntriesLeftToPrune && importance <= maxImportance; ++importance)
1228         pruneResources(m_resourceStatisticsMap, resourcesToPrunePerImportance[importance], numberOfEntriesLeftToPrune);
1229
1230     ASSERT(!numberOfEntriesLeftToPrune);
1231 }
1232
1233 void ResourceLoadStatisticsMemoryStore::resetParametersToDefaultValues()
1234 {
1235     ASSERT(!RunLoop::isMain());
1236
1237     m_parameters = { };
1238 }
1239
1240 void ResourceLoadStatisticsMemoryStore::logTestingEvent(const String& event)
1241 {
1242     ASSERT(!RunLoop::isMain());
1243
1244     RunLoop::main().dispatch([store = makeRef(m_store), event = event.isolatedCopy()] {
1245         store->logTestingEvent(event);
1246     });
1247 }
1248
1249 void ResourceLoadStatisticsMemoryStore::setLastSeen(const String& primaryDomain, Seconds seconds)
1250 {
1251     ASSERT(!RunLoop::isMain());
1252
1253     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
1254     statistics.lastSeen = WallTime::fromRawSeconds(seconds.seconds());
1255 }
1256
1257 void ResourceLoadStatisticsMemoryStore::setPrevalentResource(const String& primaryDomain)
1258 {
1259     ASSERT(!RunLoop::isMain());
1260
1261     auto& resourceStatistic = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
1262     setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
1263 }
1264
1265 void ResourceLoadStatisticsMemoryStore::setVeryPrevalentResource(const String& primaryDomain)
1266 {
1267     ASSERT(!RunLoop::isMain());
1268
1269     auto& resourceStatistic = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
1270     setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::VeryHigh);
1271 }
1272
1273 void ResourceLoadStatisticsMemoryStore::removeAllStorageAccess(CompletionHandler<void()>&& completionHandler)
1274 {
1275     ASSERT(!RunLoop::isMain());
1276     RunLoop::main().dispatch([store = makeRef(m_store), completionHandler = WTFMove(completionHandler)]() mutable {
1277         store->removeAllStorageAccess([store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1278             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler)]() mutable {
1279                 completionHandler();
1280             });
1281         });
1282     });
1283 }
1284
1285 } // namespace WebKit