Adopt WebCore::RegistrableDomain in WebCore::ResourceLoadStatistics and WebKit::Netwo...
[WebKit-https.git] / Source / WebKit / NetworkProcess / Classifier / ResourceLoadStatisticsMemoryStore.cpp
1 /*
2  * Copyright (C) 2017-2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ResourceLoadStatisticsMemoryStore.h"
28
29 #if ENABLE(RESOURCE_LOAD_STATISTICS)
30
31 #include "Logging.h"
32 #include "NetworkSession.h"
33 #include "PluginProcessManager.h"
34 #include "PluginProcessProxy.h"
35 #include "ResourceLoadStatisticsPersistentStorage.h"
36 #include "StorageAccessStatus.h"
37 #include "WebProcessProxy.h"
38 #include "WebResourceLoadStatisticsTelemetry.h"
39 #include "WebsiteDataStore.h"
40 #include <WebCore/KeyedCoding.h>
41 #include <WebCore/NetworkStorageSession.h>
42 #include <WebCore/ResourceLoadStatistics.h>
43 #include <wtf/CallbackAggregator.h>
44 #include <wtf/DateMath.h>
45 #include <wtf/MathExtras.h>
46 #include <wtf/text/StringBuilder.h>
47
48 namespace WebKit {
49 using namespace WebCore;
50
51 constexpr unsigned statisticsModelVersion { 15 };
52 constexpr unsigned maxNumberOfRecursiveCallsInRedirectTraceBack { 50 };
53 constexpr Seconds minimumStatisticsProcessingInterval { 5_s };
54 constexpr unsigned operatingDatesWindow { 30 };
55 constexpr unsigned maxImportance { 3 };
56
57 #if !RELEASE_LOG_DISABLED
58 static String domainsToString(const Vector<RegistrableDomain>& domains)
59 {
60     StringBuilder builder;
61     for (auto& domain : domains) {
62         if (!builder.isEmpty())
63             builder.appendLiteral(", ");
64         builder.append(domain.string());
65     }
66     return builder.toString();
67 }
68 #endif
69
70 class OperatingDate {
71 public:
72     OperatingDate() = default;
73
74     static OperatingDate fromWallTime(WallTime time)
75     {
76         double ms = time.secondsSinceEpoch().milliseconds();
77         int year = msToYear(ms);
78         int yearDay = dayInYear(ms, year);
79         int month = monthFromDayInYear(yearDay, isLeapYear(year));
80         int monthDay = dayInMonthFromDayInYear(yearDay, isLeapYear(year));
81
82         return OperatingDate { year, month, monthDay };
83     }
84
85     static OperatingDate today()
86     {
87         return OperatingDate::fromWallTime(WallTime::now());
88     }
89
90     Seconds secondsSinceEpoch() const
91     {
92         return Seconds { dateToDaysFrom1970(m_year, m_month, m_monthDay) * secondsPerDay };
93     }
94
95     bool operator==(const OperatingDate& other) const
96     {
97         return m_monthDay == other.m_monthDay && m_month == other.m_month && m_year == other.m_year;
98     }
99
100     bool operator<(const OperatingDate& other) const
101     {
102         return secondsSinceEpoch() < other.secondsSinceEpoch();
103     }
104
105     bool operator<=(const OperatingDate& other) const
106     {
107         return secondsSinceEpoch() <= other.secondsSinceEpoch();
108     }
109
110 private:
111     OperatingDate(int year, int month, int monthDay)
112         : m_year(year)
113         , m_month(month)
114         , m_monthDay(monthDay)
115     { }
116
117     int m_year { 0 };
118     int m_month { 0 }; // [0, 11].
119     int m_monthDay { 0 }; // [1, 31].
120 };
121
122 static Vector<OperatingDate> mergeOperatingDates(const Vector<OperatingDate>& existingDates, Vector<OperatingDate>&& newDates)
123 {
124     if (existingDates.isEmpty())
125         return WTFMove(newDates);
126
127     Vector<OperatingDate> mergedDates(existingDates.size() + newDates.size());
128
129     // Merge the two sorted vectors of dates.
130     std::merge(existingDates.begin(), existingDates.end(), newDates.begin(), newDates.end(), mergedDates.begin());
131     // Remove duplicate dates.
132     removeRepeatedElements(mergedDates);
133
134     // Drop old dates until the Vector size reaches operatingDatesWindow.
135     while (mergedDates.size() > operatingDatesWindow)
136         mergedDates.remove(0);
137
138     return mergedDates;
139 }
140
141 struct StatisticsLastSeen {
142     RegistrableDomain domain;
143     WallTime lastSeen;
144 };
145
146 static void pruneResources(HashMap<RegistrableDomain, ResourceLoadStatistics>& statisticsMap, Vector<StatisticsLastSeen>& statisticsToPrune, size_t& numberOfEntriesToPrune)
147 {
148     if (statisticsToPrune.size() > numberOfEntriesToPrune) {
149         std::sort(statisticsToPrune.begin(), statisticsToPrune.end(), [](const StatisticsLastSeen& a, const StatisticsLastSeen& b) {
150             return a.lastSeen < b.lastSeen;
151         });
152     }
153
154     for (size_t i = 0, end = std::min(numberOfEntriesToPrune, statisticsToPrune.size()); i != end; ++i, --numberOfEntriesToPrune)
155         statisticsMap.remove(statisticsToPrune[i].domain);
156 }
157
158 static unsigned computeImportance(const ResourceLoadStatistics& resourceStatistic)
159 {
160     unsigned importance = maxImportance;
161     if (!resourceStatistic.isPrevalentResource)
162         importance -= 1;
163     if (!resourceStatistic.hadUserInteraction)
164         importance -= 2;
165     return importance;
166 }
167
168 ResourceLoadStatisticsMemoryStore::ResourceLoadStatisticsMemoryStore(WebResourceLoadStatisticsStore& store, WorkQueue& workQueue)
169     : m_store(store)
170     , m_workQueue(workQueue)
171 {
172     ASSERT(!RunLoop::isMain());
173
174 #if PLATFORM(COCOA)
175     registerUserDefaultsIfNeeded();
176 #endif
177     includeTodayAsOperatingDateIfNecessary();
178
179     m_workQueue->dispatchAfter(5_s, [weakThis = makeWeakPtr(*this)] {
180         if (weakThis)
181             weakThis->calculateAndSubmitTelemetry();
182     });
183 }
184
185 ResourceLoadStatisticsMemoryStore::~ResourceLoadStatisticsMemoryStore()
186 {
187     ASSERT(!RunLoop::isMain());
188 }
189
190 void ResourceLoadStatisticsMemoryStore::setPersistentStorage(ResourceLoadStatisticsPersistentStorage& persistentStorage)
191 {
192     m_persistentStorage = makeWeakPtr(persistentStorage);
193 }
194
195 void ResourceLoadStatisticsMemoryStore::calculateAndSubmitTelemetry() const
196 {
197     ASSERT(!RunLoop::isMain());
198
199     if (m_parameters.shouldSubmitTelemetry)
200         WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
201 }
202
203 void ResourceLoadStatisticsMemoryStore::setNotifyPagesWhenDataRecordsWereScanned(bool value)
204 {
205     ASSERT(!RunLoop::isMain());
206     m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned = value;
207 }
208
209 void ResourceLoadStatisticsMemoryStore::setShouldClassifyResourcesBeforeDataRecordsRemoval(bool value)
210 {
211     ASSERT(!RunLoop::isMain());
212     m_parameters.shouldClassifyResourcesBeforeDataRecordsRemoval = value;
213 }
214
215 void ResourceLoadStatisticsMemoryStore::setShouldSubmitTelemetry(bool value)
216 {
217     ASSERT(!RunLoop::isMain());
218     m_parameters.shouldSubmitTelemetry = value;
219 }
220
221 void ResourceLoadStatisticsMemoryStore::removeDataRecords(CompletionHandler<void()>&& callback)
222 {
223     ASSERT(!RunLoop::isMain());
224
225     if (!shouldRemoveDataRecords()) {
226         callback();
227         return;
228     }
229
230 #if ENABLE(NETSCAPE_PLUGIN_API)
231     m_activePluginTokens.clear();
232     for (const auto& plugin : PluginProcessManager::singleton().pluginProcesses())
233         m_activePluginTokens.add(plugin->pluginProcessToken());
234 #endif
235
236     auto prevalentResourceDomains = registrableDomainsToRemoveWebsiteDataFor();
237     if (prevalentResourceDomains.isEmpty()) {
238         callback();
239         return;
240     }
241
242 #if !RELEASE_LOG_DISABLED
243     RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "About to remove data records for %{public}s.", domainsToString(prevalentResourceDomains).utf8().data());
244 #endif
245
246     setDataRecordsBeingRemoved(true);
247
248     RunLoop::main().dispatch([prevalentResourceDomains = crossThreadCopy(prevalentResourceDomains), callback = WTFMove(callback), weakThis = makeWeakPtr(*this), shouldNotifyPagesWhenDataRecordsWereScanned = m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, workQueue = m_workQueue.copyRef()] () mutable {
249         if (!weakThis) {
250             callback();
251             return;
252         }
253
254         weakThis->m_store.deleteWebsiteDataForRegistrableDomainsInAllPersistentDataStores(WebResourceLoadStatisticsStore::monitoredDataTypes(), WTFMove(prevalentResourceDomains), shouldNotifyPagesWhenDataRecordsWereScanned, [callback = WTFMove(callback), weakThis = WTFMove(weakThis), workQueue = workQueue.copyRef()](const HashSet<RegistrableDomain>& domainsWithDeletedWebsiteData) mutable {
255             workQueue->dispatch([domainsWithDeletedWebsiteData = crossThreadCopy(domainsWithDeletedWebsiteData), callback = WTFMove(callback), weakThis = WTFMove(weakThis)] () mutable {
256                 if (!weakThis) {
257                     callback();
258                     return;
259                 }
260                 for (auto& domain : domainsWithDeletedWebsiteData) {
261                     auto& statistic = weakThis->ensureResourceStatisticsForRegistrableDomain(domain);
262                     ++statistic.dataRecordsRemoved;
263                 }
264                 weakThis->setDataRecordsBeingRemoved(false);
265                 callback();
266 #if !RELEASE_LOG_DISABLED
267                 RELEASE_LOG_INFO_IF(weakThis->m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Done removing data records.");
268 #endif
269             });
270         });
271     });
272 }
273
274 unsigned ResourceLoadStatisticsMemoryStore::recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(const ResourceLoadStatistics& resourceStatistic, HashSet<RegistrableDomain>& domainsThatHaveRedirectedTo, unsigned numberOfRecursiveCalls) const
275 {
276     ASSERT(!RunLoop::isMain());
277
278     if (numberOfRecursiveCalls >= maxNumberOfRecursiveCallsInRedirectTraceBack) {
279         // Model version 14 invokes a deliberate re-classification of the whole set.
280         if (statisticsModelVersion != 14)
281             ASSERT_NOT_REACHED();
282         RELEASE_LOG(ResourceLoadStatistics, "Hit %u recursive calls in redirect backtrace. Returning early.", maxNumberOfRecursiveCallsInRedirectTraceBack);
283         return numberOfRecursiveCalls;
284     }
285
286     numberOfRecursiveCalls++;
287
288     for (auto& subresourceUniqueRedirectFromDomain : resourceStatistic.subresourceUniqueRedirectsFrom) {
289         auto mapEntry = m_resourceStatisticsMap.find(RegistrableDomain { subresourceUniqueRedirectFromDomain });
290         if (mapEntry == m_resourceStatisticsMap.end() || mapEntry->value.isPrevalentResource)
291             continue;
292         if (domainsThatHaveRedirectedTo.add(mapEntry->value.registrableDomain).isNewEntry)
293             numberOfRecursiveCalls = recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(mapEntry->value, domainsThatHaveRedirectedTo, numberOfRecursiveCalls);
294     }
295     for (auto& topFrameUniqueRedirectFromDomain : resourceStatistic.topFrameUniqueRedirectsFrom) {
296         auto mapEntry = m_resourceStatisticsMap.find(RegistrableDomain { topFrameUniqueRedirectFromDomain });
297         if (mapEntry == m_resourceStatisticsMap.end() || mapEntry->value.isPrevalentResource)
298             continue;
299         if (domainsThatHaveRedirectedTo.add(mapEntry->value.registrableDomain).isNewEntry)
300             numberOfRecursiveCalls = recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(mapEntry->value, domainsThatHaveRedirectedTo, numberOfRecursiveCalls);
301     }
302
303     return numberOfRecursiveCalls;
304 }
305
306 void ResourceLoadStatisticsMemoryStore::markAsPrevalentIfHasRedirectedToPrevalent(ResourceLoadStatistics& resourceStatistic)
307 {
308     ASSERT(!RunLoop::isMain());
309
310     if (resourceStatistic.isPrevalentResource)
311         return;
312
313     for (auto& subresourceDomainRedirectedTo : resourceStatistic.subresourceUniqueRedirectsTo) {
314         auto mapEntry = m_resourceStatisticsMap.find(RegistrableDomain { subresourceDomainRedirectedTo });
315         if (mapEntry != m_resourceStatisticsMap.end() && mapEntry->value.isPrevalentResource) {
316             setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
317             return;
318         }
319     }
320
321     for (auto& topFrameDomainRedirectedTo : resourceStatistic.topFrameUniqueRedirectsTo) {
322         auto mapEntry = m_resourceStatisticsMap.find(RegistrableDomain { topFrameDomainRedirectedTo });
323         if (mapEntry != m_resourceStatisticsMap.end() && mapEntry->value.isPrevalentResource) {
324             setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
325             return;
326         }
327     }
328 }
329
330 bool ResourceLoadStatisticsMemoryStore::isPrevalentDueToDebugMode(ResourceLoadStatistics& resourceStatistic)
331 {
332     if (!m_debugModeEnabled)
333         return false;
334
335     return resourceStatistic.registrableDomain == m_debugStaticPrevalentResource || resourceStatistic.registrableDomain == m_debugManualPrevalentResource;
336 }
337
338 void ResourceLoadStatisticsMemoryStore::processStatisticsAndDataRecords()
339 {
340     ASSERT(!RunLoop::isMain());
341
342     if (m_parameters.shouldClassifyResourcesBeforeDataRecordsRemoval) {
343         for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
344             if (isPrevalentDueToDebugMode(resourceStatistic))
345                 setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
346             else if (!resourceStatistic.isVeryPrevalentResource) {
347                 markAsPrevalentIfHasRedirectedToPrevalent(resourceStatistic);
348                 auto currentPrevalence = resourceStatistic.isPrevalentResource ? ResourceLoadPrevalence::High : ResourceLoadPrevalence::Low;
349                 auto newPrevalence = m_resourceLoadStatisticsClassifier.calculateResourcePrevalence(resourceStatistic, currentPrevalence);
350                 if (newPrevalence != currentPrevalence)
351                     setPrevalentResource(resourceStatistic, newPrevalence);
352             }
353         }
354     }
355
356     removeDataRecords([this, weakThis = makeWeakPtr(*this)]() mutable {
357         ASSERT(!RunLoop::isMain());
358         if (!weakThis)
359             return;
360
361         pruneStatisticsIfNeeded();
362         if (m_persistentStorage)
363             m_persistentStorage->scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::No);
364
365         if (!m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned)
366             return;
367
368         RunLoop::main().dispatch([this, weakThis = WTFMove(weakThis)]() {
369             ASSERT(RunLoop::isMain());
370             if (!weakThis)
371                 return;
372
373             m_store.notifyResourceLoadStatisticsProcessed();
374         });
375     });
376 }
377
378 void ResourceLoadStatisticsMemoryStore::hasStorageAccess(const RegistrableDomain& subFrameDomain, const RegistrableDomain& topFrameDomain, Optional<uint64_t> frameID, uint64_t pageID, CompletionHandler<void(bool)>&& completionHandler)
379 {
380     ASSERT(!RunLoop::isMain());
381
382     auto& subFrameStatistic = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
383     if (shouldBlockAndPurgeCookies(subFrameStatistic)) {
384         completionHandler(false);
385         return;
386     }
387
388     if (!shouldBlockAndKeepCookies(subFrameStatistic)) {
389         completionHandler(true);
390         return;
391     }
392
393     RunLoop::main().dispatch([store = makeRef(m_store), subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, completionHandler = WTFMove(completionHandler)]() mutable {
394         store->callHasStorageAccessForFrameHandler(subFrameDomain, topFrameDomain, frameID.value(), pageID, [store = store.copyRef(), completionHandler = WTFMove(completionHandler)](bool result) mutable {
395             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler), result] () mutable {
396                 completionHandler(result);
397             });
398         });
399     });
400 }
401
402 void ResourceLoadStatisticsMemoryStore::requestStorageAccess(RegistrableDomain&& subFrameDomain, RegistrableDomain&& topFrameDomain, uint64_t frameID, uint64_t pageID, bool promptEnabled, CompletionHandler<void(StorageAccessStatus)>&& completionHandler)
403 {
404     ASSERT(!RunLoop::isMain());
405
406     auto& subFrameStatistic = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
407     if (shouldBlockAndPurgeCookies(subFrameStatistic)) {
408 #if !RELEASE_LOG_DISABLED
409         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Cannot grant storage access to %{public}s since its cookies are blocked in third-party contexts and it has not received user interaction as first-party.", subFrameDomain.string().utf8().data());
410 #endif
411         completionHandler(StorageAccessStatus::CannotRequestAccess);
412         return;
413     }
414
415     if (!shouldBlockAndKeepCookies(subFrameStatistic)) {
416 #if !RELEASE_LOG_DISABLED
417         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "No need to grant storage access to %{public}s since its cookies are not blocked in third-party contexts.", subFrameDomain.string().utf8().data());
418 #endif
419         completionHandler(StorageAccessStatus::HasAccess);
420         return;
421     }
422
423     auto userWasPromptedEarlier = promptEnabled && hasUserGrantedStorageAccessThroughPrompt(subFrameStatistic, topFrameDomain);
424     if (promptEnabled && !userWasPromptedEarlier) {
425 #if !RELEASE_LOG_DISABLED
426         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "About to ask the user whether they want to grant storage access to %{public}s under %{public}s or not.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
427 #endif
428         completionHandler(StorageAccessStatus::RequiresUserPrompt);
429         return;
430     } else if (userWasPromptedEarlier) {
431 #if !RELEASE_LOG_DISABLED
432         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Storage access was granted to %{public}s under %{public}s.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
433 #endif
434     }
435
436     subFrameStatistic.timesAccessedAsFirstPartyDueToStorageAccessAPI++;
437
438     grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, userWasPromptedEarlier, [completionHandler = WTFMove(completionHandler)] (bool wasGrantedAccess) mutable {
439         completionHandler(wasGrantedAccess ? StorageAccessStatus::HasAccess : StorageAccessStatus::CannotRequestAccess);
440     });
441 }
442
443 void ResourceLoadStatisticsMemoryStore::requestStorageAccessUnderOpener(RegistrableDomain&& domainInNeedOfStorageAccess, uint64_t openerPageID, RegistrableDomain&& openerDomain)
444 {
445     ASSERT(domainInNeedOfStorageAccess != openerDomain);
446     ASSERT(!RunLoop::isMain());
447
448     if (domainInNeedOfStorageAccess == openerDomain)
449         return;
450
451     auto& domainInNeedOfStorageAccessStatistic = ensureResourceStatisticsForRegistrableDomain(domainInNeedOfStorageAccess);
452     auto cookiesBlockedAndPurged = shouldBlockAndPurgeCookies(domainInNeedOfStorageAccessStatistic);
453
454     // The domain already has access if its cookies are not blocked.
455     if (!cookiesBlockedAndPurged && !shouldBlockAndKeepCookies(domainInNeedOfStorageAccessStatistic))
456         return;
457
458 #if !RELEASE_LOG_DISABLED
459     RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "[Temporary combatibility fix] Storage access was granted for %{public}s under opener page from %{public}s, with user interaction in the opened window.", domainInNeedOfStorageAccess.string().utf8().data(), openerDomain.string().utf8().data());
460 #endif
461     grantStorageAccessInternal(WTFMove(domainInNeedOfStorageAccess), WTFMove(openerDomain), WTF::nullopt, openerPageID, false, [](bool) { });
462 }
463
464 void ResourceLoadStatisticsMemoryStore::grantStorageAccess(RegistrableDomain&& subFrameDomain, RegistrableDomain&& topFrameDomain, uint64_t frameID, uint64_t pageID, bool userWasPromptedNow, CompletionHandler<void(bool)>&& completionHandler)
465 {
466     ASSERT(!RunLoop::isMain());
467
468     if (userWasPromptedNow) {
469         auto& subFrameStatistic = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
470         ASSERT(subFrameStatistic.hadUserInteraction);
471         subFrameStatistic.storageAccessUnderTopFrameDomains.add(topFrameDomain);
472     }
473     grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, userWasPromptedNow, WTFMove(completionHandler));
474 }
475
476 void ResourceLoadStatisticsMemoryStore::grantStorageAccessInternal(RegistrableDomain&& subFrameDomain, RegistrableDomain&& topFrameDomain, Optional<uint64_t> frameID, uint64_t pageID, bool userWasPromptedNowOrEarlier, CompletionHandler<void(bool)>&& callback)
477 {
478     ASSERT(!RunLoop::isMain());
479
480     if (subFrameDomain == topFrameDomain) {
481         callback(true);
482         return;
483     }
484
485     // FIXME: Remove m_storageAccessPromptsEnabled check if prompting is no longer experimental.
486     if (userWasPromptedNowOrEarlier && m_storageAccessPromptsEnabled) {
487         auto& subFrameStatistic = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
488         ASSERT(subFrameStatistic.hadUserInteraction);
489         ASSERT(subFrameStatistic.storageAccessUnderTopFrameDomains.contains(topFrameDomain));
490         subFrameStatistic.mostRecentUserInteractionTime = WallTime::now();
491     }
492
493     RunLoop::main().dispatch([subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, store = makeRef(m_store), callback = WTFMove(callback)]() mutable {
494         store->callGrantStorageAccessHandler(subFrameDomain, topFrameDomain, frameID, pageID, [callback = WTFMove(callback), store = store.copyRef()](bool value) mutable {
495             store->statisticsQueue().dispatch([callback = WTFMove(callback), value] () mutable {
496                 callback(value);
497             });
498         });
499     });
500 }
501
502 void ResourceLoadStatisticsMemoryStore::grandfatherExistingWebsiteData(CompletionHandler<void()>&& callback)
503 {
504     ASSERT(!RunLoop::isMain());
505
506     RunLoop::main().dispatch([weakThis = makeWeakPtr(*this), callback = WTFMove(callback), shouldNotifyPagesWhenDataRecordsWereScanned = m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, workQueue = m_workQueue.copyRef(), store = makeRef(m_store)] () mutable {
507         store->registrableDomainsWithWebsiteData(WebResourceLoadStatisticsStore::monitoredDataTypes(), shouldNotifyPagesWhenDataRecordsWereScanned, [weakThis = WTFMove(weakThis), callback = WTFMove(callback), workQueue = workQueue.copyRef()] (HashSet<RegistrableDomain>&& domainsWithWebsiteData) mutable {
508             workQueue->dispatch([weakThis = WTFMove(weakThis), domainsWithWebsiteData = crossThreadCopy(domainsWithWebsiteData), callback = WTFMove(callback)] () mutable {
509                 if (!weakThis) {
510                     callback();
511                     return;
512                 }
513
514                 for (auto& domain : domainsWithWebsiteData) {
515                     auto& statistic = weakThis->ensureResourceStatisticsForRegistrableDomain(domain);
516                     statistic.grandfathered = true;
517                 }
518                 weakThis->m_endOfGrandfatheringTimestamp = WallTime::now() + weakThis->m_parameters.grandfatheringTime;
519                 if (weakThis->m_persistentStorage)
520                     weakThis->m_persistentStorage->scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::Yes);
521                 callback();
522                 weakThis->logTestingEvent("Grandfathered"_s);
523             });
524         });
525     });
526 }
527
528 Vector<RegistrableDomain> ResourceLoadStatisticsMemoryStore::ensurePrevalentResourcesForDebugMode()
529 {
530     if (!m_debugModeEnabled)
531         return { };
532
533     Vector<RegistrableDomain> domainsToBlock;
534     domainsToBlock.reserveInitialCapacity(2);
535
536     auto& staticSesourceStatistic = ensureResourceStatisticsForRegistrableDomain(m_debugStaticPrevalentResource);
537     setPrevalentResource(staticSesourceStatistic, ResourceLoadPrevalence::High);
538     domainsToBlock.uncheckedAppend(m_debugStaticPrevalentResource);
539
540     if (!m_debugManualPrevalentResource.isEmpty()) {
541         auto& manualResourceStatistic = ensureResourceStatisticsForRegistrableDomain(m_debugManualPrevalentResource);
542         setPrevalentResource(manualResourceStatistic, ResourceLoadPrevalence::High);
543         domainsToBlock.uncheckedAppend(m_debugManualPrevalentResource);
544 #if !RELEASE_LOG_DISABLED
545         RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "Did set %{public}s as prevalent resource for the purposes of ITP Debug Mode.", m_debugManualPrevalentResource.string().utf8().data());
546 #endif
547     }
548     
549     return domainsToBlock;
550 }
551
552 void ResourceLoadStatisticsMemoryStore::setResourceLoadStatisticsDebugMode(bool enable)
553 {
554     ASSERT(!RunLoop::isMain());
555
556 #if !RELEASE_LOG_DISABLED
557     if (enable)
558         RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "Turned ITP Debug Mode on.");
559 #endif
560
561     m_debugModeEnabled = enable;
562     m_debugLoggingEnabled = enable;
563
564     ensurePrevalentResourcesForDebugMode();
565     // This will log the current cookie blocking state.
566     if (enable)
567         updateCookieBlocking([]() { });
568 }
569
570 void ResourceLoadStatisticsMemoryStore::setPrevalentResourceForDebugMode(const RegistrableDomain& domain)
571 {
572     m_debugManualPrevalentResource = domain;
573 }
574
575 void ResourceLoadStatisticsMemoryStore::scheduleStatisticsProcessingRequestIfNecessary()
576 {
577     ASSERT(!RunLoop::isMain());
578
579     m_pendingStatisticsProcessingRequestIdentifier = ++m_lastStatisticsProcessingRequestIdentifier;
580     m_workQueue->dispatchAfter(minimumStatisticsProcessingInterval, [this, weakThis = makeWeakPtr(*this), statisticsProcessingRequestIdentifier = *m_pendingStatisticsProcessingRequestIdentifier] {
581         if (!weakThis)
582             return;
583
584         if (!m_pendingStatisticsProcessingRequestIdentifier || *m_pendingStatisticsProcessingRequestIdentifier != statisticsProcessingRequestIdentifier) {
585             // This request has been canceled.
586             return;
587         }
588
589         updateCookieBlocking([]() { });
590         processStatisticsAndDataRecords();
591     });
592 }
593
594 void ResourceLoadStatisticsMemoryStore::cancelPendingStatisticsProcessingRequest()
595 {
596     ASSERT(!RunLoop::isMain());
597
598     m_pendingStatisticsProcessingRequestIdentifier = WTF::nullopt;
599 }
600
601 void ResourceLoadStatisticsMemoryStore::logFrameNavigation(const RegistrableDomain& targetDomain, const RegistrableDomain& topFrameDomain, const RegistrableDomain& sourceDomain, bool isRedirect, bool isMainFrame)
602 {
603     ASSERT(!RunLoop::isMain());
604
605     bool areTargetAndTopFrameDomainsSameSite = targetDomain == topFrameDomain;
606     bool areTargetAndSourceDomainsSameSite = targetDomain == sourceDomain;
607
608     bool statisticsWereUpdated = false;
609     if (!isMainFrame && !(areTargetAndTopFrameDomainsSameSite || areTargetAndSourceDomainsSameSite)) {
610         auto& targetStatistics = ensureResourceStatisticsForRegistrableDomain(targetDomain);
611         targetStatistics.lastSeen = ResourceLoadStatistics::reduceTimeResolution(WallTime::now());
612         if (targetStatistics.subframeUnderTopFrameDomains.add(topFrameDomain).isNewEntry)
613             statisticsWereUpdated = true;
614     }
615
616     if (isRedirect && !areTargetAndSourceDomainsSameSite) {
617         if (isMainFrame) {
618             auto& redirectingDomainStatistics = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
619             if (redirectingDomainStatistics.topFrameUniqueRedirectsTo.add(targetDomain).isNewEntry)
620                 statisticsWereUpdated = true;
621             auto& targetStatistics = ensureResourceStatisticsForRegistrableDomain(targetDomain);
622             if (targetStatistics.topFrameUniqueRedirectsFrom.add(sourceDomain).isNewEntry)
623                 statisticsWereUpdated = true;
624         } else {
625             auto& redirectingDomainStatistics = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
626             if (redirectingDomainStatistics.subresourceUniqueRedirectsTo.add(targetDomain).isNewEntry)
627                 statisticsWereUpdated = true;
628             auto& targetStatistics = ensureResourceStatisticsForRegistrableDomain(targetDomain);
629             if (targetStatistics.subresourceUniqueRedirectsFrom.add(sourceDomain).isNewEntry)
630                 statisticsWereUpdated = true;
631         }
632     }
633
634     if (statisticsWereUpdated)
635         scheduleStatisticsProcessingRequestIfNecessary();
636 }
637
638 void ResourceLoadStatisticsMemoryStore::logSubresourceLoading(const RegistrableDomain& targetDomain, const RegistrableDomain& topFrameDomain, WallTime lastSeen)
639 {
640     ASSERT(!RunLoop::isMain());
641
642     auto& targetStatistics = ensureResourceStatisticsForRegistrableDomain(targetDomain);
643     targetStatistics.lastSeen = lastSeen;
644     if (targetStatistics.subresourceUnderTopFrameDomains.add(topFrameDomain).isNewEntry)
645         scheduleStatisticsProcessingRequestIfNecessary();
646 }
647
648 void ResourceLoadStatisticsMemoryStore::logSubresourceRedirect(const RegistrableDomain& sourceDomain, const RegistrableDomain& targetDomain)
649 {
650     ASSERT(!RunLoop::isMain());
651
652     auto& redirectingDomainStatistics = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
653     bool isNewRedirectToEntry = redirectingDomainStatistics.subresourceUniqueRedirectsTo.add(targetDomain).isNewEntry;
654     auto& targetStatistics = ensureResourceStatisticsForRegistrableDomain(targetDomain);
655     bool isNewRedirectFromEntry = targetStatistics.subresourceUniqueRedirectsFrom.add(sourceDomain).isNewEntry;
656
657     if (isNewRedirectToEntry || isNewRedirectFromEntry)
658         scheduleStatisticsProcessingRequestIfNecessary();
659 }
660
661 void ResourceLoadStatisticsMemoryStore::logUserInteraction(const RegistrableDomain& domain)
662 {
663     ASSERT(!RunLoop::isMain());
664
665     auto& statistics = ensureResourceStatisticsForRegistrableDomain(domain);
666     statistics.hadUserInteraction = true;
667     statistics.mostRecentUserInteractionTime = WallTime::now();
668 }
669
670 void ResourceLoadStatisticsMemoryStore::clearUserInteraction(const RegistrableDomain& domain)
671 {
672     ASSERT(!RunLoop::isMain());
673
674     auto& statistics = ensureResourceStatisticsForRegistrableDomain(domain);
675     statistics.hadUserInteraction = false;
676     statistics.mostRecentUserInteractionTime = { };
677 }
678
679 bool ResourceLoadStatisticsMemoryStore::hasHadUserInteraction(const RegistrableDomain& domain)
680 {
681     ASSERT(!RunLoop::isMain());
682
683     auto mapEntry = m_resourceStatisticsMap.find(domain);
684     return mapEntry == m_resourceStatisticsMap.end() ? false: hasHadUnexpiredRecentUserInteraction(mapEntry->value);
685 }
686
687 void ResourceLoadStatisticsMemoryStore::setPrevalentResource(ResourceLoadStatistics& resourceStatistic, ResourceLoadPrevalence newPrevalence)
688 {
689     ASSERT(!RunLoop::isMain());
690
691     resourceStatistic.isPrevalentResource = true;
692     resourceStatistic.isVeryPrevalentResource = newPrevalence == ResourceLoadPrevalence::VeryHigh;
693     HashSet<RegistrableDomain> domainsThatHaveRedirectedTo;
694     recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(resourceStatistic, domainsThatHaveRedirectedTo, 0);
695     for (auto& domain : domainsThatHaveRedirectedTo) {
696         auto mapEntry = m_resourceStatisticsMap.find(domain);
697         if (mapEntry == m_resourceStatisticsMap.end())
698             continue;
699         ASSERT(!mapEntry->value.isPrevalentResource);
700         mapEntry->value.isPrevalentResource = true;
701     }
702 }
703     
704 String ResourceLoadStatisticsMemoryStore::dumpResourceLoadStatistics() const
705 {
706     ASSERT(!RunLoop::isMain());
707
708     StringBuilder result;
709     result.appendLiteral("Resource load statistics:\n\n");
710     for (auto& mapEntry : m_resourceStatisticsMap.values())
711         result.append(mapEntry.toString());
712     return result.toString();
713 }
714
715 bool ResourceLoadStatisticsMemoryStore::isPrevalentResource(const RegistrableDomain& domain) const
716 {
717     ASSERT(!RunLoop::isMain());
718
719     auto mapEntry = m_resourceStatisticsMap.find(domain);
720     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource;
721 }
722
723 bool ResourceLoadStatisticsMemoryStore::isVeryPrevalentResource(const RegistrableDomain& domain) const
724 {
725     ASSERT(!RunLoop::isMain());
726
727     auto mapEntry = m_resourceStatisticsMap.find(domain);
728     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource && mapEntry->value.isVeryPrevalentResource;
729 }
730
731 bool ResourceLoadStatisticsMemoryStore::isRegisteredAsSubresourceUnder(const RegistrableDomain& subresourceDomain, const RegistrableDomain& topFrameDomain) const
732 {
733     ASSERT(!RunLoop::isMain());
734
735     auto mapEntry = m_resourceStatisticsMap.find(subresourceDomain);
736     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subresourceUnderTopFrameDomains.contains(topFrameDomain);
737 }
738
739 bool ResourceLoadStatisticsMemoryStore::isRegisteredAsSubFrameUnder(const RegistrableDomain& subFrameDomain, const RegistrableDomain& topFrameDomain) const
740 {
741     ASSERT(!RunLoop::isMain());
742
743     auto mapEntry = m_resourceStatisticsMap.find(subFrameDomain);
744     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subframeUnderTopFrameDomains.contains(topFrameDomain);
745 }
746
747 bool ResourceLoadStatisticsMemoryStore::isRegisteredAsRedirectingTo(const RegistrableDomain& redirectedFromDomain, const RegistrableDomain& redirectedToDomain) const
748 {
749     ASSERT(!RunLoop::isMain());
750
751     auto mapEntry = m_resourceStatisticsMap.find(redirectedFromDomain);
752     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subresourceUniqueRedirectsTo.contains(redirectedToDomain);
753 }
754
755 void ResourceLoadStatisticsMemoryStore::clearPrevalentResource(const RegistrableDomain& domain)
756 {
757     ASSERT(!RunLoop::isMain());
758
759     auto& statistics = ensureResourceStatisticsForRegistrableDomain(domain);
760     statistics.isPrevalentResource = false;
761     statistics.isVeryPrevalentResource = false;
762 }
763
764 void ResourceLoadStatisticsMemoryStore::setGrandfathered(const RegistrableDomain& domain, bool value)
765 {
766     ASSERT(!RunLoop::isMain());
767
768     auto& statistics = ensureResourceStatisticsForRegistrableDomain(domain);
769     statistics.grandfathered = value;
770 }
771
772 bool ResourceLoadStatisticsMemoryStore::isGrandfathered(const RegistrableDomain& domain) const
773 {
774     ASSERT(!RunLoop::isMain());
775
776     auto mapEntry = m_resourceStatisticsMap.find(domain);
777     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.grandfathered;
778 }
779
780 void ResourceLoadStatisticsMemoryStore::setSubframeUnderTopFrameDomain(const RegistrableDomain& subFrameDomain, const RegistrableDomain& topFrameDomain)
781 {
782     ASSERT(!RunLoop::isMain());
783
784     auto& statistics = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
785     statistics.subframeUnderTopFrameDomains.add(topFrameDomain);
786     // For consistency, make sure we also have a statistics entry for the top frame domain.
787     ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
788 }
789
790 void ResourceLoadStatisticsMemoryStore::setSubresourceUnderTopFrameDomain(const RegistrableDomain& subresourceDomain, const RegistrableDomain& topFrameDomain)
791 {
792     ASSERT(!RunLoop::isMain());
793
794     auto& statistics = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
795     statistics.subresourceUnderTopFrameDomains.add(topFrameDomain);
796     // For consistency, make sure we also have a statistics entry for the top frame domain.
797     ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
798 }
799
800 void ResourceLoadStatisticsMemoryStore::setSubresourceUniqueRedirectTo(const RegistrableDomain& subresourceDomain, const RegistrableDomain& redirectDomain)
801 {
802     ASSERT(!RunLoop::isMain());
803
804     auto& statistics = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
805     statistics.subresourceUniqueRedirectsTo.add(redirectDomain);
806     // For consistency, make sure we also have a statistics entry for the redirect domain.
807     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
808 }
809
810 void ResourceLoadStatisticsMemoryStore::setSubresourceUniqueRedirectFrom(const RegistrableDomain& subresourceDomain, const RegistrableDomain& redirectDomain)
811 {
812     ASSERT(!RunLoop::isMain());
813
814     auto& statistics = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
815     statistics.subresourceUniqueRedirectsFrom.add(redirectDomain);
816     // For consistency, make sure we also have a statistics entry for the redirect domain.
817     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
818 }
819
820 void ResourceLoadStatisticsMemoryStore::setTopFrameUniqueRedirectTo(const RegistrableDomain& topFrameDomain, const RegistrableDomain& redirectDomain)
821 {
822     ASSERT(!RunLoop::isMain());
823
824     auto& statistics = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
825     statistics.topFrameUniqueRedirectsTo.add(redirectDomain);
826     // For consistency, make sure we also have a statistics entry for the redirect domain.
827     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
828 }
829
830 void ResourceLoadStatisticsMemoryStore::setTopFrameUniqueRedirectFrom(const RegistrableDomain& topFrameDomain, const RegistrableDomain& redirectDomain)
831 {
832     ASSERT(!RunLoop::isMain());
833
834     auto& statistics = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
835     statistics.topFrameUniqueRedirectsFrom.add(redirectDomain);
836     // For consistency, make sure we also have a statistics entry for the redirect domain.
837     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
838 }
839
840 void ResourceLoadStatisticsMemoryStore::setTimeToLiveUserInteraction(Seconds seconds)
841 {
842     ASSERT(!RunLoop::isMain());
843     ASSERT(seconds >= 0_s);
844
845     m_parameters.timeToLiveUserInteraction = seconds;
846 }
847
848 void ResourceLoadStatisticsMemoryStore::setMinimumTimeBetweenDataRecordsRemoval(Seconds seconds)
849 {
850     ASSERT(!RunLoop::isMain());
851     ASSERT(seconds >= 0_s);
852
853     m_parameters.minimumTimeBetweenDataRecordsRemoval = seconds;
854 }
855
856 void ResourceLoadStatisticsMemoryStore::setGrandfatheringTime(Seconds seconds)
857 {
858     ASSERT(!RunLoop::isMain());
859     ASSERT(seconds >= 0_s);
860
861     m_parameters.grandfatheringTime = seconds;
862 }
863
864 void ResourceLoadStatisticsMemoryStore::setCacheMaxAgeCap(Seconds seconds)
865 {
866     ASSERT(!RunLoop::isMain());
867     ASSERT(seconds >= 0_s);
868
869     m_parameters.cacheMaxAgeCapTime = seconds;
870     updateCacheMaxAgeCap();
871 }
872
873 void ResourceLoadStatisticsMemoryStore::updateCacheMaxAgeCap()
874 {
875     ASSERT(!RunLoop::isMain());
876     
877     RunLoop::main().dispatch([store = makeRef(m_store), seconds = m_parameters.cacheMaxAgeCapTime] () {
878         store->setCacheMaxAgeCap(seconds, [] { });
879     });
880 }
881
882 void ResourceLoadStatisticsMemoryStore::setAgeCapForClientSideCookies(Seconds seconds)
883 {
884     ASSERT(!RunLoop::isMain());
885     ASSERT(seconds >= 0_s);
886     
887     m_parameters.clientSideCookiesAgeCapTime = seconds;
888     updateClientSideCookiesAgeCap();
889 }
890
891 void ResourceLoadStatisticsMemoryStore::updateClientSideCookiesAgeCap()
892 {
893     ASSERT(!RunLoop::isMain());
894
895 #if ENABLE(RESOURCE_LOAD_STATISTICS)
896     RunLoop::main().dispatch([store = makeRef(m_store), seconds = m_parameters.clientSideCookiesAgeCapTime] () {
897         if (auto* networkSession = store->networkSession())
898             networkSession->networkStorageSession().setAgeCapForClientSideCookies(seconds);
899     });
900 #endif
901 }
902
903 bool ResourceLoadStatisticsMemoryStore::shouldRemoveDataRecords() const
904 {
905     ASSERT(!RunLoop::isMain());
906
907     if (m_dataRecordsBeingRemoved)
908         return false;
909
910 #if ENABLE(NETSCAPE_PLUGIN_API)
911     for (const auto& plugin : PluginProcessManager::singleton().pluginProcesses()) {
912         if (!m_activePluginTokens.contains(plugin->pluginProcessToken()))
913             return true;
914     }
915 #endif
916
917     return !m_lastTimeDataRecordsWereRemoved || MonotonicTime::now() >= (m_lastTimeDataRecordsWereRemoved + m_parameters.minimumTimeBetweenDataRecordsRemoval);
918 }
919
920 void ResourceLoadStatisticsMemoryStore::setDataRecordsBeingRemoved(bool value)
921 {
922     ASSERT(!RunLoop::isMain());
923
924     m_dataRecordsBeingRemoved = value;
925     if (m_dataRecordsBeingRemoved)
926         m_lastTimeDataRecordsWereRemoved = MonotonicTime::now();
927 }
928
929 ResourceLoadStatistics& ResourceLoadStatisticsMemoryStore::ensureResourceStatisticsForRegistrableDomain(const RegistrableDomain& domain)
930 {
931     ASSERT(!RunLoop::isMain());
932
933     return m_resourceStatisticsMap.ensure(domain, [&domain] {
934         return ResourceLoadStatistics(domain);
935     }).iterator->value;
936 }
937
938 std::unique_ptr<KeyedEncoder> ResourceLoadStatisticsMemoryStore::createEncoderFromData() const
939 {
940     ASSERT(!RunLoop::isMain());
941
942     auto encoder = KeyedEncoder::encoder();
943     encoder->encodeUInt32("version", statisticsModelVersion);
944     encoder->encodeDouble("endOfGrandfatheringTimestamp", m_endOfGrandfatheringTimestamp.secondsSinceEpoch().value());
945
946     encoder->encodeObjects("browsingStatistics", m_resourceStatisticsMap.begin(), m_resourceStatisticsMap.end(), [](KeyedEncoder& encoderInner, const auto& domain) {
947         domain.value.encode(encoderInner);
948     });
949
950     encoder->encodeObjects("operatingDates", m_operatingDates.begin(), m_operatingDates.end(), [](KeyedEncoder& encoderInner, OperatingDate date) {
951         encoderInner.encodeDouble("date", date.secondsSinceEpoch().value());
952     });
953
954     return encoder;
955 }
956
957 void ResourceLoadStatisticsMemoryStore::mergeWithDataFromDecoder(KeyedDecoder& decoder)
958 {
959     ASSERT(!RunLoop::isMain());
960
961     unsigned versionOnDisk;
962     if (!decoder.decodeUInt32("version", versionOnDisk))
963         return;
964
965     if (versionOnDisk > statisticsModelVersion) {
966         WTFLogAlways("Found resource load statistics on disk with model version %u whereas the highest supported version is %u. Resetting.", versionOnDisk, statisticsModelVersion);
967         return;
968     }
969
970     double endOfGrandfatheringTimestamp;
971     if (decoder.decodeDouble("endOfGrandfatheringTimestamp", endOfGrandfatheringTimestamp))
972         m_endOfGrandfatheringTimestamp = WallTime::fromRawSeconds(endOfGrandfatheringTimestamp);
973     else
974         m_endOfGrandfatheringTimestamp = { };
975
976     Vector<ResourceLoadStatistics> loadedStatistics;
977     bool succeeded = decoder.decodeObjects("browsingStatistics", loadedStatistics, [versionOnDisk](KeyedDecoder& decoderInner, ResourceLoadStatistics& statistics) {
978         return statistics.decode(decoderInner, versionOnDisk);
979     });
980
981     if (!succeeded)
982         return;
983
984     mergeStatistics(WTFMove(loadedStatistics));
985     updateCookieBlocking([]() { });
986
987     Vector<OperatingDate> operatingDates;
988     succeeded = decoder.decodeObjects("operatingDates", operatingDates, [](KeyedDecoder& decoder, OperatingDate& date) {
989         double value;
990         if (!decoder.decodeDouble("date", value))
991             return false;
992
993         date = OperatingDate::fromWallTime(WallTime::fromRawSeconds(value));
994         return true;
995     });
996
997     if (!succeeded)
998         return;
999
1000     m_operatingDates = mergeOperatingDates(m_operatingDates, WTFMove(operatingDates));
1001 }
1002
1003 void ResourceLoadStatisticsMemoryStore::clear(CompletionHandler<void()>&& completionHandler)
1004 {
1005     ASSERT(!RunLoop::isMain());
1006
1007     m_resourceStatisticsMap.clear();
1008     m_operatingDates.clear();
1009
1010     auto callbackAggregator = CallbackAggregator::create(WTFMove(completionHandler));
1011
1012     removeAllStorageAccess([callbackAggregator = callbackAggregator.copyRef()] { });
1013
1014     auto primaryDomainsToBlock = ensurePrevalentResourcesForDebugMode();
1015     updateCookieBlockingForDomains(primaryDomainsToBlock, [callbackAggregator = callbackAggregator.copyRef()] { });
1016 }
1017
1018 bool ResourceLoadStatisticsMemoryStore::wasAccessedAsFirstPartyDueToUserInteraction(const ResourceLoadStatistics& current, const ResourceLoadStatistics& updated) const
1019 {
1020     if (!current.hadUserInteraction && !updated.hadUserInteraction)
1021         return false;
1022
1023     auto mostRecentUserInteractionTime = std::max(current.mostRecentUserInteractionTime, updated.mostRecentUserInteractionTime);
1024
1025     return updated.lastSeen <= mostRecentUserInteractionTime + 24_h;
1026 }
1027
1028 void ResourceLoadStatisticsMemoryStore::mergeStatistics(Vector<ResourceLoadStatistics>&& statistics)
1029 {
1030     ASSERT(!RunLoop::isMain());
1031
1032     for (auto& statistic : statistics) {
1033         auto result = m_resourceStatisticsMap.ensure(statistic.registrableDomain, [&statistic] {
1034             return WTFMove(statistic);
1035         });
1036         if (!result.isNewEntry) {
1037             if (wasAccessedAsFirstPartyDueToUserInteraction(result.iterator->value, statistic))
1038                 result.iterator->value.timesAccessedAsFirstPartyDueToUserInteraction++;
1039             result.iterator->value.merge(statistic);
1040         }
1041     }
1042 }
1043
1044 bool ResourceLoadStatisticsMemoryStore::shouldBlockAndKeepCookies(const ResourceLoadStatistics& statistic)
1045 {
1046     return statistic.isPrevalentResource && statistic.hadUserInteraction;
1047 }
1048
1049 bool ResourceLoadStatisticsMemoryStore::shouldBlockAndPurgeCookies(const ResourceLoadStatistics& statistic)
1050 {
1051     return statistic.isPrevalentResource && !statistic.hadUserInteraction;
1052 }
1053
1054 bool ResourceLoadStatisticsMemoryStore::hasUserGrantedStorageAccessThroughPrompt(const ResourceLoadStatistics& statistic, const RegistrableDomain& firstPartyDomain)
1055 {
1056     return statistic.storageAccessUnderTopFrameDomains.contains(firstPartyDomain);
1057 }
1058
1059 static void debugLogDomainsInBatches(const char* action, const Vector<RegistrableDomain>& domains)
1060 {
1061 #if !RELEASE_LOG_DISABLED
1062     static const auto maxNumberOfDomainsInOneLogStatement = 50;
1063     if (domains.isEmpty())
1064         return;
1065
1066     if (domains.size() <= maxNumberOfDomainsInOneLogStatement) {
1067         RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "About to %{public}s cookies in third-party contexts for: %{public}s.", action, domainsToString(domains).utf8().data());
1068         return;
1069     }
1070     
1071     Vector<RegistrableDomain> batch;
1072     batch.reserveInitialCapacity(maxNumberOfDomainsInOneLogStatement);
1073     auto batchNumber = 1;
1074     unsigned numberOfBatches = std::ceil(domains.size() / static_cast<float>(maxNumberOfDomainsInOneLogStatement));
1075
1076     for (auto& domain : domains) {
1077         if (batch.size() == maxNumberOfDomainsInOneLogStatement) {
1078             RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "About to %{public}s cookies in third-party contexts for (%{public}d of %u): %{public}s.", action, batchNumber, numberOfBatches, domainsToString(batch).utf8().data());
1079             batch.shrink(0);
1080             ++batchNumber;
1081         }
1082         batch.append(domain);
1083     }
1084     if (!batch.isEmpty())
1085         RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "About to %{public}s cookies in third-party contexts for (%{public}d of %u): %{public}s.", action, batchNumber, numberOfBatches, domainsToString(batch).utf8().data());
1086 #else
1087     UNUSED_PARAM(action);
1088     UNUSED_PARAM(domains);
1089 #endif
1090 }
1091
1092 void ResourceLoadStatisticsMemoryStore::updateCookieBlocking(CompletionHandler<void()>&& completionHandler)
1093 {
1094     ASSERT(!RunLoop::isMain());
1095
1096     Vector<RegistrableDomain> domainsToBlock;
1097     for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
1098         if (resourceStatistic.isPrevalentResource)
1099             domainsToBlock.append(resourceStatistic.registrableDomain);
1100     }
1101
1102     if (domainsToBlock.isEmpty()) {
1103         completionHandler();
1104         return;
1105     }
1106
1107     if (m_debugLoggingEnabled && !domainsToBlock.isEmpty())
1108         debugLogDomainsInBatches("block", domainsToBlock);
1109
1110     RunLoop::main().dispatch([weakThis = makeWeakPtr(*this), store = makeRef(m_store), domainsToBlock = crossThreadCopy(domainsToBlock), completionHandler = WTFMove(completionHandler)] () mutable {
1111         store->callUpdatePrevalentDomainsToBlockCookiesForHandler(domainsToBlock, [weakThis = WTFMove(weakThis), store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1112             store->statisticsQueue().dispatch([weakThis = WTFMove(weakThis), completionHandler = WTFMove(completionHandler)]() mutable {
1113                 completionHandler();
1114                 if (!weakThis)
1115                     return;
1116 #if !RELEASE_LOG_DISABLED
1117                 RELEASE_LOG_INFO_IF(weakThis->m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Done updating cookie blocking.");
1118 #endif
1119             });
1120         });
1121     });
1122 }
1123
1124 void ResourceLoadStatisticsMemoryStore::updateCookieBlockingForDomains(const Vector<RegistrableDomain>& domainsToBlock, CompletionHandler<void()>&& completionHandler)
1125 {
1126     ASSERT(!RunLoop::isMain());
1127
1128     RunLoop::main().dispatch([store = makeRef(m_store), domainsToBlock = crossThreadCopy(domainsToBlock), completionHandler = WTFMove(completionHandler)] () mutable {
1129         store->callUpdatePrevalentDomainsToBlockCookiesForHandler(domainsToBlock, [store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1130             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler)]() mutable {
1131                 completionHandler();
1132             });
1133         });
1134     });
1135 }
1136
1137 void ResourceLoadStatisticsMemoryStore::clearBlockingStateForDomains(const Vector<RegistrableDomain>& domains, CompletionHandler<void()>&& completionHandler)
1138 {
1139     ASSERT(!RunLoop::isMain());
1140
1141     if (domains.isEmpty()) {
1142         completionHandler();
1143         return;
1144     }
1145
1146     RunLoop::main().dispatch([store = makeRef(m_store), domains = crossThreadCopy(domains)] {
1147         store->callRemoveDomainsHandler(domains);
1148     });
1149
1150     completionHandler();
1151 }
1152
1153 void ResourceLoadStatisticsMemoryStore::processStatistics(const Function<void(const ResourceLoadStatistics&)>& processFunction) const
1154 {
1155     ASSERT(!RunLoop::isMain());
1156
1157     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
1158         processFunction(resourceStatistic);
1159 }
1160
1161 bool ResourceLoadStatisticsMemoryStore::hasHadUnexpiredRecentUserInteraction(ResourceLoadStatistics& resourceStatistic) const
1162 {
1163     ASSERT(!RunLoop::isMain());
1164
1165     if (resourceStatistic.hadUserInteraction && hasStatisticsExpired(resourceStatistic)) {
1166         // Drop privacy sensitive data because we no longer need it.
1167         // Set timestamp to 0 so that statistics merge will know
1168         // it has been reset as opposed to its default -1.
1169         resourceStatistic.mostRecentUserInteractionTime = { };
1170         resourceStatistic.storageAccessUnderTopFrameDomains.clear();
1171         resourceStatistic.hadUserInteraction = false;
1172     }
1173
1174     return resourceStatistic.hadUserInteraction;
1175 }
1176
1177 Vector<RegistrableDomain> ResourceLoadStatisticsMemoryStore::registrableDomainsToRemoveWebsiteDataFor()
1178 {
1179     ASSERT(!RunLoop::isMain());
1180
1181     bool shouldCheckForGrandfathering = m_endOfGrandfatheringTimestamp > WallTime::now();
1182     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && m_endOfGrandfatheringTimestamp;
1183
1184     if (shouldClearGrandfathering)
1185         m_endOfGrandfatheringTimestamp = { };
1186
1187     Vector<RegistrableDomain> prevalentResources;
1188     for (auto& statistic : m_resourceStatisticsMap.values()) {
1189         if (statistic.isPrevalentResource && !hasHadUnexpiredRecentUserInteraction(statistic) && (!shouldCheckForGrandfathering || !statistic.grandfathered))
1190             prevalentResources.append(statistic.registrableDomain);
1191
1192         if (shouldClearGrandfathering && statistic.grandfathered)
1193             statistic.grandfathered = false;
1194     }
1195
1196     return prevalentResources;
1197 }
1198
1199 void ResourceLoadStatisticsMemoryStore::includeTodayAsOperatingDateIfNecessary()
1200 {
1201     ASSERT(!RunLoop::isMain());
1202
1203     auto today = OperatingDate::today();
1204     if (!m_operatingDates.isEmpty() && today <= m_operatingDates.last())
1205         return;
1206
1207     while (m_operatingDates.size() >= operatingDatesWindow)
1208         m_operatingDates.remove(0);
1209
1210     m_operatingDates.append(today);
1211 }
1212
1213 bool ResourceLoadStatisticsMemoryStore::hasStatisticsExpired(const ResourceLoadStatistics& resourceStatistic) const
1214 {
1215     ASSERT(!RunLoop::isMain());
1216
1217     if (m_operatingDates.size() >= operatingDatesWindow) {
1218         if (OperatingDate::fromWallTime(resourceStatistic.mostRecentUserInteractionTime) < m_operatingDates.first())
1219             return true;
1220     }
1221
1222     // If we don't meet the real criteria for an expired statistic, check the user setting for a tighter restriction (mainly for testing).
1223     if (m_parameters.timeToLiveUserInteraction) {
1224         if (WallTime::now() > resourceStatistic.mostRecentUserInteractionTime + m_parameters.timeToLiveUserInteraction.value())
1225             return true;
1226     }
1227
1228     return false;
1229 }
1230
1231 void ResourceLoadStatisticsMemoryStore::setMaxStatisticsEntries(size_t maximumEntryCount)
1232 {
1233     ASSERT(!RunLoop::isMain());
1234
1235     m_parameters.maxStatisticsEntries = maximumEntryCount;
1236 }
1237
1238 void ResourceLoadStatisticsMemoryStore::setPruneEntriesDownTo(size_t pruneTargetCount)
1239 {
1240     ASSERT(!RunLoop::isMain());
1241
1242     m_parameters.pruneEntriesDownTo = pruneTargetCount;
1243 }
1244
1245 void ResourceLoadStatisticsMemoryStore::pruneStatisticsIfNeeded()
1246 {
1247     ASSERT(!RunLoop::isMain());
1248
1249     if (m_resourceStatisticsMap.size() <= m_parameters.maxStatisticsEntries)
1250         return;
1251
1252     ASSERT(m_parameters.pruneEntriesDownTo <= m_parameters.maxStatisticsEntries);
1253
1254     size_t numberOfEntriesLeftToPrune = m_resourceStatisticsMap.size() - m_parameters.pruneEntriesDownTo;
1255     ASSERT(numberOfEntriesLeftToPrune);
1256
1257     Vector<StatisticsLastSeen> resourcesToPrunePerImportance[maxImportance + 1];
1258     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
1259         resourcesToPrunePerImportance[computeImportance(resourceStatistic)].append({ resourceStatistic.registrableDomain, resourceStatistic.lastSeen });
1260
1261     for (unsigned importance = 0; numberOfEntriesLeftToPrune && importance <= maxImportance; ++importance)
1262         pruneResources(m_resourceStatisticsMap, resourcesToPrunePerImportance[importance], numberOfEntriesLeftToPrune);
1263
1264     ASSERT(!numberOfEntriesLeftToPrune);
1265 }
1266
1267 void ResourceLoadStatisticsMemoryStore::resetParametersToDefaultValues()
1268 {
1269     ASSERT(!RunLoop::isMain());
1270
1271     m_parameters = { };
1272 }
1273
1274 void ResourceLoadStatisticsMemoryStore::logTestingEvent(const String& event)
1275 {
1276     ASSERT(!RunLoop::isMain());
1277
1278     RunLoop::main().dispatch([store = makeRef(m_store), event = event.isolatedCopy()] {
1279         store->logTestingEvent(event);
1280     });
1281 }
1282
1283 void ResourceLoadStatisticsMemoryStore::setLastSeen(const RegistrableDomain& domain, Seconds seconds)
1284 {
1285     ASSERT(!RunLoop::isMain());
1286
1287     auto& statistics = ensureResourceStatisticsForRegistrableDomain(domain);
1288     statistics.lastSeen = WallTime::fromRawSeconds(seconds.seconds());
1289 }
1290
1291 void ResourceLoadStatisticsMemoryStore::setPrevalentResource(const RegistrableDomain& domain)
1292 {
1293     ASSERT(!RunLoop::isMain());
1294
1295     auto& resourceStatistic = ensureResourceStatisticsForRegistrableDomain(domain);
1296     setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
1297 }
1298
1299 void ResourceLoadStatisticsMemoryStore::setVeryPrevalentResource(const RegistrableDomain& domain)
1300 {
1301     ASSERT(!RunLoop::isMain());
1302
1303     auto& resourceStatistic = ensureResourceStatisticsForRegistrableDomain(domain);
1304     setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::VeryHigh);
1305 }
1306
1307 void ResourceLoadStatisticsMemoryStore::removeAllStorageAccess(CompletionHandler<void()>&& completionHandler)
1308 {
1309     ASSERT(!RunLoop::isMain());
1310     RunLoop::main().dispatch([store = makeRef(m_store), completionHandler = WTFMove(completionHandler)]() mutable {
1311         store->removeAllStorageAccess([store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1312             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler)]() mutable {
1313                 completionHandler();
1314             });
1315         });
1316     });
1317 }
1318
1319 void ResourceLoadStatisticsMemoryStore::didCreateNetworkProcess()
1320 {
1321     ASSERT(!RunLoop::isMain());
1322
1323     updateCookieBlocking([]() { });
1324     updateCacheMaxAgeCap();
1325     updateClientSideCookiesAgeCap();
1326 }
1327
1328 } // namespace WebKit
1329
1330 #endif