19c38e1c47ced6082578e7d1938151fd50f754fd
[WebKit-https.git] / Source / WebKit / NetworkProcess / Classifier / ResourceLoadStatisticsMemoryStore.cpp
1 /*
2  * Copyright (C) 2017-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ResourceLoadStatisticsMemoryStore.h"
28
29 #include "Logging.h"
30 #include "PluginProcessManager.h"
31 #include "PluginProcessProxy.h"
32 #include "ResourceLoadStatisticsPersistentStorage.h"
33 #include "WebProcessProxy.h"
34 #include "WebResourceLoadStatisticsTelemetry.h"
35 #include "WebsiteDataStore.h"
36 #include <WebCore/KeyedCoding.h>
37 #include <WebCore/ResourceLoadStatistics.h>
38 #include <wtf/CallbackAggregator.h>
39 #include <wtf/DateMath.h>
40 #include <wtf/MathExtras.h>
41 #include <wtf/text/StringBuilder.h>
42
43 namespace WebKit {
44 using namespace WebCore;
45
46 constexpr unsigned statisticsModelVersion { 14 };
47 constexpr unsigned maxNumberOfRecursiveCallsInRedirectTraceBack { 50 };
48 constexpr Seconds minimumStatisticsProcessingInterval { 5_s };
49 constexpr unsigned operatingDatesWindow { 30 };
50 constexpr unsigned maxImportance { 3 };
51 static const char* debugStaticPrevalentResource { "3rdpartytestwebkit.org" };
52
53 #if !RELEASE_LOG_DISABLED
54 static String domainsToString(const Vector<String>& domains)
55 {
56     StringBuilder builder;
57     for (auto& domain : domains) {
58         if (!builder.isEmpty())
59             builder.appendLiteral(", ");
60         builder.append(domain);
61     }
62     return builder.toString();
63 }
64 #endif
65
66 class OperatingDate {
67 public:
68     OperatingDate() = default;
69
70     static OperatingDate fromWallTime(WallTime time)
71     {
72         double ms = time.secondsSinceEpoch().milliseconds();
73         int year = msToYear(ms);
74         int yearDay = dayInYear(ms, year);
75         int month = monthFromDayInYear(yearDay, isLeapYear(year));
76         int monthDay = dayInMonthFromDayInYear(yearDay, isLeapYear(year));
77
78         return OperatingDate { year, month, monthDay };
79     }
80
81     static OperatingDate today()
82     {
83         return OperatingDate::fromWallTime(WallTime::now());
84     }
85
86     Seconds secondsSinceEpoch() const
87     {
88         return Seconds { dateToDaysFrom1970(m_year, m_month, m_monthDay) * secondsPerDay };
89     }
90
91     bool operator==(const OperatingDate& other) const
92     {
93         return m_monthDay == other.m_monthDay && m_month == other.m_month && m_year == other.m_year;
94     }
95
96     bool operator<(const OperatingDate& other) const
97     {
98         return secondsSinceEpoch() < other.secondsSinceEpoch();
99     }
100
101     bool operator<=(const OperatingDate& other) const
102     {
103         return secondsSinceEpoch() <= other.secondsSinceEpoch();
104     }
105
106 private:
107     OperatingDate(int year, int month, int monthDay)
108         : m_year(year)
109         , m_month(month)
110         , m_monthDay(monthDay)
111     { }
112
113     int m_year { 0 };
114     int m_month { 0 }; // [0, 11].
115     int m_monthDay { 0 }; // [1, 31].
116 };
117
118 static Vector<OperatingDate> mergeOperatingDates(const Vector<OperatingDate>& existingDates, Vector<OperatingDate>&& newDates)
119 {
120     if (existingDates.isEmpty())
121         return WTFMove(newDates);
122
123     Vector<OperatingDate> mergedDates(existingDates.size() + newDates.size());
124
125     // Merge the two sorted vectors of dates.
126     std::merge(existingDates.begin(), existingDates.end(), newDates.begin(), newDates.end(), mergedDates.begin());
127     // Remove duplicate dates.
128     removeRepeatedElements(mergedDates);
129
130     // Drop old dates until the Vector size reaches operatingDatesWindow.
131     while (mergedDates.size() > operatingDatesWindow)
132         mergedDates.remove(0);
133
134     return mergedDates;
135 }
136
137 struct StatisticsLastSeen {
138     String topPrivatelyOwnedDomain;
139     WallTime lastSeen;
140 };
141
142 static void pruneResources(HashMap<String, WebCore::ResourceLoadStatistics>& statisticsMap, Vector<StatisticsLastSeen>& statisticsToPrune, size_t& numberOfEntriesToPrune)
143 {
144     if (statisticsToPrune.size() > numberOfEntriesToPrune) {
145         std::sort(statisticsToPrune.begin(), statisticsToPrune.end(), [](const StatisticsLastSeen& a, const StatisticsLastSeen& b) {
146             return a.lastSeen < b.lastSeen;
147         });
148     }
149
150     for (size_t i = 0, end = std::min(numberOfEntriesToPrune, statisticsToPrune.size()); i != end; ++i, --numberOfEntriesToPrune)
151         statisticsMap.remove(statisticsToPrune[i].topPrivatelyOwnedDomain);
152 }
153
154 static unsigned computeImportance(const ResourceLoadStatistics& resourceStatistic)
155 {
156     unsigned importance = maxImportance;
157     if (!resourceStatistic.isPrevalentResource)
158         importance -= 1;
159     if (!resourceStatistic.hadUserInteraction)
160         importance -= 2;
161     return importance;
162 }
163
164 ResourceLoadStatisticsMemoryStore::ResourceLoadStatisticsMemoryStore(WebResourceLoadStatisticsStore& store, WorkQueue& workQueue)
165     : m_store(store)
166     , m_workQueue(workQueue)
167 {
168     ASSERT(!RunLoop::isMain());
169
170 #if PLATFORM(COCOA)
171     registerUserDefaultsIfNeeded();
172 #endif
173     includeTodayAsOperatingDateIfNecessary();
174
175     m_workQueue->dispatchAfter(5_s, [weakThis = makeWeakPtr(*this)] {
176         if (weakThis)
177             weakThis->calculateAndSubmitTelemetry();
178     });
179 }
180
181 ResourceLoadStatisticsMemoryStore::~ResourceLoadStatisticsMemoryStore()
182 {
183     ASSERT(!RunLoop::isMain());
184 }
185
186 void ResourceLoadStatisticsMemoryStore::setPersistentStorage(ResourceLoadStatisticsPersistentStorage& persistentStorage)
187 {
188     m_persistentStorage = makeWeakPtr(persistentStorage);
189 }
190
191 void ResourceLoadStatisticsMemoryStore::calculateAndSubmitTelemetry() const
192 {
193     ASSERT(!RunLoop::isMain());
194
195     if (m_parameters.shouldSubmitTelemetry)
196         WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
197 }
198
199 void ResourceLoadStatisticsMemoryStore::setNotifyPagesWhenDataRecordsWereScanned(bool value)
200 {
201     ASSERT(!RunLoop::isMain());
202     m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned = value;
203 }
204
205 void ResourceLoadStatisticsMemoryStore::setShouldClassifyResourcesBeforeDataRecordsRemoval(bool value)
206 {
207     ASSERT(!RunLoop::isMain());
208     m_parameters.shouldClassifyResourcesBeforeDataRecordsRemoval = value;
209 }
210
211 void ResourceLoadStatisticsMemoryStore::setShouldSubmitTelemetry(bool value)
212 {
213     ASSERT(!RunLoop::isMain());
214     m_parameters.shouldSubmitTelemetry = value;
215 }
216
217 void ResourceLoadStatisticsMemoryStore::removeDataRecords(CompletionHandler<void()>&& callback)
218 {
219     ASSERT(!RunLoop::isMain());
220
221     if (!shouldRemoveDataRecords()) {
222         callback();
223         return;
224     }
225
226 #if ENABLE(NETSCAPE_PLUGIN_API)
227     m_activePluginTokens.clear();
228     for (const auto& plugin : PluginProcessManager::singleton().pluginProcesses())
229         m_activePluginTokens.add(plugin->pluginProcessToken());
230 #endif
231
232     auto prevalentResourceDomains = topPrivatelyControlledDomainsToRemoveWebsiteDataFor();
233     if (prevalentResourceDomains.isEmpty()) {
234         callback();
235         return;
236     }
237
238 #if !RELEASE_LOG_DISABLED
239     RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "About to remove data records for %{public}s.", domainsToString(prevalentResourceDomains).utf8().data());
240 #endif
241
242     setDataRecordsBeingRemoved(true);
243
244     RunLoop::main().dispatch([prevalentResourceDomains = crossThreadCopy(prevalentResourceDomains), callback = WTFMove(callback), weakThis = makeWeakPtr(*this), shouldNotifyPagesWhenDataRecordsWereScanned = m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, workQueue = m_workQueue.copyRef()] () mutable {
245         WebProcessProxy::deleteWebsiteDataForTopPrivatelyControlledDomainsInAllPersistentDataStores(WebResourceLoadStatisticsStore::monitoredDataTypes(), WTFMove(prevalentResourceDomains), shouldNotifyPagesWhenDataRecordsWereScanned, [callback = WTFMove(callback), weakThis = WTFMove(weakThis), workQueue = workQueue.copyRef()](const HashSet<String>& domainsWithDeletedWebsiteData) mutable {
246             workQueue->dispatch([topDomains = crossThreadCopy(domainsWithDeletedWebsiteData), callback = WTFMove(callback), weakThis = WTFMove(weakThis)] () mutable {
247                 if (!weakThis) {
248                     callback();
249                     return;
250                 }
251                 for (auto& prevalentResourceDomain : topDomains) {
252                     auto& statistic = weakThis->ensureResourceStatisticsForPrimaryDomain(prevalentResourceDomain);
253                     ++statistic.dataRecordsRemoved;
254                 }
255                 weakThis->setDataRecordsBeingRemoved(false);
256                 callback();
257 #if !RELEASE_LOG_DISABLED
258                 RELEASE_LOG_INFO_IF(weakThis->m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Done removing data records.");
259 #endif
260             });
261         });
262     });
263 }
264
265 unsigned ResourceLoadStatisticsMemoryStore::recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(const WebCore::ResourceLoadStatistics& resourceStatistic, HashSet<String>& domainsThatHaveRedirectedTo, unsigned numberOfRecursiveCalls) const
266 {
267     ASSERT(!RunLoop::isMain());
268
269     if (numberOfRecursiveCalls >= maxNumberOfRecursiveCallsInRedirectTraceBack) {
270         // Model version 14 invokes a deliberate re-classification of the whole set.
271         if (statisticsModelVersion != 14)
272             ASSERT_NOT_REACHED();
273         RELEASE_LOG(ResourceLoadStatistics, "Hit %u recursive calls in redirect backtrace. Returning early.", maxNumberOfRecursiveCallsInRedirectTraceBack);
274         return numberOfRecursiveCalls;
275     }
276
277     numberOfRecursiveCalls++;
278
279     for (auto& subresourceUniqueRedirectFromDomain : resourceStatistic.subresourceUniqueRedirectsFrom.values()) {
280         auto mapEntry = m_resourceStatisticsMap.find(subresourceUniqueRedirectFromDomain);
281         if (mapEntry == m_resourceStatisticsMap.end() || mapEntry->value.isPrevalentResource)
282             continue;
283         if (domainsThatHaveRedirectedTo.add(mapEntry->value.highLevelDomain).isNewEntry)
284             numberOfRecursiveCalls = recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(mapEntry->value, domainsThatHaveRedirectedTo, numberOfRecursiveCalls);
285     }
286     for (auto& topFrameUniqueRedirectFromDomain : resourceStatistic.topFrameUniqueRedirectsFrom.values()) {
287         auto mapEntry = m_resourceStatisticsMap.find(topFrameUniqueRedirectFromDomain);
288         if (mapEntry == m_resourceStatisticsMap.end() || mapEntry->value.isPrevalentResource)
289             continue;
290         if (domainsThatHaveRedirectedTo.add(mapEntry->value.highLevelDomain).isNewEntry)
291             numberOfRecursiveCalls = recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(mapEntry->value, domainsThatHaveRedirectedTo, numberOfRecursiveCalls);
292     }
293
294     return numberOfRecursiveCalls;
295 }
296
297 void ResourceLoadStatisticsMemoryStore::markAsPrevalentIfHasRedirectedToPrevalent(WebCore::ResourceLoadStatistics& resourceStatistic)
298 {
299     ASSERT(!RunLoop::isMain());
300
301     if (resourceStatistic.isPrevalentResource)
302         return;
303
304     for (auto& subresourceDomainRedirectedTo : resourceStatistic.subresourceUniqueRedirectsTo.values()) {
305         auto mapEntry = m_resourceStatisticsMap.find(subresourceDomainRedirectedTo);
306         if (mapEntry != m_resourceStatisticsMap.end() && mapEntry->value.isPrevalentResource) {
307             setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
308             return;
309         }
310     }
311
312     for (auto& topFrameDomainRedirectedTo : resourceStatistic.topFrameUniqueRedirectsTo.values()) {
313         auto mapEntry = m_resourceStatisticsMap.find(topFrameDomainRedirectedTo);
314         if (mapEntry != m_resourceStatisticsMap.end() && mapEntry->value.isPrevalentResource) {
315             setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
316             return;
317         }
318     }
319 }
320
321 bool ResourceLoadStatisticsMemoryStore::isPrevalentDueToDebugMode(ResourceLoadStatistics& resourceStatistic)
322 {
323     if (!m_debugModeEnabled)
324         return false;
325
326     return resourceStatistic.highLevelDomain == debugStaticPrevalentResource || resourceStatistic.highLevelDomain == m_debugManualPrevalentResource;
327 }
328
329 void ResourceLoadStatisticsMemoryStore::processStatisticsAndDataRecords()
330 {
331     ASSERT(!RunLoop::isMain());
332
333     if (m_parameters.shouldClassifyResourcesBeforeDataRecordsRemoval) {
334         for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
335             if (isPrevalentDueToDebugMode(resourceStatistic))
336                 setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
337             else if (!resourceStatistic.isVeryPrevalentResource) {
338                 markAsPrevalentIfHasRedirectedToPrevalent(resourceStatistic);
339                 auto currentPrevalence = resourceStatistic.isPrevalentResource ? ResourceLoadPrevalence::High : ResourceLoadPrevalence::Low;
340                 auto newPrevalence = m_resourceLoadStatisticsClassifier.calculateResourcePrevalence(resourceStatistic, currentPrevalence);
341                 if (newPrevalence != currentPrevalence)
342                     setPrevalentResource(resourceStatistic, newPrevalence);
343             }
344         }
345     }
346
347     removeDataRecords([this, weakThis = makeWeakPtr(*this)] {
348         ASSERT(!RunLoop::isMain());
349         if (!weakThis)
350             return;
351
352         pruneStatisticsIfNeeded();
353         if (m_persistentStorage)
354             m_persistentStorage->scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::No);
355
356         if (!m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned)
357             return;
358
359         RunLoop::main().dispatch([] {
360             WebProcessProxy::notifyPageStatisticsAndDataRecordsProcessed();
361         });
362     });
363 }
364
365 void ResourceLoadStatisticsMemoryStore::hasStorageAccess(const String& subFramePrimaryDomain, const String& topFramePrimaryDomain, uint64_t frameID, uint64_t pageID, CompletionHandler<void(bool)>&& completionHandler)
366 {
367     ASSERT(!RunLoop::isMain());
368
369     auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
370     if (shouldBlockAndPurgeCookies(subFrameStatistic)) {
371         completionHandler(false);
372         return;
373     }
374
375     if (!shouldBlockAndKeepCookies(subFrameStatistic)) {
376         completionHandler(true);
377         return;
378     }
379
380     RunLoop::main().dispatch([store = makeRef(m_store), subFramePrimaryDomain = subFramePrimaryDomain.isolatedCopy(), topFramePrimaryDomain = topFramePrimaryDomain.isolatedCopy(), frameID, pageID, completionHandler = WTFMove(completionHandler)]() mutable {
381         store->callHasStorageAccessForFrameHandler(subFramePrimaryDomain, topFramePrimaryDomain, frameID, pageID, [store = store.copyRef(), completionHandler = WTFMove(completionHandler)](bool result) mutable {
382             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler), result] () mutable {
383                 completionHandler(result);
384             });
385         });
386     });
387 }
388
389 void ResourceLoadStatisticsMemoryStore::requestStorageAccess(String&& subFramePrimaryDomain, String&& topFramePrimaryDomain, uint64_t frameID, uint64_t pageID, bool promptEnabled, CompletionHandler<void(StorageAccessStatus)>&& completionHandler)
390 {
391     ASSERT(!RunLoop::isMain());
392
393     auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
394     if (shouldBlockAndPurgeCookies(subFrameStatistic)) {
395 #if !RELEASE_LOG_DISABLED
396         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Cannot grant storage access to %{public}s since its cookies are blocked in third-party contexts and it has not received user interaction as first-party.", subFramePrimaryDomain.utf8().data());
397 #endif
398         completionHandler(StorageAccessStatus::CannotRequestAccess);
399         return;
400     }
401
402     if (!shouldBlockAndKeepCookies(subFrameStatistic)) {
403 #if !RELEASE_LOG_DISABLED
404         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "No need to grant storage access to %{public}s since its cookies are not blocked in third-party contexts.", subFramePrimaryDomain.utf8().data());
405 #endif
406         completionHandler(StorageAccessStatus::HasAccess);
407         return;
408     }
409
410     auto userWasPromptedEarlier = promptEnabled && hasUserGrantedStorageAccessThroughPrompt(subFrameStatistic, topFramePrimaryDomain);
411     if (promptEnabled && !userWasPromptedEarlier) {
412 #if !RELEASE_LOG_DISABLED
413         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "About to ask the user whether they want to grant storage access to %{public}s under %{public}s or not.", subFramePrimaryDomain.utf8().data(), topFramePrimaryDomain.utf8().data());
414 #endif
415         completionHandler(StorageAccessStatus::RequiresUserPrompt);
416         return;
417     } else if (userWasPromptedEarlier) {
418 #if !RELEASE_LOG_DISABLED
419         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Storage access was granted to %{public}s under %{public}s.", subFramePrimaryDomain.utf8().data(), topFramePrimaryDomain.utf8().data());
420 #endif
421     }
422
423     subFrameStatistic.timesAccessedAsFirstPartyDueToStorageAccessAPI++;
424
425     grantStorageAccessInternal(WTFMove(subFramePrimaryDomain), WTFMove(topFramePrimaryDomain), frameID, pageID, userWasPromptedEarlier, [completionHandler = WTFMove(completionHandler)] (bool wasGrantedAccess) mutable {
426         completionHandler(wasGrantedAccess ? StorageAccessStatus::HasAccess : StorageAccessStatus::CannotRequestAccess);
427     });
428 }
429
430 void ResourceLoadStatisticsMemoryStore::requestStorageAccessUnderOpener(String&& primaryDomainInNeedOfStorageAccess, uint64_t openerPageID, String&& openerPrimaryDomain)
431 {
432     ASSERT(primaryDomainInNeedOfStorageAccess != openerPrimaryDomain);
433     ASSERT(!RunLoop::isMain());
434
435     if (primaryDomainInNeedOfStorageAccess == openerPrimaryDomain)
436         return;
437
438     auto& domainInNeedOfStorageAccessStatistic = ensureResourceStatisticsForPrimaryDomain(primaryDomainInNeedOfStorageAccess);
439     auto cookiesBlockedAndPurged = shouldBlockAndPurgeCookies(domainInNeedOfStorageAccessStatistic);
440
441     // The domain already has access if its cookies are not blocked.
442     if (!cookiesBlockedAndPurged && !shouldBlockAndKeepCookies(domainInNeedOfStorageAccessStatistic))
443         return;
444
445 #if !RELEASE_LOG_DISABLED
446     RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "[Temporary combatibility fix] Storage access was granted for %{public}s under opener page from %{public}s, with user interaction in the opened window.", primaryDomainInNeedOfStorageAccess.utf8().data(), openerPrimaryDomain.utf8().data());
447 #endif
448     grantStorageAccessInternal(WTFMove(primaryDomainInNeedOfStorageAccess), WTFMove(openerPrimaryDomain), WTF::nullopt, openerPageID, false, [](bool) { });
449 }
450
451 void ResourceLoadStatisticsMemoryStore::grantStorageAccess(String&& subFrameHost, String&& topFrameHost, uint64_t frameID, uint64_t pageID, bool userWasPromptedNow, CompletionHandler<void(bool)>&& completionHandler)
452 {
453     ASSERT(!RunLoop::isMain());
454
455     auto subFramePrimaryDomain = ResourceLoadStatistics::primaryDomain(subFrameHost);
456     auto topFramePrimaryDomain = ResourceLoadStatistics::primaryDomain(topFrameHost);
457     if (userWasPromptedNow) {
458         auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
459         ASSERT(subFrameStatistic.hadUserInteraction);
460         subFrameStatistic.storageAccessUnderTopFrameOrigins.add(topFramePrimaryDomain);
461     }
462     grantStorageAccessInternal(WTFMove(subFramePrimaryDomain), WTFMove(topFramePrimaryDomain), frameID, pageID, userWasPromptedNow, WTFMove(completionHandler));
463 }
464
465 void ResourceLoadStatisticsMemoryStore::grantStorageAccessInternal(String&& subFramePrimaryDomain, String&& topFramePrimaryDomain, Optional<uint64_t> frameID, uint64_t pageID, bool userWasPromptedNowOrEarlier, CompletionHandler<void(bool)>&& callback)
466 {
467     ASSERT(!RunLoop::isMain());
468
469     if (subFramePrimaryDomain == topFramePrimaryDomain) {
470         callback(true);
471         return;
472     }
473
474     // FIXME: Remove m_storageAccessPromptsEnabled check if prompting is no longer experimental.
475     if (userWasPromptedNowOrEarlier && m_storageAccessPromptsEnabled) {
476         auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
477         ASSERT(subFrameStatistic.hadUserInteraction);
478         ASSERT(subFrameStatistic.storageAccessUnderTopFrameOrigins.contains(topFramePrimaryDomain));
479         subFrameStatistic.mostRecentUserInteractionTime = WallTime::now();
480     }
481
482     RunLoop::main().dispatch([subFramePrimaryDomain = subFramePrimaryDomain.isolatedCopy(), topFramePrimaryDomain = topFramePrimaryDomain.isolatedCopy(), frameID, pageID, store = makeRef(m_store), callback = WTFMove(callback)]() mutable {
483         store->callGrantStorageAccessHandler(subFramePrimaryDomain, topFramePrimaryDomain, frameID, pageID, [callback = WTFMove(callback), store = store.copyRef()](bool value) mutable {
484             store->statisticsQueue().dispatch([callback = WTFMove(callback), value] () mutable {
485                 callback(value);
486             });
487         });
488     });
489 }
490
491 void ResourceLoadStatisticsMemoryStore::grandfatherExistingWebsiteData(CompletionHandler<void()>&& callback)
492 {
493     ASSERT(!RunLoop::isMain());
494
495     RunLoop::main().dispatch([weakThis = makeWeakPtr(*this), callback = WTFMove(callback), shouldNotifyPagesWhenDataRecordsWereScanned = m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, workQueue = m_workQueue.copyRef()] () mutable {
496         // FIXME: This method being a static call on WebProcessProxy is wrong.
497         // It should be on the data store that this object belongs to.
498         WebProcessProxy::topPrivatelyControlledDomainsWithWebsiteData(WebResourceLoadStatisticsStore::monitoredDataTypes(), shouldNotifyPagesWhenDataRecordsWereScanned, [weakThis = WTFMove(weakThis), callback = WTFMove(callback), workQueue = workQueue.copyRef()] (HashSet<String>&& topPrivatelyControlledDomainsWithWebsiteData) mutable {
499             workQueue->dispatch([weakThis = WTFMove(weakThis), topDomains = crossThreadCopy(topPrivatelyControlledDomainsWithWebsiteData), callback = WTFMove(callback)] () mutable {
500                 if (!weakThis) {
501                     callback();
502                     return;
503                 }
504
505                 for (auto& topPrivatelyControlledDomain : topDomains) {
506                     auto& statistic = weakThis->ensureResourceStatisticsForPrimaryDomain(topPrivatelyControlledDomain);
507                     statistic.grandfathered = true;
508                 }
509                 weakThis->m_endOfGrandfatheringTimestamp = WallTime::now() + weakThis->m_parameters.grandfatheringTime;
510                 if (weakThis->m_persistentStorage)
511                     weakThis->m_persistentStorage->scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::Yes);
512                 callback();
513                 weakThis->logTestingEvent("Grandfathered"_s);
514             });
515         });
516     });
517 }
518
519 Vector<String> ResourceLoadStatisticsMemoryStore::ensurePrevalentResourcesForDebugMode()
520 {
521     if (!m_debugModeEnabled)
522         return { };
523
524     Vector<String> primaryDomainsToBlock;
525     primaryDomainsToBlock.reserveInitialCapacity(2);
526
527     auto& staticSesourceStatistic = ensureResourceStatisticsForPrimaryDomain(debugStaticPrevalentResource);
528     setPrevalentResource(staticSesourceStatistic, ResourceLoadPrevalence::High);
529     primaryDomainsToBlock.uncheckedAppend(debugStaticPrevalentResource);
530
531     if (!m_debugManualPrevalentResource.isEmpty()) {
532         auto& manualResourceStatistic = ensureResourceStatisticsForPrimaryDomain(m_debugManualPrevalentResource);
533         setPrevalentResource(manualResourceStatistic, ResourceLoadPrevalence::High);
534         primaryDomainsToBlock.uncheckedAppend(m_debugManualPrevalentResource);
535     }
536     
537     return primaryDomainsToBlock;
538 }
539
540 void ResourceLoadStatisticsMemoryStore::setResourceLoadStatisticsDebugMode(bool enable)
541 {
542     ASSERT(!RunLoop::isMain());
543
544     m_debugModeEnabled = enable;
545     m_debugLoggingEnabled = enable;
546
547     ensurePrevalentResourcesForDebugMode();
548 }
549
550 void ResourceLoadStatisticsMemoryStore::setPrevalentResourceForDebugMode(const String& domain)
551 {
552     if (!m_debugModeEnabled)
553         return;
554
555     m_debugManualPrevalentResource = domain;
556     auto& resourceStatistic = ensureResourceStatisticsForPrimaryDomain(domain);
557     setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
558
559 #if !RELEASE_LOG_DISABLED
560     RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "Did set %{public}s as prevalent resource for the purposes of ITP Debug Mode.", domain.utf8().data());
561 #endif
562 }
563
564 void ResourceLoadStatisticsMemoryStore::scheduleStatisticsProcessingRequestIfNecessary()
565 {
566     ASSERT(!RunLoop::isMain());
567
568     m_pendingStatisticsProcessingRequestIdentifier = ++m_lastStatisticsProcessingRequestIdentifier;
569     m_workQueue->dispatchAfter(minimumStatisticsProcessingInterval, [this, weakThis = makeWeakPtr(*this), statisticsProcessingRequestIdentifier = *m_pendingStatisticsProcessingRequestIdentifier] {
570         if (!weakThis)
571             return;
572
573         if (!m_pendingStatisticsProcessingRequestIdentifier || *m_pendingStatisticsProcessingRequestIdentifier != statisticsProcessingRequestIdentifier) {
574             // This request has been canceled.
575             return;
576         }
577
578         updateCookieBlocking([]() { });
579         processStatisticsAndDataRecords();
580     });
581 }
582
583 void ResourceLoadStatisticsMemoryStore::cancelPendingStatisticsProcessingRequest()
584 {
585     ASSERT(!RunLoop::isMain());
586
587     m_pendingStatisticsProcessingRequestIdentifier = WTF::nullopt;
588 }
589
590 void ResourceLoadStatisticsMemoryStore::logFrameNavigation(const String& targetPrimaryDomain, const String& mainFramePrimaryDomain, const String& sourcePrimaryDomain, const String& targetHost, const String& mainFrameHost, bool isRedirect, bool isMainFrame)
591 {
592     ASSERT(!RunLoop::isMain());
593
594     bool areTargetAndMainFrameDomainsAssociated = targetPrimaryDomain == mainFramePrimaryDomain;
595     bool areTargetAndSourceDomainsAssociated = targetPrimaryDomain == sourcePrimaryDomain;
596
597     bool statisticsWereUpdated = false;
598     if (!isMainFrame && targetHost != mainFrameHost && !(areTargetAndMainFrameDomainsAssociated || areTargetAndSourceDomainsAssociated)) {
599         auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
600         targetStatistics.lastSeen = ResourceLoadStatistics::reduceTimeResolution(WallTime::now());
601         if (targetStatistics.subframeUnderTopFrameOrigins.add(mainFramePrimaryDomain).isNewEntry)
602             statisticsWereUpdated = true;
603     }
604
605     if (isRedirect && !areTargetAndSourceDomainsAssociated) {
606         if (isMainFrame) {
607             auto& redirectingOriginStatistics = ensureResourceStatisticsForPrimaryDomain(sourcePrimaryDomain);
608             if (redirectingOriginStatistics.topFrameUniqueRedirectsTo.add(targetPrimaryDomain).isNewEntry)
609                 statisticsWereUpdated = true;
610             auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
611             if (targetStatistics.topFrameUniqueRedirectsFrom.add(sourcePrimaryDomain).isNewEntry)
612                 statisticsWereUpdated = true;
613         } else {
614             auto& redirectingOriginStatistics = ensureResourceStatisticsForPrimaryDomain(sourcePrimaryDomain);
615             if (redirectingOriginStatistics.subresourceUniqueRedirectsTo.add(targetPrimaryDomain).isNewEntry)
616                 statisticsWereUpdated = true;
617             auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
618             if (targetStatistics.subresourceUniqueRedirectsFrom.add(sourcePrimaryDomain).isNewEntry)
619                 statisticsWereUpdated = true;
620         }
621     }
622
623     if (statisticsWereUpdated)
624         scheduleStatisticsProcessingRequestIfNecessary();
625 }
626
627 void ResourceLoadStatisticsMemoryStore::logUserInteraction(const String& primaryDomain)
628 {
629     ASSERT(!RunLoop::isMain());
630
631     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
632     statistics.hadUserInteraction = true;
633     statistics.mostRecentUserInteractionTime = WallTime::now();
634 }
635
636 void ResourceLoadStatisticsMemoryStore::clearUserInteraction(const String& primaryDomain)
637 {
638     ASSERT(!RunLoop::isMain());
639
640     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
641     statistics.hadUserInteraction = false;
642     statistics.mostRecentUserInteractionTime = { };
643 }
644
645 bool ResourceLoadStatisticsMemoryStore::hasHadUserInteraction(const String& primaryDomain)
646 {
647     ASSERT(!RunLoop::isMain());
648
649     auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
650     return mapEntry == m_resourceStatisticsMap.end() ? false: hasHadUnexpiredRecentUserInteraction(mapEntry->value);
651 }
652
653 void ResourceLoadStatisticsMemoryStore::setPrevalentResource(WebCore::ResourceLoadStatistics& resourceStatistic, ResourceLoadPrevalence newPrevalence)
654 {
655     ASSERT(!RunLoop::isMain());
656
657     resourceStatistic.isPrevalentResource = true;
658     resourceStatistic.isVeryPrevalentResource = newPrevalence == ResourceLoadPrevalence::VeryHigh;
659     HashSet<String> domainsThatHaveRedirectedTo;
660     recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(resourceStatistic, domainsThatHaveRedirectedTo, 0);
661     for (auto& domain : domainsThatHaveRedirectedTo) {
662         auto mapEntry = m_resourceStatisticsMap.find(domain);
663         if (mapEntry == m_resourceStatisticsMap.end())
664             continue;
665         ASSERT(!mapEntry->value.isPrevalentResource);
666         mapEntry->value.isPrevalentResource = true;
667     }
668 }
669     
670 String ResourceLoadStatisticsMemoryStore::dumpResourceLoadStatistics() const
671 {
672     ASSERT(!RunLoop::isMain());
673
674     StringBuilder result;
675     result.appendLiteral("Resource load statistics:\n\n");
676     for (auto& mapEntry : m_resourceStatisticsMap.values())
677         result.append(mapEntry.toString());
678     return result.toString();
679 }
680
681 bool ResourceLoadStatisticsMemoryStore::isPrevalentResource(const String& primaryDomain) const
682 {
683     ASSERT(!RunLoop::isMain());
684
685     auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
686     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource;
687 }
688
689 bool ResourceLoadStatisticsMemoryStore::isVeryPrevalentResource(const String& primaryDomain) const
690 {
691     ASSERT(!RunLoop::isMain());
692
693     auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
694     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource && mapEntry->value.isVeryPrevalentResource;
695 }
696
697 bool ResourceLoadStatisticsMemoryStore::isRegisteredAsSubresourceUnder(const String& subresourcePrimaryDomain, const String& topFramePrimaryDomain) const
698 {
699     ASSERT(!RunLoop::isMain());
700
701     auto mapEntry = m_resourceStatisticsMap.find(subresourcePrimaryDomain);
702     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subresourceUnderTopFrameOrigins.contains(topFramePrimaryDomain);
703 }
704
705 bool ResourceLoadStatisticsMemoryStore::isRegisteredAsSubFrameUnder(const String& subFramePrimaryDomain, const String& topFramePrimaryDomain) const
706 {
707     ASSERT(!RunLoop::isMain());
708
709     auto mapEntry = m_resourceStatisticsMap.find(subFramePrimaryDomain);
710     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subframeUnderTopFrameOrigins.contains(topFramePrimaryDomain);
711 }
712
713 bool ResourceLoadStatisticsMemoryStore::isRegisteredAsRedirectingTo(const String& hostRedirectedFromPrimaryDomain, const String& hostRedirectedToPrimaryDomain) const
714 {
715     ASSERT(!RunLoop::isMain());
716
717     auto mapEntry = m_resourceStatisticsMap.find(hostRedirectedFromPrimaryDomain);
718     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subresourceUniqueRedirectsTo.contains(hostRedirectedToPrimaryDomain);
719 }
720
721 void ResourceLoadStatisticsMemoryStore::clearPrevalentResource(const String& primaryDomain)
722 {
723     ASSERT(!RunLoop::isMain());
724
725     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
726     statistics.isPrevalentResource = false;
727     statistics.isVeryPrevalentResource = false;
728 }
729
730 void ResourceLoadStatisticsMemoryStore::setGrandfathered(const String& primaryDomain, bool value)
731 {
732     ASSERT(!RunLoop::isMain());
733
734     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
735     statistics.grandfathered = value;
736 }
737
738 bool ResourceLoadStatisticsMemoryStore::isGrandfathered(const String& primaryDomain) const
739 {
740     ASSERT(!RunLoop::isMain());
741
742     auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
743     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.grandfathered;
744 }
745
746 void ResourceLoadStatisticsMemoryStore::setSubframeUnderTopFrameOrigin(const String& primarySubFrameDomain, const String& primaryTopFrameDomain)
747 {
748     ASSERT(!RunLoop::isMain());
749
750     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubFrameDomain);
751     statistics.subframeUnderTopFrameOrigins.add(primaryTopFrameDomain);
752     // For consistency, make sure we also have a statistics entry for the top frame domain.
753     ensureResourceStatisticsForPrimaryDomain(primaryTopFrameDomain);
754 }
755
756 void ResourceLoadStatisticsMemoryStore::setSubresourceUnderTopFrameOrigin(const String& primarySubresourceDomain, const String& primaryTopFrameDomain)
757 {
758     ASSERT(!RunLoop::isMain());
759
760     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
761     statistics.subresourceUnderTopFrameOrigins.add(primaryTopFrameDomain);
762     // For consistency, make sure we also have a statistics entry for the top frame domain.
763     ensureResourceStatisticsForPrimaryDomain(primaryTopFrameDomain);
764 }
765
766 void ResourceLoadStatisticsMemoryStore::setSubresourceUniqueRedirectTo(const String& primarySubresourceDomain, const String& primaryRedirectDomain)
767 {
768     ASSERT(!RunLoop::isMain());
769
770     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
771     statistics.subresourceUniqueRedirectsTo.add(primaryRedirectDomain);
772     // For consistency, make sure we also have a statistics entry for the redirect domain.
773     ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
774 }
775
776 void ResourceLoadStatisticsMemoryStore::setSubresourceUniqueRedirectFrom(const String& primarySubresourceDomain, const String& primaryRedirectDomain)
777 {
778     ASSERT(!RunLoop::isMain());
779
780     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
781     statistics.subresourceUniqueRedirectsFrom.add(primaryRedirectDomain);
782     // For consistency, make sure we also have a statistics entry for the redirect domain.
783     ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
784 }
785
786 void ResourceLoadStatisticsMemoryStore::setTopFrameUniqueRedirectTo(const String& topFramePrimaryDomain, const String& primaryRedirectDomain)
787 {
788     ASSERT(!RunLoop::isMain());
789
790     auto& statistics = ensureResourceStatisticsForPrimaryDomain(topFramePrimaryDomain);
791     statistics.topFrameUniqueRedirectsTo.add(primaryRedirectDomain);
792     // For consistency, make sure we also have a statistics entry for the redirect domain.
793     ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
794 }
795
796 void ResourceLoadStatisticsMemoryStore::setTopFrameUniqueRedirectFrom(const String& topFramePrimaryDomain, const String& primaryRedirectDomain)
797 {
798     ASSERT(!RunLoop::isMain());
799
800     auto& statistics = ensureResourceStatisticsForPrimaryDomain(topFramePrimaryDomain);
801     statistics.topFrameUniqueRedirectsFrom.add(primaryRedirectDomain);
802     // For consistency, make sure we also have a statistics entry for the redirect domain.
803     ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
804 }
805
806 void ResourceLoadStatisticsMemoryStore::setTimeToLiveUserInteraction(Seconds seconds)
807 {
808     ASSERT(!RunLoop::isMain());
809     ASSERT(seconds >= 0_s);
810
811     m_parameters.timeToLiveUserInteraction = seconds;
812 }
813
814 void ResourceLoadStatisticsMemoryStore::setMinimumTimeBetweenDataRecordsRemoval(Seconds seconds)
815 {
816     ASSERT(!RunLoop::isMain());
817     ASSERT(seconds >= 0_s);
818
819     m_parameters.minimumTimeBetweenDataRecordsRemoval = seconds;
820 }
821
822 void ResourceLoadStatisticsMemoryStore::setGrandfatheringTime(Seconds seconds)
823 {
824     ASSERT(!RunLoop::isMain());
825     ASSERT(seconds >= 0_s);
826
827     m_parameters.grandfatheringTime = seconds;
828 }
829
830 void ResourceLoadStatisticsMemoryStore::setCacheMaxAgeCap(Seconds seconds)
831 {
832     ASSERT(!RunLoop::isMain());
833     ASSERT(seconds >= 0_s);
834
835     m_parameters.cacheMaxAgeCapTime = seconds;
836     updateCacheMaxAgeCap();
837 }
838
839 void ResourceLoadStatisticsMemoryStore::updateCacheMaxAgeCap()
840 {
841     ASSERT(!RunLoop::isMain());
842     
843     RunLoop::main().dispatch([store = makeRef(m_store), seconds = m_parameters.cacheMaxAgeCapTime] () {
844         store->setCacheMaxAgeCap(seconds, [] { });
845     });
846 }
847
848 void ResourceLoadStatisticsMemoryStore::setAgeCapForClientSideCookies(Seconds seconds)
849 {
850     ASSERT(!RunLoop::isMain());
851     ASSERT(seconds >= 0_s);
852     
853     m_parameters.clientSideCookiesAgeCapTime = seconds;
854     updateClientSideCookiesAgeCap();
855 }
856
857 void ResourceLoadStatisticsMemoryStore::updateClientSideCookiesAgeCap()
858 {
859     ASSERT(!RunLoop::isMain());
860
861 #if ENABLE(RESOURCE_LOAD_STATISTICS)
862     RunLoop::main().dispatch([store = makeRef(m_store), seconds = m_parameters.clientSideCookiesAgeCapTime] () {
863         if (auto* websiteDataStore = store->websiteDataStore())
864             websiteDataStore->setAgeCapForClientSideCookies(seconds, [] { });
865     });
866 #endif
867 }
868
869 bool ResourceLoadStatisticsMemoryStore::shouldRemoveDataRecords() const
870 {
871     ASSERT(!RunLoop::isMain());
872
873     if (m_dataRecordsBeingRemoved)
874         return false;
875
876 #if ENABLE(NETSCAPE_PLUGIN_API)
877     for (const auto& plugin : PluginProcessManager::singleton().pluginProcesses()) {
878         if (!m_activePluginTokens.contains(plugin->pluginProcessToken()))
879             return true;
880     }
881 #endif
882
883     return !m_lastTimeDataRecordsWereRemoved || MonotonicTime::now() >= (m_lastTimeDataRecordsWereRemoved + m_parameters.minimumTimeBetweenDataRecordsRemoval);
884 }
885
886 void ResourceLoadStatisticsMemoryStore::setDataRecordsBeingRemoved(bool value)
887 {
888     ASSERT(!RunLoop::isMain());
889
890     m_dataRecordsBeingRemoved = value;
891     if (m_dataRecordsBeingRemoved)
892         m_lastTimeDataRecordsWereRemoved = MonotonicTime::now();
893 }
894
895 ResourceLoadStatistics& ResourceLoadStatisticsMemoryStore::ensureResourceStatisticsForPrimaryDomain(const String& primaryDomain)
896 {
897     ASSERT(!RunLoop::isMain());
898
899     return m_resourceStatisticsMap.ensure(primaryDomain, [&primaryDomain] {
900         return ResourceLoadStatistics(primaryDomain);
901     }).iterator->value;
902 }
903
904 std::unique_ptr<KeyedEncoder> ResourceLoadStatisticsMemoryStore::createEncoderFromData() const
905 {
906     ASSERT(!RunLoop::isMain());
907
908     auto encoder = KeyedEncoder::encoder();
909     encoder->encodeUInt32("version", statisticsModelVersion);
910     encoder->encodeDouble("endOfGrandfatheringTimestamp", m_endOfGrandfatheringTimestamp.secondsSinceEpoch().value());
911
912     encoder->encodeObjects("browsingStatistics", m_resourceStatisticsMap.begin(), m_resourceStatisticsMap.end(), [](KeyedEncoder& encoderInner, const auto& origin) {
913         origin.value.encode(encoderInner);
914     });
915
916     encoder->encodeObjects("operatingDates", m_operatingDates.begin(), m_operatingDates.end(), [](KeyedEncoder& encoderInner, OperatingDate date) {
917         encoderInner.encodeDouble("date", date.secondsSinceEpoch().value());
918     });
919
920     return encoder;
921 }
922
923 void ResourceLoadStatisticsMemoryStore::mergeWithDataFromDecoder(KeyedDecoder& decoder)
924 {
925     ASSERT(!RunLoop::isMain());
926
927     unsigned versionOnDisk;
928     if (!decoder.decodeUInt32("version", versionOnDisk))
929         return;
930
931     if (versionOnDisk > statisticsModelVersion) {
932         WTFLogAlways("Found resource load statistics on disk with model version %u whereas the highest supported version is %u. Resetting.", versionOnDisk, statisticsModelVersion);
933         return;
934     }
935
936     double endOfGrandfatheringTimestamp;
937     if (decoder.decodeDouble("endOfGrandfatheringTimestamp", endOfGrandfatheringTimestamp))
938         m_endOfGrandfatheringTimestamp = WallTime::fromRawSeconds(endOfGrandfatheringTimestamp);
939     else
940         m_endOfGrandfatheringTimestamp = { };
941
942     Vector<ResourceLoadStatistics> loadedStatistics;
943     bool succeeded = decoder.decodeObjects("browsingStatistics", loadedStatistics, [versionOnDisk](KeyedDecoder& decoderInner, ResourceLoadStatistics& statistics) {
944         return statistics.decode(decoderInner, versionOnDisk);
945     });
946
947     if (!succeeded)
948         return;
949
950     mergeStatistics(WTFMove(loadedStatistics));
951     updateCookieBlocking([]() { });
952
953     Vector<OperatingDate> operatingDates;
954     succeeded = decoder.decodeObjects("operatingDates", operatingDates, [](KeyedDecoder& decoder, OperatingDate& date) {
955         double value;
956         if (!decoder.decodeDouble("date", value))
957             return false;
958
959         date = OperatingDate::fromWallTime(WallTime::fromRawSeconds(value));
960         return true;
961     });
962
963     if (!succeeded)
964         return;
965
966     m_operatingDates = mergeOperatingDates(m_operatingDates, WTFMove(operatingDates));
967 }
968
969 void ResourceLoadStatisticsMemoryStore::clear(CompletionHandler<void()>&& completionHandler)
970 {
971     ASSERT(!RunLoop::isMain());
972
973     m_resourceStatisticsMap.clear();
974     m_operatingDates.clear();
975
976     auto callbackAggregator = CallbackAggregator::create(WTFMove(completionHandler));
977
978     removeAllStorageAccess([callbackAggregator = callbackAggregator.copyRef()] { });
979
980     auto primaryDomainsToBlock = ensurePrevalentResourcesForDebugMode();
981     updateCookieBlockingForDomains(primaryDomainsToBlock, [callbackAggregator = callbackAggregator.copyRef()] { });
982 }
983
984 bool ResourceLoadStatisticsMemoryStore::wasAccessedAsFirstPartyDueToUserInteraction(const ResourceLoadStatistics& current, const ResourceLoadStatistics& updated) const
985 {
986     if (!current.hadUserInteraction && !updated.hadUserInteraction)
987         return false;
988
989     auto mostRecentUserInteractionTime = std::max(current.mostRecentUserInteractionTime, updated.mostRecentUserInteractionTime);
990
991     return updated.lastSeen <= mostRecentUserInteractionTime + 24_h;
992 }
993
994 void ResourceLoadStatisticsMemoryStore::mergeStatistics(Vector<ResourceLoadStatistics>&& statistics)
995 {
996     ASSERT(!RunLoop::isMain());
997
998     for (auto& statistic : statistics) {
999         auto result = m_resourceStatisticsMap.ensure(statistic.highLevelDomain, [&statistic] {
1000             return WTFMove(statistic);
1001         });
1002         if (!result.isNewEntry) {
1003             if (wasAccessedAsFirstPartyDueToUserInteraction(result.iterator->value, statistic))
1004                 result.iterator->value.timesAccessedAsFirstPartyDueToUserInteraction++;
1005             result.iterator->value.merge(statistic);
1006         }
1007     }
1008 }
1009
1010 bool ResourceLoadStatisticsMemoryStore::shouldBlockAndKeepCookies(const ResourceLoadStatistics& statistic)
1011 {
1012     return statistic.isPrevalentResource && statistic.hadUserInteraction;
1013 }
1014
1015 bool ResourceLoadStatisticsMemoryStore::shouldBlockAndPurgeCookies(const ResourceLoadStatistics& statistic)
1016 {
1017     return statistic.isPrevalentResource && !statistic.hadUserInteraction;
1018 }
1019
1020 bool ResourceLoadStatisticsMemoryStore::hasUserGrantedStorageAccessThroughPrompt(const ResourceLoadStatistics& statistic, const String& firstPartyPrimaryDomain)
1021 {
1022     return statistic.storageAccessUnderTopFrameOrigins.contains(firstPartyPrimaryDomain);
1023 }
1024
1025 static void debugLogDomainsInBatches(const char* action, const Vector<String>& domains)
1026 {
1027 #if !RELEASE_LOG_DISABLED
1028     static const auto maxNumberOfDomainsInOneLogStatement = 50;
1029     if (domains.isEmpty())
1030         return;
1031
1032     if (domains.size() <= maxNumberOfDomainsInOneLogStatement) {
1033         RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "About to %{public}s cookies in third-party contexts for: %{public}s.", action, domainsToString(domains).utf8().data());
1034         return;
1035     }
1036     
1037     Vector<String> batch;
1038     batch.reserveInitialCapacity(maxNumberOfDomainsInOneLogStatement);
1039     auto batchNumber = 1;
1040     unsigned numberOfBatches = std::ceil(domains.size() / static_cast<float>(maxNumberOfDomainsInOneLogStatement));
1041
1042     for (auto& domain : domains) {
1043         if (batch.size() == maxNumberOfDomainsInOneLogStatement) {
1044             RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "About to %{public}s cookies in third-party contexts for (%{public}d of %u): %{public}s.", action, batchNumber, numberOfBatches, domainsToString(batch).utf8().data());
1045             batch.shrink(0);
1046             ++batchNumber;
1047         }
1048         batch.append(domain);
1049     }
1050     if (!batch.isEmpty())
1051         RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "About to %{public}s cookies in third-party contexts for (%{public}d of %u): %{public}s.", action, batchNumber, numberOfBatches, domainsToString(batch).utf8().data());
1052 #else
1053     UNUSED_PARAM(action);
1054     UNUSED_PARAM(domains);
1055 #endif
1056 }
1057
1058 void ResourceLoadStatisticsMemoryStore::updateCookieBlocking(CompletionHandler<void()>&& completionHandler)
1059 {
1060     ASSERT(!RunLoop::isMain());
1061
1062     Vector<String> domainsToBlock;
1063     for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
1064         if (resourceStatistic.isPrevalentResource)
1065             domainsToBlock.append(resourceStatistic.highLevelDomain);
1066     }
1067
1068     if (domainsToBlock.isEmpty()) {
1069         completionHandler();
1070         return;
1071     }
1072
1073     if (m_debugLoggingEnabled && !domainsToBlock.isEmpty())
1074         debugLogDomainsInBatches("block", domainsToBlock);
1075
1076     RunLoop::main().dispatch([weakThis = makeWeakPtr(*this), store = makeRef(m_store), domainsToBlock = crossThreadCopy(domainsToBlock), completionHandler = WTFMove(completionHandler)] () mutable {
1077         store->callUpdatePrevalentDomainsToBlockCookiesForHandler(domainsToBlock, [weakThis = WTFMove(weakThis), store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1078             store->statisticsQueue().dispatch([weakThis = WTFMove(weakThis), completionHandler = WTFMove(completionHandler)]() mutable {
1079                 completionHandler();
1080                 if (!weakThis)
1081                     return;
1082 #if !RELEASE_LOG_DISABLED
1083                 RELEASE_LOG_INFO_IF(weakThis->m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Done updating cookie blocking.");
1084 #endif
1085             });
1086         });
1087     });
1088 }
1089
1090 void ResourceLoadStatisticsMemoryStore::updateCookieBlockingForDomains(const Vector<String>& domainsToBlock, CompletionHandler<void()>&& completionHandler)
1091 {
1092     ASSERT(!RunLoop::isMain());
1093
1094     RunLoop::main().dispatch([store = makeRef(m_store), domainsToBlock = crossThreadCopy(domainsToBlock), completionHandler = WTFMove(completionHandler)] () mutable {
1095         store->callUpdatePrevalentDomainsToBlockCookiesForHandler(domainsToBlock, [store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1096             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler)]() mutable {
1097                 completionHandler();
1098             });
1099         });
1100     });
1101 }
1102
1103 void ResourceLoadStatisticsMemoryStore::clearBlockingStateForDomains(const Vector<String>& domains, CompletionHandler<void()>&& completionHandler)
1104 {
1105     ASSERT(!RunLoop::isMain());
1106
1107     if (domains.isEmpty()) {
1108         completionHandler();
1109         return;
1110     }
1111
1112     RunLoop::main().dispatch([store = makeRef(m_store), domains = crossThreadCopy(domains)] {
1113         store->callRemoveDomainsHandler(domains);
1114     });
1115
1116     completionHandler();
1117 }
1118
1119 void ResourceLoadStatisticsMemoryStore::processStatistics(const Function<void(const ResourceLoadStatistics&)>& processFunction) const
1120 {
1121     ASSERT(!RunLoop::isMain());
1122
1123     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
1124         processFunction(resourceStatistic);
1125 }
1126
1127 bool ResourceLoadStatisticsMemoryStore::hasHadUnexpiredRecentUserInteraction(ResourceLoadStatistics& resourceStatistic) const
1128 {
1129     ASSERT(!RunLoop::isMain());
1130
1131     if (resourceStatistic.hadUserInteraction && hasStatisticsExpired(resourceStatistic)) {
1132         // Drop privacy sensitive data because we no longer need it.
1133         // Set timestamp to 0 so that statistics merge will know
1134         // it has been reset as opposed to its default -1.
1135         resourceStatistic.mostRecentUserInteractionTime = { };
1136         resourceStatistic.storageAccessUnderTopFrameOrigins.clear();
1137         resourceStatistic.hadUserInteraction = false;
1138     }
1139
1140     return resourceStatistic.hadUserInteraction;
1141 }
1142
1143 Vector<String> ResourceLoadStatisticsMemoryStore::topPrivatelyControlledDomainsToRemoveWebsiteDataFor()
1144 {
1145     ASSERT(!RunLoop::isMain());
1146
1147     bool shouldCheckForGrandfathering = m_endOfGrandfatheringTimestamp > WallTime::now();
1148     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && m_endOfGrandfatheringTimestamp;
1149
1150     if (shouldClearGrandfathering)
1151         m_endOfGrandfatheringTimestamp = { };
1152
1153     Vector<String> prevalentResources;
1154     for (auto& statistic : m_resourceStatisticsMap.values()) {
1155         if (statistic.isPrevalentResource && !hasHadUnexpiredRecentUserInteraction(statistic) && (!shouldCheckForGrandfathering || !statistic.grandfathered))
1156             prevalentResources.append(statistic.highLevelDomain);
1157
1158         if (shouldClearGrandfathering && statistic.grandfathered)
1159             statistic.grandfathered = false;
1160     }
1161
1162     return prevalentResources;
1163 }
1164
1165 void ResourceLoadStatisticsMemoryStore::includeTodayAsOperatingDateIfNecessary()
1166 {
1167     ASSERT(!RunLoop::isMain());
1168
1169     auto today = OperatingDate::today();
1170     if (!m_operatingDates.isEmpty() && today <= m_operatingDates.last())
1171         return;
1172
1173     while (m_operatingDates.size() >= operatingDatesWindow)
1174         m_operatingDates.remove(0);
1175
1176     m_operatingDates.append(today);
1177 }
1178
1179 bool ResourceLoadStatisticsMemoryStore::hasStatisticsExpired(const ResourceLoadStatistics& resourceStatistic) const
1180 {
1181     ASSERT(!RunLoop::isMain());
1182
1183     if (m_operatingDates.size() >= operatingDatesWindow) {
1184         if (OperatingDate::fromWallTime(resourceStatistic.mostRecentUserInteractionTime) < m_operatingDates.first())
1185             return true;
1186     }
1187
1188     // If we don't meet the real criteria for an expired statistic, check the user setting for a tighter restriction (mainly for testing).
1189     if (m_parameters.timeToLiveUserInteraction) {
1190         if (WallTime::now() > resourceStatistic.mostRecentUserInteractionTime + m_parameters.timeToLiveUserInteraction.value())
1191             return true;
1192     }
1193
1194     return false;
1195 }
1196
1197 void ResourceLoadStatisticsMemoryStore::setMaxStatisticsEntries(size_t maximumEntryCount)
1198 {
1199     ASSERT(!RunLoop::isMain());
1200
1201     m_parameters.maxStatisticsEntries = maximumEntryCount;
1202 }
1203
1204 void ResourceLoadStatisticsMemoryStore::setPruneEntriesDownTo(size_t pruneTargetCount)
1205 {
1206     ASSERT(!RunLoop::isMain());
1207
1208     m_parameters.pruneEntriesDownTo = pruneTargetCount;
1209 }
1210
1211 void ResourceLoadStatisticsMemoryStore::pruneStatisticsIfNeeded()
1212 {
1213     ASSERT(!RunLoop::isMain());
1214
1215     if (m_resourceStatisticsMap.size() <= m_parameters.maxStatisticsEntries)
1216         return;
1217
1218     ASSERT(m_parameters.pruneEntriesDownTo <= m_parameters.maxStatisticsEntries);
1219
1220     size_t numberOfEntriesLeftToPrune = m_resourceStatisticsMap.size() - m_parameters.pruneEntriesDownTo;
1221     ASSERT(numberOfEntriesLeftToPrune);
1222
1223     Vector<StatisticsLastSeen> resourcesToPrunePerImportance[maxImportance + 1];
1224     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
1225         resourcesToPrunePerImportance[computeImportance(resourceStatistic)].append({ resourceStatistic.highLevelDomain, resourceStatistic.lastSeen });
1226
1227     for (unsigned importance = 0; numberOfEntriesLeftToPrune && importance <= maxImportance; ++importance)
1228         pruneResources(m_resourceStatisticsMap, resourcesToPrunePerImportance[importance], numberOfEntriesLeftToPrune);
1229
1230     ASSERT(!numberOfEntriesLeftToPrune);
1231 }
1232
1233 void ResourceLoadStatisticsMemoryStore::resetParametersToDefaultValues()
1234 {
1235     ASSERT(!RunLoop::isMain());
1236
1237     m_parameters = { };
1238 }
1239
1240 void ResourceLoadStatisticsMemoryStore::logTestingEvent(const String& event)
1241 {
1242     ASSERT(!RunLoop::isMain());
1243
1244     RunLoop::main().dispatch([store = makeRef(m_store), event = event.isolatedCopy()] {
1245         store->logTestingEvent(event);
1246     });
1247 }
1248
1249 void ResourceLoadStatisticsMemoryStore::setLastSeen(const String& primaryDomain, Seconds seconds)
1250 {
1251     ASSERT(!RunLoop::isMain());
1252
1253     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
1254     statistics.lastSeen = WallTime::fromRawSeconds(seconds.seconds());
1255 }
1256
1257 void ResourceLoadStatisticsMemoryStore::setPrevalentResource(const String& primaryDomain)
1258 {
1259     ASSERT(!RunLoop::isMain());
1260
1261     auto& resourceStatistic = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
1262     setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
1263 }
1264
1265 void ResourceLoadStatisticsMemoryStore::setVeryPrevalentResource(const String& primaryDomain)
1266 {
1267     ASSERT(!RunLoop::isMain());
1268
1269     auto& resourceStatistic = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
1270     setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::VeryHigh);
1271 }
1272
1273 void ResourceLoadStatisticsMemoryStore::removeAllStorageAccess(CompletionHandler<void()>&& completionHandler)
1274 {
1275     ASSERT(!RunLoop::isMain());
1276     RunLoop::main().dispatch([store = makeRef(m_store), completionHandler = WTFMove(completionHandler)]() mutable {
1277         store->removeAllStorageAccess([store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1278             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler)]() mutable {
1279                 completionHandler();
1280             });
1281         });
1282     });
1283 }
1284
1285 void ResourceLoadStatisticsMemoryStore::didCreateNetworkProcess()
1286 {
1287     ASSERT(!RunLoop::isMain());
1288
1289     updateCookieBlocking([]() { });
1290     updateCacheMaxAgeCap();
1291     updateClientSideCookiesAgeCap();
1292 }
1293
1294 } // namespace WebKit