Create a WebResourceLoadStatisticsStore attached to the NetworkSession
[WebKit-https.git] / Source / WebKit / NetworkProcess / Classifier / ResourceLoadStatisticsMemoryStore.cpp
1 /*
2  * Copyright (C) 2017-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ResourceLoadStatisticsMemoryStore.h"
28
29 #include "Logging.h"
30 #include "NetworkSession.h"
31 #include "PluginProcessManager.h"
32 #include "PluginProcessProxy.h"
33 #include "ResourceLoadStatisticsPersistentStorage.h"
34 #include "WebProcessProxy.h"
35 #include "WebResourceLoadStatisticsTelemetry.h"
36 #include "WebsiteDataStore.h"
37 #include <WebCore/KeyedCoding.h>
38 #include <WebCore/NetworkStorageSession.h>
39 #include <WebCore/ResourceLoadStatistics.h>
40 #include <wtf/CallbackAggregator.h>
41 #include <wtf/DateMath.h>
42 #include <wtf/MathExtras.h>
43 #include <wtf/text/StringBuilder.h>
44
45 namespace WebKit {
46 using namespace WebCore;
47
48 constexpr unsigned statisticsModelVersion { 14 };
49 constexpr unsigned maxNumberOfRecursiveCallsInRedirectTraceBack { 50 };
50 constexpr Seconds minimumStatisticsProcessingInterval { 5_s };
51 constexpr unsigned operatingDatesWindow { 30 };
52 constexpr unsigned maxImportance { 3 };
53 static const char* debugStaticPrevalentResource { "3rdpartytestwebkit.org" };
54
55 #if !RELEASE_LOG_DISABLED
56 static String domainsToString(const Vector<String>& domains)
57 {
58     StringBuilder builder;
59     for (auto& domain : domains) {
60         if (!builder.isEmpty())
61             builder.appendLiteral(", ");
62         builder.append(domain);
63     }
64     return builder.toString();
65 }
66 #endif
67
68 class OperatingDate {
69 public:
70     OperatingDate() = default;
71
72     static OperatingDate fromWallTime(WallTime time)
73     {
74         double ms = time.secondsSinceEpoch().milliseconds();
75         int year = msToYear(ms);
76         int yearDay = dayInYear(ms, year);
77         int month = monthFromDayInYear(yearDay, isLeapYear(year));
78         int monthDay = dayInMonthFromDayInYear(yearDay, isLeapYear(year));
79
80         return OperatingDate { year, month, monthDay };
81     }
82
83     static OperatingDate today()
84     {
85         return OperatingDate::fromWallTime(WallTime::now());
86     }
87
88     Seconds secondsSinceEpoch() const
89     {
90         return Seconds { dateToDaysFrom1970(m_year, m_month, m_monthDay) * secondsPerDay };
91     }
92
93     bool operator==(const OperatingDate& other) const
94     {
95         return m_monthDay == other.m_monthDay && m_month == other.m_month && m_year == other.m_year;
96     }
97
98     bool operator<(const OperatingDate& other) const
99     {
100         return secondsSinceEpoch() < other.secondsSinceEpoch();
101     }
102
103     bool operator<=(const OperatingDate& other) const
104     {
105         return secondsSinceEpoch() <= other.secondsSinceEpoch();
106     }
107
108 private:
109     OperatingDate(int year, int month, int monthDay)
110         : m_year(year)
111         , m_month(month)
112         , m_monthDay(monthDay)
113     { }
114
115     int m_year { 0 };
116     int m_month { 0 }; // [0, 11].
117     int m_monthDay { 0 }; // [1, 31].
118 };
119
120 static Vector<OperatingDate> mergeOperatingDates(const Vector<OperatingDate>& existingDates, Vector<OperatingDate>&& newDates)
121 {
122     if (existingDates.isEmpty())
123         return WTFMove(newDates);
124
125     Vector<OperatingDate> mergedDates(existingDates.size() + newDates.size());
126
127     // Merge the two sorted vectors of dates.
128     std::merge(existingDates.begin(), existingDates.end(), newDates.begin(), newDates.end(), mergedDates.begin());
129     // Remove duplicate dates.
130     removeRepeatedElements(mergedDates);
131
132     // Drop old dates until the Vector size reaches operatingDatesWindow.
133     while (mergedDates.size() > operatingDatesWindow)
134         mergedDates.remove(0);
135
136     return mergedDates;
137 }
138
139 struct StatisticsLastSeen {
140     String topPrivatelyOwnedDomain;
141     WallTime lastSeen;
142 };
143
144 static void pruneResources(HashMap<String, WebCore::ResourceLoadStatistics>& statisticsMap, Vector<StatisticsLastSeen>& statisticsToPrune, size_t& numberOfEntriesToPrune)
145 {
146     if (statisticsToPrune.size() > numberOfEntriesToPrune) {
147         std::sort(statisticsToPrune.begin(), statisticsToPrune.end(), [](const StatisticsLastSeen& a, const StatisticsLastSeen& b) {
148             return a.lastSeen < b.lastSeen;
149         });
150     }
151
152     for (size_t i = 0, end = std::min(numberOfEntriesToPrune, statisticsToPrune.size()); i != end; ++i, --numberOfEntriesToPrune)
153         statisticsMap.remove(statisticsToPrune[i].topPrivatelyOwnedDomain);
154 }
155
156 static unsigned computeImportance(const ResourceLoadStatistics& resourceStatistic)
157 {
158     unsigned importance = maxImportance;
159     if (!resourceStatistic.isPrevalentResource)
160         importance -= 1;
161     if (!resourceStatistic.hadUserInteraction)
162         importance -= 2;
163     return importance;
164 }
165
166 ResourceLoadStatisticsMemoryStore::ResourceLoadStatisticsMemoryStore(WebResourceLoadStatisticsStore& store, WorkQueue& workQueue)
167     : m_store(store)
168     , m_workQueue(workQueue)
169 {
170     ASSERT(!RunLoop::isMain());
171
172 #if PLATFORM(COCOA)
173     registerUserDefaultsIfNeeded();
174 #endif
175     includeTodayAsOperatingDateIfNecessary();
176
177     m_workQueue->dispatchAfter(5_s, [weakThis = makeWeakPtr(*this)] {
178         if (weakThis)
179             weakThis->calculateAndSubmitTelemetry();
180     });
181 }
182
183 ResourceLoadStatisticsMemoryStore::~ResourceLoadStatisticsMemoryStore()
184 {
185     ASSERT(!RunLoop::isMain());
186 }
187
188 void ResourceLoadStatisticsMemoryStore::setPersistentStorage(ResourceLoadStatisticsPersistentStorage& persistentStorage)
189 {
190     m_persistentStorage = makeWeakPtr(persistentStorage);
191 }
192
193 void ResourceLoadStatisticsMemoryStore::calculateAndSubmitTelemetry() const
194 {
195     ASSERT(!RunLoop::isMain());
196
197     if (m_parameters.shouldSubmitTelemetry)
198         WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
199 }
200
201 void ResourceLoadStatisticsMemoryStore::setNotifyPagesWhenDataRecordsWereScanned(bool value)
202 {
203     ASSERT(!RunLoop::isMain());
204     m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned = value;
205 }
206
207 void ResourceLoadStatisticsMemoryStore::setShouldClassifyResourcesBeforeDataRecordsRemoval(bool value)
208 {
209     ASSERT(!RunLoop::isMain());
210     m_parameters.shouldClassifyResourcesBeforeDataRecordsRemoval = value;
211 }
212
213 void ResourceLoadStatisticsMemoryStore::setShouldSubmitTelemetry(bool value)
214 {
215     ASSERT(!RunLoop::isMain());
216     m_parameters.shouldSubmitTelemetry = value;
217 }
218
219 void ResourceLoadStatisticsMemoryStore::removeDataRecords(CompletionHandler<void()>&& callback)
220 {
221     ASSERT(!RunLoop::isMain());
222
223     if (!shouldRemoveDataRecords()) {
224         callback();
225         return;
226     }
227
228 #if ENABLE(NETSCAPE_PLUGIN_API)
229     m_activePluginTokens.clear();
230     for (const auto& plugin : PluginProcessManager::singleton().pluginProcesses())
231         m_activePluginTokens.add(plugin->pluginProcessToken());
232 #endif
233
234     auto prevalentResourceDomains = topPrivatelyControlledDomainsToRemoveWebsiteDataFor();
235     if (prevalentResourceDomains.isEmpty()) {
236         callback();
237         return;
238     }
239
240 #if !RELEASE_LOG_DISABLED
241     RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "About to remove data records for %{public}s.", domainsToString(prevalentResourceDomains).utf8().data());
242 #endif
243
244     setDataRecordsBeingRemoved(true);
245
246     RunLoop::main().dispatch([prevalentResourceDomains = crossThreadCopy(prevalentResourceDomains), callback = WTFMove(callback), weakThis = makeWeakPtr(*this), shouldNotifyPagesWhenDataRecordsWereScanned = m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, workQueue = m_workQueue.copyRef()] () mutable {
247         WebProcessProxy::deleteWebsiteDataForTopPrivatelyControlledDomainsInAllPersistentDataStores(WebResourceLoadStatisticsStore::monitoredDataTypes(), WTFMove(prevalentResourceDomains), shouldNotifyPagesWhenDataRecordsWereScanned, [callback = WTFMove(callback), weakThis = WTFMove(weakThis), workQueue = workQueue.copyRef()](const HashSet<String>& domainsWithDeletedWebsiteData) mutable {
248             workQueue->dispatch([topDomains = crossThreadCopy(domainsWithDeletedWebsiteData), callback = WTFMove(callback), weakThis = WTFMove(weakThis)] () mutable {
249                 if (!weakThis) {
250                     callback();
251                     return;
252                 }
253                 for (auto& prevalentResourceDomain : topDomains) {
254                     auto& statistic = weakThis->ensureResourceStatisticsForPrimaryDomain(prevalentResourceDomain);
255                     ++statistic.dataRecordsRemoved;
256                 }
257                 weakThis->setDataRecordsBeingRemoved(false);
258                 callback();
259 #if !RELEASE_LOG_DISABLED
260                 RELEASE_LOG_INFO_IF(weakThis->m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Done removing data records.");
261 #endif
262             });
263         });
264     });
265 }
266
267 unsigned ResourceLoadStatisticsMemoryStore::recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(const WebCore::ResourceLoadStatistics& resourceStatistic, HashSet<String>& domainsThatHaveRedirectedTo, unsigned numberOfRecursiveCalls) const
268 {
269     ASSERT(!RunLoop::isMain());
270
271     if (numberOfRecursiveCalls >= maxNumberOfRecursiveCallsInRedirectTraceBack) {
272         // Model version 14 invokes a deliberate re-classification of the whole set.
273         if (statisticsModelVersion != 14)
274             ASSERT_NOT_REACHED();
275         RELEASE_LOG(ResourceLoadStatistics, "Hit %u recursive calls in redirect backtrace. Returning early.", maxNumberOfRecursiveCallsInRedirectTraceBack);
276         return numberOfRecursiveCalls;
277     }
278
279     numberOfRecursiveCalls++;
280
281     for (auto& subresourceUniqueRedirectFromDomain : resourceStatistic.subresourceUniqueRedirectsFrom.values()) {
282         auto mapEntry = m_resourceStatisticsMap.find(subresourceUniqueRedirectFromDomain);
283         if (mapEntry == m_resourceStatisticsMap.end() || mapEntry->value.isPrevalentResource)
284             continue;
285         if (domainsThatHaveRedirectedTo.add(mapEntry->value.highLevelDomain).isNewEntry)
286             numberOfRecursiveCalls = recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(mapEntry->value, domainsThatHaveRedirectedTo, numberOfRecursiveCalls);
287     }
288     for (auto& topFrameUniqueRedirectFromDomain : resourceStatistic.topFrameUniqueRedirectsFrom.values()) {
289         auto mapEntry = m_resourceStatisticsMap.find(topFrameUniqueRedirectFromDomain);
290         if (mapEntry == m_resourceStatisticsMap.end() || mapEntry->value.isPrevalentResource)
291             continue;
292         if (domainsThatHaveRedirectedTo.add(mapEntry->value.highLevelDomain).isNewEntry)
293             numberOfRecursiveCalls = recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(mapEntry->value, domainsThatHaveRedirectedTo, numberOfRecursiveCalls);
294     }
295
296     return numberOfRecursiveCalls;
297 }
298
299 void ResourceLoadStatisticsMemoryStore::markAsPrevalentIfHasRedirectedToPrevalent(WebCore::ResourceLoadStatistics& resourceStatistic)
300 {
301     ASSERT(!RunLoop::isMain());
302
303     if (resourceStatistic.isPrevalentResource)
304         return;
305
306     for (auto& subresourceDomainRedirectedTo : resourceStatistic.subresourceUniqueRedirectsTo.values()) {
307         auto mapEntry = m_resourceStatisticsMap.find(subresourceDomainRedirectedTo);
308         if (mapEntry != m_resourceStatisticsMap.end() && mapEntry->value.isPrevalentResource) {
309             setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
310             return;
311         }
312     }
313
314     for (auto& topFrameDomainRedirectedTo : resourceStatistic.topFrameUniqueRedirectsTo.values()) {
315         auto mapEntry = m_resourceStatisticsMap.find(topFrameDomainRedirectedTo);
316         if (mapEntry != m_resourceStatisticsMap.end() && mapEntry->value.isPrevalentResource) {
317             setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
318             return;
319         }
320     }
321 }
322
323 bool ResourceLoadStatisticsMemoryStore::isPrevalentDueToDebugMode(ResourceLoadStatistics& resourceStatistic)
324 {
325     if (!m_debugModeEnabled)
326         return false;
327
328     return resourceStatistic.highLevelDomain == debugStaticPrevalentResource || resourceStatistic.highLevelDomain == m_debugManualPrevalentResource;
329 }
330
331 void ResourceLoadStatisticsMemoryStore::processStatisticsAndDataRecords()
332 {
333     ASSERT(!RunLoop::isMain());
334
335     if (m_parameters.shouldClassifyResourcesBeforeDataRecordsRemoval) {
336         for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
337             if (isPrevalentDueToDebugMode(resourceStatistic))
338                 setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
339             else if (!resourceStatistic.isVeryPrevalentResource) {
340                 markAsPrevalentIfHasRedirectedToPrevalent(resourceStatistic);
341                 auto currentPrevalence = resourceStatistic.isPrevalentResource ? ResourceLoadPrevalence::High : ResourceLoadPrevalence::Low;
342                 auto newPrevalence = m_resourceLoadStatisticsClassifier.calculateResourcePrevalence(resourceStatistic, currentPrevalence);
343                 if (newPrevalence != currentPrevalence)
344                     setPrevalentResource(resourceStatistic, newPrevalence);
345             }
346         }
347     }
348
349     removeDataRecords([this, weakThis = makeWeakPtr(*this)] {
350         ASSERT(!RunLoop::isMain());
351         if (!weakThis)
352             return;
353
354         pruneStatisticsIfNeeded();
355         if (m_persistentStorage)
356             m_persistentStorage->scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::No);
357
358         if (!m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned)
359             return;
360
361         RunLoop::main().dispatch([] {
362             WebProcessProxy::notifyPageStatisticsAndDataRecordsProcessed();
363         });
364     });
365 }
366
367 void ResourceLoadStatisticsMemoryStore::hasStorageAccess(const String& subFramePrimaryDomain, const String& topFramePrimaryDomain, uint64_t frameID, uint64_t pageID, CompletionHandler<void(bool)>&& completionHandler)
368 {
369     ASSERT(!RunLoop::isMain());
370
371     auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
372     if (shouldBlockAndPurgeCookies(subFrameStatistic)) {
373         completionHandler(false);
374         return;
375     }
376
377     if (!shouldBlockAndKeepCookies(subFrameStatistic)) {
378         completionHandler(true);
379         return;
380     }
381
382     RunLoop::main().dispatch([store = makeRef(m_store), subFramePrimaryDomain = subFramePrimaryDomain.isolatedCopy(), topFramePrimaryDomain = topFramePrimaryDomain.isolatedCopy(), frameID, pageID, completionHandler = WTFMove(completionHandler)]() mutable {
383         store->callHasStorageAccessForFrameHandler(subFramePrimaryDomain, topFramePrimaryDomain, frameID, pageID, [store = store.copyRef(), completionHandler = WTFMove(completionHandler)](bool result) mutable {
384             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler), result] () mutable {
385                 completionHandler(result);
386             });
387         });
388     });
389 }
390
391 void ResourceLoadStatisticsMemoryStore::requestStorageAccess(String&& subFramePrimaryDomain, String&& topFramePrimaryDomain, uint64_t frameID, uint64_t pageID, bool promptEnabled, CompletionHandler<void(StorageAccessStatus)>&& completionHandler)
392 {
393     ASSERT(!RunLoop::isMain());
394
395     auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
396     if (shouldBlockAndPurgeCookies(subFrameStatistic)) {
397 #if !RELEASE_LOG_DISABLED
398         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Cannot grant storage access to %{public}s since its cookies are blocked in third-party contexts and it has not received user interaction as first-party.", subFramePrimaryDomain.utf8().data());
399 #endif
400         completionHandler(StorageAccessStatus::CannotRequestAccess);
401         return;
402     }
403
404     if (!shouldBlockAndKeepCookies(subFrameStatistic)) {
405 #if !RELEASE_LOG_DISABLED
406         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "No need to grant storage access to %{public}s since its cookies are not blocked in third-party contexts.", subFramePrimaryDomain.utf8().data());
407 #endif
408         completionHandler(StorageAccessStatus::HasAccess);
409         return;
410     }
411
412     auto userWasPromptedEarlier = promptEnabled && hasUserGrantedStorageAccessThroughPrompt(subFrameStatistic, topFramePrimaryDomain);
413     if (promptEnabled && !userWasPromptedEarlier) {
414 #if !RELEASE_LOG_DISABLED
415         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "About to ask the user whether they want to grant storage access to %{public}s under %{public}s or not.", subFramePrimaryDomain.utf8().data(), topFramePrimaryDomain.utf8().data());
416 #endif
417         completionHandler(StorageAccessStatus::RequiresUserPrompt);
418         return;
419     } else if (userWasPromptedEarlier) {
420 #if !RELEASE_LOG_DISABLED
421         RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Storage access was granted to %{public}s under %{public}s.", subFramePrimaryDomain.utf8().data(), topFramePrimaryDomain.utf8().data());
422 #endif
423     }
424
425     subFrameStatistic.timesAccessedAsFirstPartyDueToStorageAccessAPI++;
426
427     grantStorageAccessInternal(WTFMove(subFramePrimaryDomain), WTFMove(topFramePrimaryDomain), frameID, pageID, userWasPromptedEarlier, [completionHandler = WTFMove(completionHandler)] (bool wasGrantedAccess) mutable {
428         completionHandler(wasGrantedAccess ? StorageAccessStatus::HasAccess : StorageAccessStatus::CannotRequestAccess);
429     });
430 }
431
432 void ResourceLoadStatisticsMemoryStore::requestStorageAccessUnderOpener(String&& primaryDomainInNeedOfStorageAccess, uint64_t openerPageID, String&& openerPrimaryDomain)
433 {
434     ASSERT(primaryDomainInNeedOfStorageAccess != openerPrimaryDomain);
435     ASSERT(!RunLoop::isMain());
436
437     if (primaryDomainInNeedOfStorageAccess == openerPrimaryDomain)
438         return;
439
440     auto& domainInNeedOfStorageAccessStatistic = ensureResourceStatisticsForPrimaryDomain(primaryDomainInNeedOfStorageAccess);
441     auto cookiesBlockedAndPurged = shouldBlockAndPurgeCookies(domainInNeedOfStorageAccessStatistic);
442
443     // The domain already has access if its cookies are not blocked.
444     if (!cookiesBlockedAndPurged && !shouldBlockAndKeepCookies(domainInNeedOfStorageAccessStatistic))
445         return;
446
447 #if !RELEASE_LOG_DISABLED
448     RELEASE_LOG_INFO_IF(m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "[Temporary combatibility fix] Storage access was granted for %{public}s under opener page from %{public}s, with user interaction in the opened window.", primaryDomainInNeedOfStorageAccess.utf8().data(), openerPrimaryDomain.utf8().data());
449 #endif
450     grantStorageAccessInternal(WTFMove(primaryDomainInNeedOfStorageAccess), WTFMove(openerPrimaryDomain), WTF::nullopt, openerPageID, false, [](bool) { });
451 }
452
453 void ResourceLoadStatisticsMemoryStore::grantStorageAccess(String&& subFrameHost, String&& topFrameHost, uint64_t frameID, uint64_t pageID, bool userWasPromptedNow, CompletionHandler<void(bool)>&& completionHandler)
454 {
455     ASSERT(!RunLoop::isMain());
456
457     auto subFramePrimaryDomain = ResourceLoadStatistics::primaryDomain(subFrameHost);
458     auto topFramePrimaryDomain = ResourceLoadStatistics::primaryDomain(topFrameHost);
459     if (userWasPromptedNow) {
460         auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
461         ASSERT(subFrameStatistic.hadUserInteraction);
462         subFrameStatistic.storageAccessUnderTopFrameOrigins.add(topFramePrimaryDomain);
463     }
464     grantStorageAccessInternal(WTFMove(subFramePrimaryDomain), WTFMove(topFramePrimaryDomain), frameID, pageID, userWasPromptedNow, WTFMove(completionHandler));
465 }
466
467 void ResourceLoadStatisticsMemoryStore::grantStorageAccessInternal(String&& subFramePrimaryDomain, String&& topFramePrimaryDomain, Optional<uint64_t> frameID, uint64_t pageID, bool userWasPromptedNowOrEarlier, CompletionHandler<void(bool)>&& callback)
468 {
469     ASSERT(!RunLoop::isMain());
470
471     if (subFramePrimaryDomain == topFramePrimaryDomain) {
472         callback(true);
473         return;
474     }
475
476     // FIXME: Remove m_storageAccessPromptsEnabled check if prompting is no longer experimental.
477     if (userWasPromptedNowOrEarlier && m_storageAccessPromptsEnabled) {
478         auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
479         ASSERT(subFrameStatistic.hadUserInteraction);
480         ASSERT(subFrameStatistic.storageAccessUnderTopFrameOrigins.contains(topFramePrimaryDomain));
481         subFrameStatistic.mostRecentUserInteractionTime = WallTime::now();
482     }
483
484     RunLoop::main().dispatch([subFramePrimaryDomain = subFramePrimaryDomain.isolatedCopy(), topFramePrimaryDomain = topFramePrimaryDomain.isolatedCopy(), frameID, pageID, store = makeRef(m_store), callback = WTFMove(callback)]() mutable {
485         store->callGrantStorageAccessHandler(subFramePrimaryDomain, topFramePrimaryDomain, frameID, pageID, [callback = WTFMove(callback), store = store.copyRef()](bool value) mutable {
486             store->statisticsQueue().dispatch([callback = WTFMove(callback), value] () mutable {
487                 callback(value);
488             });
489         });
490     });
491 }
492
493 void ResourceLoadStatisticsMemoryStore::grandfatherExistingWebsiteData(CompletionHandler<void()>&& callback)
494 {
495     ASSERT(!RunLoop::isMain());
496
497     RunLoop::main().dispatch([weakThis = makeWeakPtr(*this), callback = WTFMove(callback), shouldNotifyPagesWhenDataRecordsWereScanned = m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, workQueue = m_workQueue.copyRef()] () mutable {
498         // FIXME: This method being a static call on WebProcessProxy is wrong.
499         // It should be on the data store that this object belongs to.
500         WebProcessProxy::topPrivatelyControlledDomainsWithWebsiteData(WebResourceLoadStatisticsStore::monitoredDataTypes(), shouldNotifyPagesWhenDataRecordsWereScanned, [weakThis = WTFMove(weakThis), callback = WTFMove(callback), workQueue = workQueue.copyRef()] (HashSet<String>&& topPrivatelyControlledDomainsWithWebsiteData) mutable {
501             workQueue->dispatch([weakThis = WTFMove(weakThis), topDomains = crossThreadCopy(topPrivatelyControlledDomainsWithWebsiteData), callback = WTFMove(callback)] () mutable {
502                 if (!weakThis) {
503                     callback();
504                     return;
505                 }
506
507                 for (auto& topPrivatelyControlledDomain : topDomains) {
508                     auto& statistic = weakThis->ensureResourceStatisticsForPrimaryDomain(topPrivatelyControlledDomain);
509                     statistic.grandfathered = true;
510                 }
511                 weakThis->m_endOfGrandfatheringTimestamp = WallTime::now() + weakThis->m_parameters.grandfatheringTime;
512                 if (weakThis->m_persistentStorage)
513                     weakThis->m_persistentStorage->scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::Yes);
514                 callback();
515                 weakThis->logTestingEvent("Grandfathered"_s);
516             });
517         });
518     });
519 }
520
521 Vector<String> ResourceLoadStatisticsMemoryStore::ensurePrevalentResourcesForDebugMode()
522 {
523     if (!m_debugModeEnabled)
524         return { };
525
526     Vector<String> primaryDomainsToBlock;
527     primaryDomainsToBlock.reserveInitialCapacity(2);
528
529     auto& staticSesourceStatistic = ensureResourceStatisticsForPrimaryDomain(debugStaticPrevalentResource);
530     setPrevalentResource(staticSesourceStatistic, ResourceLoadPrevalence::High);
531     primaryDomainsToBlock.uncheckedAppend(debugStaticPrevalentResource);
532
533     if (!m_debugManualPrevalentResource.isEmpty()) {
534         auto& manualResourceStatistic = ensureResourceStatisticsForPrimaryDomain(m_debugManualPrevalentResource);
535         setPrevalentResource(manualResourceStatistic, ResourceLoadPrevalence::High);
536         primaryDomainsToBlock.uncheckedAppend(m_debugManualPrevalentResource);
537     }
538     
539     return primaryDomainsToBlock;
540 }
541
542 void ResourceLoadStatisticsMemoryStore::setResourceLoadStatisticsDebugMode(bool enable)
543 {
544     ASSERT(!RunLoop::isMain());
545
546     m_debugModeEnabled = enable;
547     m_debugLoggingEnabled = enable;
548
549     ensurePrevalentResourcesForDebugMode();
550 }
551
552 void ResourceLoadStatisticsMemoryStore::setPrevalentResourceForDebugMode(const String& domain)
553 {
554     if (!m_debugModeEnabled)
555         return;
556
557     m_debugManualPrevalentResource = domain;
558     auto& resourceStatistic = ensureResourceStatisticsForPrimaryDomain(domain);
559     setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
560
561 #if !RELEASE_LOG_DISABLED
562     RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "Did set %{public}s as prevalent resource for the purposes of ITP Debug Mode.", domain.utf8().data());
563 #endif
564 }
565
566 void ResourceLoadStatisticsMemoryStore::scheduleStatisticsProcessingRequestIfNecessary()
567 {
568     ASSERT(!RunLoop::isMain());
569
570     m_pendingStatisticsProcessingRequestIdentifier = ++m_lastStatisticsProcessingRequestIdentifier;
571     m_workQueue->dispatchAfter(minimumStatisticsProcessingInterval, [this, weakThis = makeWeakPtr(*this), statisticsProcessingRequestIdentifier = *m_pendingStatisticsProcessingRequestIdentifier] {
572         if (!weakThis)
573             return;
574
575         if (!m_pendingStatisticsProcessingRequestIdentifier || *m_pendingStatisticsProcessingRequestIdentifier != statisticsProcessingRequestIdentifier) {
576             // This request has been canceled.
577             return;
578         }
579
580         updateCookieBlocking([]() { });
581         processStatisticsAndDataRecords();
582     });
583 }
584
585 void ResourceLoadStatisticsMemoryStore::cancelPendingStatisticsProcessingRequest()
586 {
587     ASSERT(!RunLoop::isMain());
588
589     m_pendingStatisticsProcessingRequestIdentifier = WTF::nullopt;
590 }
591
592 void ResourceLoadStatisticsMemoryStore::logFrameNavigation(const String& targetPrimaryDomain, const String& mainFramePrimaryDomain, const String& sourcePrimaryDomain, const String& targetHost, const String& mainFrameHost, bool isRedirect, bool isMainFrame)
593 {
594     ASSERT(!RunLoop::isMain());
595
596     bool areTargetAndMainFrameDomainsAssociated = targetPrimaryDomain == mainFramePrimaryDomain;
597     bool areTargetAndSourceDomainsAssociated = targetPrimaryDomain == sourcePrimaryDomain;
598
599     bool statisticsWereUpdated = false;
600     if (!isMainFrame && targetHost != mainFrameHost && !(areTargetAndMainFrameDomainsAssociated || areTargetAndSourceDomainsAssociated)) {
601         auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
602         targetStatistics.lastSeen = ResourceLoadStatistics::reduceTimeResolution(WallTime::now());
603         if (targetStatistics.subframeUnderTopFrameOrigins.add(mainFramePrimaryDomain).isNewEntry)
604             statisticsWereUpdated = true;
605     }
606
607     if (isRedirect && !areTargetAndSourceDomainsAssociated) {
608         if (isMainFrame) {
609             auto& redirectingOriginStatistics = ensureResourceStatisticsForPrimaryDomain(sourcePrimaryDomain);
610             if (redirectingOriginStatistics.topFrameUniqueRedirectsTo.add(targetPrimaryDomain).isNewEntry)
611                 statisticsWereUpdated = true;
612             auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
613             if (targetStatistics.topFrameUniqueRedirectsFrom.add(sourcePrimaryDomain).isNewEntry)
614                 statisticsWereUpdated = true;
615         } else {
616             auto& redirectingOriginStatistics = ensureResourceStatisticsForPrimaryDomain(sourcePrimaryDomain);
617             if (redirectingOriginStatistics.subresourceUniqueRedirectsTo.add(targetPrimaryDomain).isNewEntry)
618                 statisticsWereUpdated = true;
619             auto& targetStatistics = ensureResourceStatisticsForPrimaryDomain(targetPrimaryDomain);
620             if (targetStatistics.subresourceUniqueRedirectsFrom.add(sourcePrimaryDomain).isNewEntry)
621                 statisticsWereUpdated = true;
622         }
623     }
624
625     if (statisticsWereUpdated)
626         scheduleStatisticsProcessingRequestIfNecessary();
627 }
628
629 void ResourceLoadStatisticsMemoryStore::logUserInteraction(const String& primaryDomain)
630 {
631     ASSERT(!RunLoop::isMain());
632
633     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
634     statistics.hadUserInteraction = true;
635     statistics.mostRecentUserInteractionTime = WallTime::now();
636 }
637
638 void ResourceLoadStatisticsMemoryStore::clearUserInteraction(const String& primaryDomain)
639 {
640     ASSERT(!RunLoop::isMain());
641
642     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
643     statistics.hadUserInteraction = false;
644     statistics.mostRecentUserInteractionTime = { };
645 }
646
647 bool ResourceLoadStatisticsMemoryStore::hasHadUserInteraction(const String& primaryDomain)
648 {
649     ASSERT(!RunLoop::isMain());
650
651     auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
652     return mapEntry == m_resourceStatisticsMap.end() ? false: hasHadUnexpiredRecentUserInteraction(mapEntry->value);
653 }
654
655 void ResourceLoadStatisticsMemoryStore::setPrevalentResource(WebCore::ResourceLoadStatistics& resourceStatistic, ResourceLoadPrevalence newPrevalence)
656 {
657     ASSERT(!RunLoop::isMain());
658
659     resourceStatistic.isPrevalentResource = true;
660     resourceStatistic.isVeryPrevalentResource = newPrevalence == ResourceLoadPrevalence::VeryHigh;
661     HashSet<String> domainsThatHaveRedirectedTo;
662     recursivelyGetAllDomainsThatHaveRedirectedToThisDomain(resourceStatistic, domainsThatHaveRedirectedTo, 0);
663     for (auto& domain : domainsThatHaveRedirectedTo) {
664         auto mapEntry = m_resourceStatisticsMap.find(domain);
665         if (mapEntry == m_resourceStatisticsMap.end())
666             continue;
667         ASSERT(!mapEntry->value.isPrevalentResource);
668         mapEntry->value.isPrevalentResource = true;
669     }
670 }
671     
672 String ResourceLoadStatisticsMemoryStore::dumpResourceLoadStatistics() const
673 {
674     ASSERT(!RunLoop::isMain());
675
676     StringBuilder result;
677     result.appendLiteral("Resource load statistics:\n\n");
678     for (auto& mapEntry : m_resourceStatisticsMap.values())
679         result.append(mapEntry.toString());
680     return result.toString();
681 }
682
683 bool ResourceLoadStatisticsMemoryStore::isPrevalentResource(const String& primaryDomain) const
684 {
685     ASSERT(!RunLoop::isMain());
686
687     auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
688     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource;
689 }
690
691 bool ResourceLoadStatisticsMemoryStore::isVeryPrevalentResource(const String& primaryDomain) const
692 {
693     ASSERT(!RunLoop::isMain());
694
695     auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
696     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource && mapEntry->value.isVeryPrevalentResource;
697 }
698
699 bool ResourceLoadStatisticsMemoryStore::isRegisteredAsSubresourceUnder(const String& subresourcePrimaryDomain, const String& topFramePrimaryDomain) const
700 {
701     ASSERT(!RunLoop::isMain());
702
703     auto mapEntry = m_resourceStatisticsMap.find(subresourcePrimaryDomain);
704     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subresourceUnderTopFrameOrigins.contains(topFramePrimaryDomain);
705 }
706
707 bool ResourceLoadStatisticsMemoryStore::isRegisteredAsSubFrameUnder(const String& subFramePrimaryDomain, const String& topFramePrimaryDomain) const
708 {
709     ASSERT(!RunLoop::isMain());
710
711     auto mapEntry = m_resourceStatisticsMap.find(subFramePrimaryDomain);
712     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subframeUnderTopFrameOrigins.contains(topFramePrimaryDomain);
713 }
714
715 bool ResourceLoadStatisticsMemoryStore::isRegisteredAsRedirectingTo(const String& hostRedirectedFromPrimaryDomain, const String& hostRedirectedToPrimaryDomain) const
716 {
717     ASSERT(!RunLoop::isMain());
718
719     auto mapEntry = m_resourceStatisticsMap.find(hostRedirectedFromPrimaryDomain);
720     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subresourceUniqueRedirectsTo.contains(hostRedirectedToPrimaryDomain);
721 }
722
723 void ResourceLoadStatisticsMemoryStore::clearPrevalentResource(const String& primaryDomain)
724 {
725     ASSERT(!RunLoop::isMain());
726
727     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
728     statistics.isPrevalentResource = false;
729     statistics.isVeryPrevalentResource = false;
730 }
731
732 void ResourceLoadStatisticsMemoryStore::setGrandfathered(const String& primaryDomain, bool value)
733 {
734     ASSERT(!RunLoop::isMain());
735
736     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
737     statistics.grandfathered = value;
738 }
739
740 bool ResourceLoadStatisticsMemoryStore::isGrandfathered(const String& primaryDomain) const
741 {
742     ASSERT(!RunLoop::isMain());
743
744     auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
745     return mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.grandfathered;
746 }
747
748 void ResourceLoadStatisticsMemoryStore::setSubframeUnderTopFrameOrigin(const String& primarySubFrameDomain, const String& primaryTopFrameDomain)
749 {
750     ASSERT(!RunLoop::isMain());
751
752     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubFrameDomain);
753     statistics.subframeUnderTopFrameOrigins.add(primaryTopFrameDomain);
754     // For consistency, make sure we also have a statistics entry for the top frame domain.
755     ensureResourceStatisticsForPrimaryDomain(primaryTopFrameDomain);
756 }
757
758 void ResourceLoadStatisticsMemoryStore::setSubresourceUnderTopFrameOrigin(const String& primarySubresourceDomain, const String& primaryTopFrameDomain)
759 {
760     ASSERT(!RunLoop::isMain());
761
762     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
763     statistics.subresourceUnderTopFrameOrigins.add(primaryTopFrameDomain);
764     // For consistency, make sure we also have a statistics entry for the top frame domain.
765     ensureResourceStatisticsForPrimaryDomain(primaryTopFrameDomain);
766 }
767
768 void ResourceLoadStatisticsMemoryStore::setSubresourceUniqueRedirectTo(const String& primarySubresourceDomain, const String& primaryRedirectDomain)
769 {
770     ASSERT(!RunLoop::isMain());
771
772     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
773     statistics.subresourceUniqueRedirectsTo.add(primaryRedirectDomain);
774     // For consistency, make sure we also have a statistics entry for the redirect domain.
775     ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
776 }
777
778 void ResourceLoadStatisticsMemoryStore::setSubresourceUniqueRedirectFrom(const String& primarySubresourceDomain, const String& primaryRedirectDomain)
779 {
780     ASSERT(!RunLoop::isMain());
781
782     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
783     statistics.subresourceUniqueRedirectsFrom.add(primaryRedirectDomain);
784     // For consistency, make sure we also have a statistics entry for the redirect domain.
785     ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
786 }
787
788 void ResourceLoadStatisticsMemoryStore::setTopFrameUniqueRedirectTo(const String& topFramePrimaryDomain, const String& primaryRedirectDomain)
789 {
790     ASSERT(!RunLoop::isMain());
791
792     auto& statistics = ensureResourceStatisticsForPrimaryDomain(topFramePrimaryDomain);
793     statistics.topFrameUniqueRedirectsTo.add(primaryRedirectDomain);
794     // For consistency, make sure we also have a statistics entry for the redirect domain.
795     ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
796 }
797
798 void ResourceLoadStatisticsMemoryStore::setTopFrameUniqueRedirectFrom(const String& topFramePrimaryDomain, const String& primaryRedirectDomain)
799 {
800     ASSERT(!RunLoop::isMain());
801
802     auto& statistics = ensureResourceStatisticsForPrimaryDomain(topFramePrimaryDomain);
803     statistics.topFrameUniqueRedirectsFrom.add(primaryRedirectDomain);
804     // For consistency, make sure we also have a statistics entry for the redirect domain.
805     ensureResourceStatisticsForPrimaryDomain(primaryRedirectDomain);
806 }
807
808 void ResourceLoadStatisticsMemoryStore::setTimeToLiveUserInteraction(Seconds seconds)
809 {
810     ASSERT(!RunLoop::isMain());
811     ASSERT(seconds >= 0_s);
812
813     m_parameters.timeToLiveUserInteraction = seconds;
814 }
815
816 void ResourceLoadStatisticsMemoryStore::setMinimumTimeBetweenDataRecordsRemoval(Seconds seconds)
817 {
818     ASSERT(!RunLoop::isMain());
819     ASSERT(seconds >= 0_s);
820
821     m_parameters.minimumTimeBetweenDataRecordsRemoval = seconds;
822 }
823
824 void ResourceLoadStatisticsMemoryStore::setGrandfatheringTime(Seconds seconds)
825 {
826     ASSERT(!RunLoop::isMain());
827     ASSERT(seconds >= 0_s);
828
829     m_parameters.grandfatheringTime = seconds;
830 }
831
832 void ResourceLoadStatisticsMemoryStore::setCacheMaxAgeCap(Seconds seconds)
833 {
834     ASSERT(!RunLoop::isMain());
835     ASSERT(seconds >= 0_s);
836
837     m_parameters.cacheMaxAgeCapTime = seconds;
838     updateCacheMaxAgeCap();
839 }
840
841 void ResourceLoadStatisticsMemoryStore::updateCacheMaxAgeCap()
842 {
843     ASSERT(!RunLoop::isMain());
844     
845     RunLoop::main().dispatch([store = makeRef(m_store), seconds = m_parameters.cacheMaxAgeCapTime] () {
846         store->setCacheMaxAgeCap(seconds, [] { });
847     });
848 }
849
850 void ResourceLoadStatisticsMemoryStore::setAgeCapForClientSideCookies(Seconds seconds)
851 {
852     ASSERT(!RunLoop::isMain());
853     ASSERT(seconds >= 0_s);
854     
855     m_parameters.clientSideCookiesAgeCapTime = seconds;
856     updateClientSideCookiesAgeCap();
857 }
858
859 void ResourceLoadStatisticsMemoryStore::updateClientSideCookiesAgeCap()
860 {
861     ASSERT(!RunLoop::isMain());
862
863 #if ENABLE(RESOURCE_LOAD_STATISTICS)
864     RunLoop::main().dispatch([store = makeRef(m_store), seconds = m_parameters.clientSideCookiesAgeCapTime] () {
865         if (auto* websiteDataStore = store->websiteDataStore())
866             websiteDataStore->setAgeCapForClientSideCookies(seconds, [] { });
867         if (auto* networkSession = store->networkSession())
868             networkSession->networkStorageSession().setAgeCapForClientSideCookies(seconds);
869     });
870 #endif
871 }
872
873 bool ResourceLoadStatisticsMemoryStore::shouldRemoveDataRecords() const
874 {
875     ASSERT(!RunLoop::isMain());
876
877     if (m_dataRecordsBeingRemoved)
878         return false;
879
880 #if ENABLE(NETSCAPE_PLUGIN_API)
881     for (const auto& plugin : PluginProcessManager::singleton().pluginProcesses()) {
882         if (!m_activePluginTokens.contains(plugin->pluginProcessToken()))
883             return true;
884     }
885 #endif
886
887     return !m_lastTimeDataRecordsWereRemoved || MonotonicTime::now() >= (m_lastTimeDataRecordsWereRemoved + m_parameters.minimumTimeBetweenDataRecordsRemoval);
888 }
889
890 void ResourceLoadStatisticsMemoryStore::setDataRecordsBeingRemoved(bool value)
891 {
892     ASSERT(!RunLoop::isMain());
893
894     m_dataRecordsBeingRemoved = value;
895     if (m_dataRecordsBeingRemoved)
896         m_lastTimeDataRecordsWereRemoved = MonotonicTime::now();
897 }
898
899 ResourceLoadStatistics& ResourceLoadStatisticsMemoryStore::ensureResourceStatisticsForPrimaryDomain(const String& primaryDomain)
900 {
901     ASSERT(!RunLoop::isMain());
902
903     return m_resourceStatisticsMap.ensure(primaryDomain, [&primaryDomain] {
904         return ResourceLoadStatistics(primaryDomain);
905     }).iterator->value;
906 }
907
908 std::unique_ptr<KeyedEncoder> ResourceLoadStatisticsMemoryStore::createEncoderFromData() const
909 {
910     ASSERT(!RunLoop::isMain());
911
912     auto encoder = KeyedEncoder::encoder();
913     encoder->encodeUInt32("version", statisticsModelVersion);
914     encoder->encodeDouble("endOfGrandfatheringTimestamp", m_endOfGrandfatheringTimestamp.secondsSinceEpoch().value());
915
916     encoder->encodeObjects("browsingStatistics", m_resourceStatisticsMap.begin(), m_resourceStatisticsMap.end(), [](KeyedEncoder& encoderInner, const auto& origin) {
917         origin.value.encode(encoderInner);
918     });
919
920     encoder->encodeObjects("operatingDates", m_operatingDates.begin(), m_operatingDates.end(), [](KeyedEncoder& encoderInner, OperatingDate date) {
921         encoderInner.encodeDouble("date", date.secondsSinceEpoch().value());
922     });
923
924     return encoder;
925 }
926
927 void ResourceLoadStatisticsMemoryStore::mergeWithDataFromDecoder(KeyedDecoder& decoder)
928 {
929     ASSERT(!RunLoop::isMain());
930
931     unsigned versionOnDisk;
932     if (!decoder.decodeUInt32("version", versionOnDisk))
933         return;
934
935     if (versionOnDisk > statisticsModelVersion) {
936         WTFLogAlways("Found resource load statistics on disk with model version %u whereas the highest supported version is %u. Resetting.", versionOnDisk, statisticsModelVersion);
937         return;
938     }
939
940     double endOfGrandfatheringTimestamp;
941     if (decoder.decodeDouble("endOfGrandfatheringTimestamp", endOfGrandfatheringTimestamp))
942         m_endOfGrandfatheringTimestamp = WallTime::fromRawSeconds(endOfGrandfatheringTimestamp);
943     else
944         m_endOfGrandfatheringTimestamp = { };
945
946     Vector<ResourceLoadStatistics> loadedStatistics;
947     bool succeeded = decoder.decodeObjects("browsingStatistics", loadedStatistics, [versionOnDisk](KeyedDecoder& decoderInner, ResourceLoadStatistics& statistics) {
948         return statistics.decode(decoderInner, versionOnDisk);
949     });
950
951     if (!succeeded)
952         return;
953
954     mergeStatistics(WTFMove(loadedStatistics));
955     updateCookieBlocking([]() { });
956
957     Vector<OperatingDate> operatingDates;
958     succeeded = decoder.decodeObjects("operatingDates", operatingDates, [](KeyedDecoder& decoder, OperatingDate& date) {
959         double value;
960         if (!decoder.decodeDouble("date", value))
961             return false;
962
963         date = OperatingDate::fromWallTime(WallTime::fromRawSeconds(value));
964         return true;
965     });
966
967     if (!succeeded)
968         return;
969
970     m_operatingDates = mergeOperatingDates(m_operatingDates, WTFMove(operatingDates));
971 }
972
973 void ResourceLoadStatisticsMemoryStore::clear(CompletionHandler<void()>&& completionHandler)
974 {
975     ASSERT(!RunLoop::isMain());
976
977     m_resourceStatisticsMap.clear();
978     m_operatingDates.clear();
979
980     auto callbackAggregator = CallbackAggregator::create(WTFMove(completionHandler));
981
982     removeAllStorageAccess([callbackAggregator = callbackAggregator.copyRef()] { });
983
984     auto primaryDomainsToBlock = ensurePrevalentResourcesForDebugMode();
985     updateCookieBlockingForDomains(primaryDomainsToBlock, [callbackAggregator = callbackAggregator.copyRef()] { });
986 }
987
988 bool ResourceLoadStatisticsMemoryStore::wasAccessedAsFirstPartyDueToUserInteraction(const ResourceLoadStatistics& current, const ResourceLoadStatistics& updated) const
989 {
990     if (!current.hadUserInteraction && !updated.hadUserInteraction)
991         return false;
992
993     auto mostRecentUserInteractionTime = std::max(current.mostRecentUserInteractionTime, updated.mostRecentUserInteractionTime);
994
995     return updated.lastSeen <= mostRecentUserInteractionTime + 24_h;
996 }
997
998 void ResourceLoadStatisticsMemoryStore::mergeStatistics(Vector<ResourceLoadStatistics>&& statistics)
999 {
1000     ASSERT(!RunLoop::isMain());
1001
1002     for (auto& statistic : statistics) {
1003         auto result = m_resourceStatisticsMap.ensure(statistic.highLevelDomain, [&statistic] {
1004             return WTFMove(statistic);
1005         });
1006         if (!result.isNewEntry) {
1007             if (wasAccessedAsFirstPartyDueToUserInteraction(result.iterator->value, statistic))
1008                 result.iterator->value.timesAccessedAsFirstPartyDueToUserInteraction++;
1009             result.iterator->value.merge(statistic);
1010         }
1011     }
1012 }
1013
1014 bool ResourceLoadStatisticsMemoryStore::shouldBlockAndKeepCookies(const ResourceLoadStatistics& statistic)
1015 {
1016     return statistic.isPrevalentResource && statistic.hadUserInteraction;
1017 }
1018
1019 bool ResourceLoadStatisticsMemoryStore::shouldBlockAndPurgeCookies(const ResourceLoadStatistics& statistic)
1020 {
1021     return statistic.isPrevalentResource && !statistic.hadUserInteraction;
1022 }
1023
1024 bool ResourceLoadStatisticsMemoryStore::hasUserGrantedStorageAccessThroughPrompt(const ResourceLoadStatistics& statistic, const String& firstPartyPrimaryDomain)
1025 {
1026     return statistic.storageAccessUnderTopFrameOrigins.contains(firstPartyPrimaryDomain);
1027 }
1028
1029 static void debugLogDomainsInBatches(const char* action, const Vector<String>& domains)
1030 {
1031 #if !RELEASE_LOG_DISABLED
1032     static const auto maxNumberOfDomainsInOneLogStatement = 50;
1033     if (domains.isEmpty())
1034         return;
1035
1036     if (domains.size() <= maxNumberOfDomainsInOneLogStatement) {
1037         RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "About to %{public}s cookies in third-party contexts for: %{public}s.", action, domainsToString(domains).utf8().data());
1038         return;
1039     }
1040     
1041     Vector<String> batch;
1042     batch.reserveInitialCapacity(maxNumberOfDomainsInOneLogStatement);
1043     auto batchNumber = 1;
1044     unsigned numberOfBatches = std::ceil(domains.size() / static_cast<float>(maxNumberOfDomainsInOneLogStatement));
1045
1046     for (auto& domain : domains) {
1047         if (batch.size() == maxNumberOfDomainsInOneLogStatement) {
1048             RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "About to %{public}s cookies in third-party contexts for (%{public}d of %u): %{public}s.", action, batchNumber, numberOfBatches, domainsToString(batch).utf8().data());
1049             batch.shrink(0);
1050             ++batchNumber;
1051         }
1052         batch.append(domain);
1053     }
1054     if (!batch.isEmpty())
1055         RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "About to %{public}s cookies in third-party contexts for (%{public}d of %u): %{public}s.", action, batchNumber, numberOfBatches, domainsToString(batch).utf8().data());
1056 #else
1057     UNUSED_PARAM(action);
1058     UNUSED_PARAM(domains);
1059 #endif
1060 }
1061
1062 void ResourceLoadStatisticsMemoryStore::updateCookieBlocking(CompletionHandler<void()>&& completionHandler)
1063 {
1064     ASSERT(!RunLoop::isMain());
1065
1066     Vector<String> domainsToBlock;
1067     for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
1068         if (resourceStatistic.isPrevalentResource)
1069             domainsToBlock.append(resourceStatistic.highLevelDomain);
1070     }
1071
1072     if (domainsToBlock.isEmpty()) {
1073         completionHandler();
1074         return;
1075     }
1076
1077     if (m_debugLoggingEnabled && !domainsToBlock.isEmpty())
1078         debugLogDomainsInBatches("block", domainsToBlock);
1079
1080     RunLoop::main().dispatch([weakThis = makeWeakPtr(*this), store = makeRef(m_store), domainsToBlock = crossThreadCopy(domainsToBlock), completionHandler = WTFMove(completionHandler)] () mutable {
1081         store->callUpdatePrevalentDomainsToBlockCookiesForHandler(domainsToBlock, [weakThis = WTFMove(weakThis), store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1082             store->statisticsQueue().dispatch([weakThis = WTFMove(weakThis), completionHandler = WTFMove(completionHandler)]() mutable {
1083                 completionHandler();
1084                 if (!weakThis)
1085                     return;
1086 #if !RELEASE_LOG_DISABLED
1087                 RELEASE_LOG_INFO_IF(weakThis->m_debugLoggingEnabled, ResourceLoadStatisticsDebug, "Done updating cookie blocking.");
1088 #endif
1089             });
1090         });
1091     });
1092 }
1093
1094 void ResourceLoadStatisticsMemoryStore::updateCookieBlockingForDomains(const Vector<String>& domainsToBlock, CompletionHandler<void()>&& completionHandler)
1095 {
1096     ASSERT(!RunLoop::isMain());
1097
1098     RunLoop::main().dispatch([store = makeRef(m_store), domainsToBlock = crossThreadCopy(domainsToBlock), completionHandler = WTFMove(completionHandler)] () mutable {
1099         store->callUpdatePrevalentDomainsToBlockCookiesForHandler(domainsToBlock, [store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1100             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler)]() mutable {
1101                 completionHandler();
1102             });
1103         });
1104     });
1105 }
1106
1107 void ResourceLoadStatisticsMemoryStore::clearBlockingStateForDomains(const Vector<String>& domains, CompletionHandler<void()>&& completionHandler)
1108 {
1109     ASSERT(!RunLoop::isMain());
1110
1111     if (domains.isEmpty()) {
1112         completionHandler();
1113         return;
1114     }
1115
1116     RunLoop::main().dispatch([store = makeRef(m_store), domains = crossThreadCopy(domains)] {
1117         store->callRemoveDomainsHandler(domains);
1118     });
1119
1120     completionHandler();
1121 }
1122
1123 void ResourceLoadStatisticsMemoryStore::processStatistics(const Function<void(const ResourceLoadStatistics&)>& processFunction) const
1124 {
1125     ASSERT(!RunLoop::isMain());
1126
1127     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
1128         processFunction(resourceStatistic);
1129 }
1130
1131 bool ResourceLoadStatisticsMemoryStore::hasHadUnexpiredRecentUserInteraction(ResourceLoadStatistics& resourceStatistic) const
1132 {
1133     ASSERT(!RunLoop::isMain());
1134
1135     if (resourceStatistic.hadUserInteraction && hasStatisticsExpired(resourceStatistic)) {
1136         // Drop privacy sensitive data because we no longer need it.
1137         // Set timestamp to 0 so that statistics merge will know
1138         // it has been reset as opposed to its default -1.
1139         resourceStatistic.mostRecentUserInteractionTime = { };
1140         resourceStatistic.storageAccessUnderTopFrameOrigins.clear();
1141         resourceStatistic.hadUserInteraction = false;
1142     }
1143
1144     return resourceStatistic.hadUserInteraction;
1145 }
1146
1147 Vector<String> ResourceLoadStatisticsMemoryStore::topPrivatelyControlledDomainsToRemoveWebsiteDataFor()
1148 {
1149     ASSERT(!RunLoop::isMain());
1150
1151     bool shouldCheckForGrandfathering = m_endOfGrandfatheringTimestamp > WallTime::now();
1152     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && m_endOfGrandfatheringTimestamp;
1153
1154     if (shouldClearGrandfathering)
1155         m_endOfGrandfatheringTimestamp = { };
1156
1157     Vector<String> prevalentResources;
1158     for (auto& statistic : m_resourceStatisticsMap.values()) {
1159         if (statistic.isPrevalentResource && !hasHadUnexpiredRecentUserInteraction(statistic) && (!shouldCheckForGrandfathering || !statistic.grandfathered))
1160             prevalentResources.append(statistic.highLevelDomain);
1161
1162         if (shouldClearGrandfathering && statistic.grandfathered)
1163             statistic.grandfathered = false;
1164     }
1165
1166     return prevalentResources;
1167 }
1168
1169 void ResourceLoadStatisticsMemoryStore::includeTodayAsOperatingDateIfNecessary()
1170 {
1171     ASSERT(!RunLoop::isMain());
1172
1173     auto today = OperatingDate::today();
1174     if (!m_operatingDates.isEmpty() && today <= m_operatingDates.last())
1175         return;
1176
1177     while (m_operatingDates.size() >= operatingDatesWindow)
1178         m_operatingDates.remove(0);
1179
1180     m_operatingDates.append(today);
1181 }
1182
1183 bool ResourceLoadStatisticsMemoryStore::hasStatisticsExpired(const ResourceLoadStatistics& resourceStatistic) const
1184 {
1185     ASSERT(!RunLoop::isMain());
1186
1187     if (m_operatingDates.size() >= operatingDatesWindow) {
1188         if (OperatingDate::fromWallTime(resourceStatistic.mostRecentUserInteractionTime) < m_operatingDates.first())
1189             return true;
1190     }
1191
1192     // If we don't meet the real criteria for an expired statistic, check the user setting for a tighter restriction (mainly for testing).
1193     if (m_parameters.timeToLiveUserInteraction) {
1194         if (WallTime::now() > resourceStatistic.mostRecentUserInteractionTime + m_parameters.timeToLiveUserInteraction.value())
1195             return true;
1196     }
1197
1198     return false;
1199 }
1200
1201 void ResourceLoadStatisticsMemoryStore::setMaxStatisticsEntries(size_t maximumEntryCount)
1202 {
1203     ASSERT(!RunLoop::isMain());
1204
1205     m_parameters.maxStatisticsEntries = maximumEntryCount;
1206 }
1207
1208 void ResourceLoadStatisticsMemoryStore::setPruneEntriesDownTo(size_t pruneTargetCount)
1209 {
1210     ASSERT(!RunLoop::isMain());
1211
1212     m_parameters.pruneEntriesDownTo = pruneTargetCount;
1213 }
1214
1215 void ResourceLoadStatisticsMemoryStore::pruneStatisticsIfNeeded()
1216 {
1217     ASSERT(!RunLoop::isMain());
1218
1219     if (m_resourceStatisticsMap.size() <= m_parameters.maxStatisticsEntries)
1220         return;
1221
1222     ASSERT(m_parameters.pruneEntriesDownTo <= m_parameters.maxStatisticsEntries);
1223
1224     size_t numberOfEntriesLeftToPrune = m_resourceStatisticsMap.size() - m_parameters.pruneEntriesDownTo;
1225     ASSERT(numberOfEntriesLeftToPrune);
1226
1227     Vector<StatisticsLastSeen> resourcesToPrunePerImportance[maxImportance + 1];
1228     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
1229         resourcesToPrunePerImportance[computeImportance(resourceStatistic)].append({ resourceStatistic.highLevelDomain, resourceStatistic.lastSeen });
1230
1231     for (unsigned importance = 0; numberOfEntriesLeftToPrune && importance <= maxImportance; ++importance)
1232         pruneResources(m_resourceStatisticsMap, resourcesToPrunePerImportance[importance], numberOfEntriesLeftToPrune);
1233
1234     ASSERT(!numberOfEntriesLeftToPrune);
1235 }
1236
1237 void ResourceLoadStatisticsMemoryStore::resetParametersToDefaultValues()
1238 {
1239     ASSERT(!RunLoop::isMain());
1240
1241     m_parameters = { };
1242 }
1243
1244 void ResourceLoadStatisticsMemoryStore::logTestingEvent(const String& event)
1245 {
1246     ASSERT(!RunLoop::isMain());
1247
1248     RunLoop::main().dispatch([store = makeRef(m_store), event = event.isolatedCopy()] {
1249         store->logTestingEvent(event);
1250     });
1251 }
1252
1253 void ResourceLoadStatisticsMemoryStore::setLastSeen(const String& primaryDomain, Seconds seconds)
1254 {
1255     ASSERT(!RunLoop::isMain());
1256
1257     auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
1258     statistics.lastSeen = WallTime::fromRawSeconds(seconds.seconds());
1259 }
1260
1261 void ResourceLoadStatisticsMemoryStore::setPrevalentResource(const String& primaryDomain)
1262 {
1263     ASSERT(!RunLoop::isMain());
1264
1265     auto& resourceStatistic = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
1266     setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::High);
1267 }
1268
1269 void ResourceLoadStatisticsMemoryStore::setVeryPrevalentResource(const String& primaryDomain)
1270 {
1271     ASSERT(!RunLoop::isMain());
1272
1273     auto& resourceStatistic = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
1274     setPrevalentResource(resourceStatistic, ResourceLoadPrevalence::VeryHigh);
1275 }
1276
1277 void ResourceLoadStatisticsMemoryStore::removeAllStorageAccess(CompletionHandler<void()>&& completionHandler)
1278 {
1279     ASSERT(!RunLoop::isMain());
1280     RunLoop::main().dispatch([store = makeRef(m_store), completionHandler = WTFMove(completionHandler)]() mutable {
1281         store->removeAllStorageAccess([store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1282             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler)]() mutable {
1283                 completionHandler();
1284             });
1285         });
1286     });
1287 }
1288
1289 void ResourceLoadStatisticsMemoryStore::didCreateNetworkProcess()
1290 {
1291     ASSERT(!RunLoop::isMain());
1292
1293     updateCookieBlocking([]() { });
1294     updateCacheMaxAgeCap();
1295     updateClientSideCookiesAgeCap();
1296 }
1297
1298 } // namespace WebKit