Deny third-party cookie creation for prevalent resources without interaction
[WebKit-https.git] / Source / WebKit / UIProcess / WebResourceLoadStatisticsStore.cpp
1 /*
2  * Copyright (C) 2016-2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WebResourceLoadStatisticsStore.h"
28
29 #include "Logging.h"
30 #include "WebProcessMessages.h"
31 #include "WebProcessProxy.h"
32 #include "WebResourceLoadStatisticsStoreMessages.h"
33 #include "WebResourceLoadStatisticsTelemetry.h"
34 #include "WebsiteDataFetchOption.h"
35 #include "WebsiteDataStore.h"
36 #include <WebCore/KeyedCoding.h>
37 #include <WebCore/ResourceLoadStatistics.h>
38 #include <wtf/CrossThreadCopier.h>
39 #include <wtf/DateMath.h>
40 #include <wtf/MathExtras.h>
41 #include <wtf/NeverDestroyed.h>
42
43 using namespace WebCore;
44
45 namespace WebKit {
46
47 constexpr unsigned operatingDatesWindow { 30 };
48 constexpr unsigned statisticsModelVersion { 9 };
49 constexpr unsigned maxImportance { 3 };
50
51 template<typename T> static inline String isolatedPrimaryDomain(const T& value)
52 {
53     return ResourceLoadStatistics::primaryDomain(value).isolatedCopy();
54 }
55
56 const OptionSet<WebsiteDataType>& WebResourceLoadStatisticsStore::monitoredDataTypes()
57 {
58     static NeverDestroyed<OptionSet<WebsiteDataType>> dataTypes(std::initializer_list<WebsiteDataType>({
59         WebsiteDataType::Cookies,
60         WebsiteDataType::IndexedDBDatabases,
61         WebsiteDataType::LocalStorage,
62         WebsiteDataType::MediaKeys,
63         WebsiteDataType::OfflineWebApplicationCache,
64 #if ENABLE(NETSCAPE_PLUGIN_API)
65         WebsiteDataType::PlugInData,
66 #endif
67         WebsiteDataType::SearchFieldRecentSearches,
68         WebsiteDataType::SessionStorage,
69         WebsiteDataType::WebSQLDatabases,
70     }));
71
72     ASSERT(RunLoop::isMain());
73
74     return dataTypes;
75 }
76
77 class OperatingDate {
78 public:
79     OperatingDate() = default;
80
81     static OperatingDate fromWallTime(WallTime time)
82     {
83         double ms = time.secondsSinceEpoch().milliseconds();
84         int year = msToYear(ms);
85         int yearDay = dayInYear(ms, year);
86         int month = monthFromDayInYear(yearDay, isLeapYear(year));
87         int monthDay = dayInMonthFromDayInYear(yearDay, isLeapYear(year));
88
89         return OperatingDate { year, month, monthDay };
90     }
91
92     static OperatingDate today()
93     {
94         return OperatingDate::fromWallTime(WallTime::now());
95     }
96
97     Seconds secondsSinceEpoch() const
98     {
99         return Seconds { dateToDaysFrom1970(m_year, m_month, m_monthDay) * secondsPerDay };
100     }
101
102     bool operator==(const OperatingDate& other) const
103     {
104         return m_monthDay == other.m_monthDay && m_month == other.m_month && m_year == other.m_year;
105     }
106
107     bool operator<(const OperatingDate& other) const
108     {
109         return secondsSinceEpoch() < other.secondsSinceEpoch();
110     }
111
112     bool operator<=(const OperatingDate& other) const
113     {
114         return secondsSinceEpoch() <= other.secondsSinceEpoch();
115     }
116
117 private:
118     OperatingDate(int year, int month, int monthDay)
119         : m_year(year)
120         , m_month(month)
121         , m_monthDay(monthDay)
122     { }
123
124     int m_year { 0 };
125     int m_month { 0 }; // [0, 11].
126     int m_monthDay { 0 }; // [1, 31].
127 };
128
129 static Vector<OperatingDate> mergeOperatingDates(const Vector<OperatingDate>& existingDates, Vector<OperatingDate>&& newDates)
130 {
131     if (existingDates.isEmpty())
132         return WTFMove(newDates);
133
134     Vector<OperatingDate> mergedDates(existingDates.size() + newDates.size());
135
136     // Merge the two sorted vectors of dates.
137     std::merge(existingDates.begin(), existingDates.end(), newDates.begin(), newDates.end(), mergedDates.begin());
138     // Remove duplicate dates.
139     removeRepeatedElements(mergedDates);
140
141     // Drop old dates until the Vector size reaches operatingDatesWindow.
142     while (mergedDates.size() > operatingDatesWindow)
143         mergedDates.remove(0);
144
145     return mergedDates;
146 }
147
148 WebResourceLoadStatisticsStore::WebResourceLoadStatisticsStore(const String& resourceLoadStatisticsDirectory, Function<void (const String&)>&& testingCallback, UpdatePrevalentDomainsWithAndWithoutInteractionHandler&& updatePrevalentDomainsWithAndWithoutInteractionHandler, RemovePrevalentDomainsHandler&& removeDomainsHandler)
149     : m_statisticsQueue(WorkQueue::create("WebResourceLoadStatisticsStore Process Data Queue", WorkQueue::Type::Serial, WorkQueue::QOS::Utility))
150     , m_persistentStorage(*this, resourceLoadStatisticsDirectory)
151     , m_updatePrevalentDomainsWithAndWithoutInteractionHandler(WTFMove(updatePrevalentDomainsWithAndWithoutInteractionHandler))
152     , m_removeDomainsHandler(WTFMove(removeDomainsHandler))
153     , m_dailyTasksTimer(RunLoop::main(), this, &WebResourceLoadStatisticsStore::performDailyTasks)
154     , m_statisticsTestingCallback(WTFMove(testingCallback))
155 {
156     ASSERT(RunLoop::isMain());
157
158 #if PLATFORM(COCOA)
159     registerUserDefaultsIfNeeded();
160 #endif
161
162     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
163         m_persistentStorage.initialize();
164         includeTodayAsOperatingDateIfNecessary();
165     });
166
167     m_statisticsQueue->dispatchAfter(5_s, [this, protectedThis = makeRef(*this)] {
168         if (m_parameters.shouldSubmitTelemetry)
169             WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
170     });
171
172     m_dailyTasksTimer.startRepeating(24_h);
173 }
174
175 WebResourceLoadStatisticsStore::~WebResourceLoadStatisticsStore()
176 {
177 }
178     
179 void WebResourceLoadStatisticsStore::removeDataRecords()
180 {
181     ASSERT(!RunLoop::isMain());
182     
183     if (!shouldRemoveDataRecords())
184         return;
185
186     auto prevalentResourceDomains = topPrivatelyControlledDomainsToRemoveWebsiteDataFor();
187     if (prevalentResourceDomains.isEmpty())
188         return;
189     
190     setDataRecordsBeingRemoved(true);
191
192     RunLoop::main().dispatch([prevalentResourceDomains = crossThreadCopy(prevalentResourceDomains), this, protectedThis = makeRef(*this)] () mutable {
193         WebProcessProxy::deleteWebsiteDataForTopPrivatelyControlledDomainsInAllPersistentDataStores(WebResourceLoadStatisticsStore::monitoredDataTypes(), WTFMove(prevalentResourceDomains), m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, [this, protectedThis = WTFMove(protectedThis)](const HashSet<String>& domainsWithDeletedWebsiteData) mutable {
194             m_statisticsQueue->dispatch([this, protectedThis = WTFMove(protectedThis), topDomains = crossThreadCopy(domainsWithDeletedWebsiteData)] () mutable {
195                 for (auto& prevalentResourceDomain : topDomains) {
196                     auto& statistic = ensureResourceStatisticsForPrimaryDomain(prevalentResourceDomain);
197                     ++statistic.dataRecordsRemoved;
198                 }
199                 setDataRecordsBeingRemoved(false);
200             });
201         });
202     });
203 }
204
205 void WebResourceLoadStatisticsStore::scheduleStatisticsAndDataRecordsProcessing()
206 {
207     ASSERT(RunLoop::isMain());
208     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
209         processStatisticsAndDataRecords();
210     });
211 }
212
213 void WebResourceLoadStatisticsStore::processStatisticsAndDataRecords()
214 {
215     ASSERT(!RunLoop::isMain());
216
217     if (m_parameters.shouldClassifyResourcesBeforeDataRecordsRemoval) {
218         for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
219             if (!resourceStatistic.isPrevalentResource && m_resourceLoadStatisticsClassifier.hasPrevalentResourceCharacteristics(resourceStatistic))
220                 resourceStatistic.isPrevalentResource = true;
221         }
222     }
223     removeDataRecords();
224
225     pruneStatisticsIfNeeded();
226
227     if (m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned) {
228         RunLoop::main().dispatch([] {
229             WebProcessProxy::notifyPageStatisticsAndDataRecordsProcessed();
230         });
231     }
232
233     m_persistentStorage.scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::No);
234 }
235
236 void WebResourceLoadStatisticsStore::resourceLoadStatisticsUpdated(Vector<WebCore::ResourceLoadStatistics>&& origins)
237 {
238     ASSERT(!RunLoop::isMain());
239
240     mergeStatistics(WTFMove(origins));
241     // Fire before processing statistics to propagate user interaction as fast as possible to the network process.
242     updateCookiePartitioning();
243     processStatisticsAndDataRecords();
244 }
245
246 void WebResourceLoadStatisticsStore::grandfatherExistingWebsiteData()
247 {
248     ASSERT(!RunLoop::isMain());
249
250     RunLoop::main().dispatch([this, protectedThis = makeRef(*this)] () mutable {
251         // FIXME: This method being a static call on WebProcessProxy is wrong.
252         // It should be on the data store that this object belongs to.
253         WebProcessProxy::topPrivatelyControlledDomainsWithWebsiteData(WebResourceLoadStatisticsStore::monitoredDataTypes(), m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, [this, protectedThis = WTFMove(protectedThis)] (HashSet<String>&& topPrivatelyControlledDomainsWithWebsiteData) mutable {
254             m_statisticsQueue->dispatch([this, protectedThis = WTFMove(protectedThis), topDomains = crossThreadCopy(topPrivatelyControlledDomainsWithWebsiteData)] () mutable {
255                 for (auto& topPrivatelyControlledDomain : topDomains) {
256                     auto& statistic = ensureResourceStatisticsForPrimaryDomain(topPrivatelyControlledDomain);
257                     statistic.grandfathered = true;
258                 }
259                 m_endOfGrandfatheringTimestamp = WallTime::now() + m_parameters.grandfatheringTime;
260                 m_persistentStorage.scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::Yes);
261             });
262
263             logTestingEvent(ASCIILiteral("Grandfathered"));
264         });
265     });
266 }
267     
268 void WebResourceLoadStatisticsStore::processWillOpenConnection(WebProcessProxy&, IPC::Connection& connection)
269 {
270     connection.addWorkQueueMessageReceiver(Messages::WebResourceLoadStatisticsStore::messageReceiverName(), m_statisticsQueue.get(), this);
271 }
272
273 void WebResourceLoadStatisticsStore::processDidCloseConnection(WebProcessProxy&, IPC::Connection& connection)
274 {
275     connection.removeWorkQueueMessageReceiver(Messages::WebResourceLoadStatisticsStore::messageReceiverName());
276 }
277
278 void WebResourceLoadStatisticsStore::applicationWillTerminate()
279 {
280     m_persistentStorage.finishAllPendingWorkSynchronously();
281 }
282
283 void WebResourceLoadStatisticsStore::performDailyTasks()
284 {
285     ASSERT(RunLoop::isMain());
286
287     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
288         includeTodayAsOperatingDateIfNecessary();
289     });
290     if (m_parameters.shouldSubmitTelemetry)
291         submitTelemetry();
292 }
293
294 void WebResourceLoadStatisticsStore::submitTelemetry()
295 {
296     ASSERT(RunLoop::isMain());
297     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
298         WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
299     });
300 }
301
302 void WebResourceLoadStatisticsStore::logUserInteraction(const URL& url)
303 {
304     if (url.isBlankURL() || url.isEmpty())
305         return;
306
307     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
308         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
309         statistics.hadUserInteraction = true;
310         statistics.mostRecentUserInteractionTime = WallTime::now();
311
312         updateCookiePartitioningForDomains({ primaryDomain }, { }, ShouldClearFirst::No);
313     });
314 }
315
316 void WebResourceLoadStatisticsStore::clearUserInteraction(const URL& url)
317 {
318     if (url.isBlankURL() || url.isEmpty())
319         return;
320
321     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
322         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
323         statistics.hadUserInteraction = false;
324         statistics.mostRecentUserInteractionTime = { };
325     });
326 }
327
328 void WebResourceLoadStatisticsStore::hasHadUserInteraction(const URL& url, WTF::Function<void (bool)>&& completionHandler)
329 {
330     if (url.isBlankURL() || url.isEmpty()) {
331         completionHandler(false);
332         return;
333     }
334
335     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), completionHandler = WTFMove(completionHandler)] () mutable {
336         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
337         bool hadUserInteraction = mapEntry == m_resourceStatisticsMap.end() ? false: hasHadUnexpiredRecentUserInteraction(mapEntry->value);
338         RunLoop::main().dispatch([hadUserInteraction, completionHandler = WTFMove(completionHandler)] {
339             completionHandler(hadUserInteraction);
340         });
341     });
342 }
343
344 void WebResourceLoadStatisticsStore::setLastSeen(const URL& url, Seconds seconds)
345 {
346     if (url.isBlankURL() || url.isEmpty())
347         return;
348     
349     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), seconds] {
350         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
351         statistics.lastSeen = WallTime::fromRawSeconds(seconds.seconds());
352     });
353 }
354     
355 void WebResourceLoadStatisticsStore::setPrevalentResource(const URL& url)
356 {
357     if (url.isBlankURL() || url.isEmpty())
358         return;
359
360     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
361         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
362         statistics.isPrevalentResource = true;
363     });
364 }
365
366 void WebResourceLoadStatisticsStore::isPrevalentResource(const URL& url, WTF::Function<void (bool)>&& completionHandler)
367 {
368     if (url.isBlankURL() || url.isEmpty()) {
369         completionHandler(false);
370         return;
371     }
372
373     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), completionHandler = WTFMove(completionHandler)] () mutable {
374         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
375         bool isPrevalentResource = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource;
376         RunLoop::main().dispatch([isPrevalentResource, completionHandler = WTFMove(completionHandler)] {
377             completionHandler(isPrevalentResource);
378         });
379     });
380 }
381
382 void WebResourceLoadStatisticsStore::clearPrevalentResource(const URL& url)
383 {
384     if (url.isBlankURL() || url.isEmpty())
385         return;
386
387     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
388         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
389         statistics.isPrevalentResource = false;
390     });
391 }
392
393 void WebResourceLoadStatisticsStore::setGrandfathered(const URL& url, bool value)
394 {
395     if (url.isBlankURL() || url.isEmpty())
396         return;
397
398     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), value] {
399         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
400         statistics.grandfathered = value;
401     });
402 }
403
404 void WebResourceLoadStatisticsStore::isGrandfathered(const URL& url, WTF::Function<void (bool)>&& completionHandler)
405 {
406     if (url.isBlankURL() || url.isEmpty()) {
407         completionHandler(false);
408         return;
409     }
410
411     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler), primaryDomain = isolatedPrimaryDomain(url)] () mutable {
412         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
413         bool isGrandFathered = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.grandfathered;
414         RunLoop::main().dispatch([isGrandFathered, completionHandler = WTFMove(completionHandler)] {
415             completionHandler(isGrandFathered);
416         });
417     });
418 }
419
420 void WebResourceLoadStatisticsStore::setSubframeUnderTopFrameOrigin(const URL& subframe, const URL& topFrame)
421 {
422     if (subframe.isBlankURL() || subframe.isEmpty() || topFrame.isBlankURL() || topFrame.isEmpty())
423         return;
424
425     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryTopFrameDomain = isolatedPrimaryDomain(topFrame), primarySubFrameDomain = isolatedPrimaryDomain(subframe)] {
426         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubFrameDomain);
427         statistics.subframeUnderTopFrameOrigins.add(primaryTopFrameDomain);
428     });
429 }
430
431 void WebResourceLoadStatisticsStore::setSubresourceUnderTopFrameOrigin(const URL& subresource, const URL& topFrame)
432 {
433     if (subresource.isBlankURL() || subresource.isEmpty() || topFrame.isBlankURL() || topFrame.isEmpty())
434         return;
435
436     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryTopFrameDomain = isolatedPrimaryDomain(topFrame), primarySubresourceDomain = isolatedPrimaryDomain(subresource)] {
437         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
438         statistics.subresourceUnderTopFrameOrigins.add(primaryTopFrameDomain);
439     });
440 }
441
442 void WebResourceLoadStatisticsStore::setSubresourceUniqueRedirectTo(const URL& subresource, const URL& hostNameRedirectedTo)
443 {
444     if (subresource.isBlankURL() || subresource.isEmpty() || hostNameRedirectedTo.isBlankURL() || hostNameRedirectedTo.isEmpty())
445         return;
446
447     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryRedirectDomain = isolatedPrimaryDomain(hostNameRedirectedTo), primarySubresourceDomain = isolatedPrimaryDomain(subresource)] {
448         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
449         statistics.subresourceUniqueRedirectsTo.add(primaryRedirectDomain);
450     });
451 }
452
453 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningUpdate()
454 {
455     // Helper function used by testing system. Should only be called from the main thread.
456     ASSERT(RunLoop::isMain());
457
458     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
459         updateCookiePartitioning();
460     });
461 }
462
463 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningUpdateForDomains(const Vector<String>& domainsToRemove, const Vector<String>& domainsToAdd, ShouldClearFirst shouldClearFirst)
464 {
465     // Helper function used by testing system. Should only be called from the main thread.
466     ASSERT(RunLoop::isMain());
467     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), domainsToRemove = crossThreadCopy(domainsToRemove), domainsToAdd = crossThreadCopy(domainsToAdd), shouldClearFirst] {
468         updateCookiePartitioningForDomains(domainsToRemove, domainsToAdd, shouldClearFirst);
469     });
470 }
471
472 void WebResourceLoadStatisticsStore::scheduleClearPartitioningStateForDomains(const Vector<String>& domains)
473 {
474     // Helper function used by testing system. Should only be called from the main thread.
475     ASSERT(RunLoop::isMain());
476     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), domains = crossThreadCopy(domains)] {
477         clearPartitioningStateForDomains(domains);
478     });
479 }
480
481 #if HAVE(CFNETWORK_STORAGE_PARTITIONING)
482 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningStateReset()
483 {
484     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
485         resetCookiePartitioningState();
486     });
487 }
488 #endif
489
490 void WebResourceLoadStatisticsStore::scheduleClearInMemory()
491 {
492     ASSERT(RunLoop::isMain());
493     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
494         clearInMemory();
495     });
496 }
497
498 void WebResourceLoadStatisticsStore::scheduleClearInMemoryAndPersistent(ShouldGrandfather shouldGrandfather)
499 {
500     ASSERT(RunLoop::isMain());
501     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), shouldGrandfather] {
502         clearInMemory();
503         m_persistentStorage.clear();
504         
505         if (shouldGrandfather == ShouldGrandfather::Yes)
506             grandfatherExistingWebsiteData();
507     });
508 }
509
510 void WebResourceLoadStatisticsStore::scheduleClearInMemoryAndPersistent(std::chrono::system_clock::time_point modifiedSince, ShouldGrandfather shouldGrandfather)
511 {
512     // For now, be conservative and clear everything regardless of modifiedSince.
513     UNUSED_PARAM(modifiedSince);
514     scheduleClearInMemoryAndPersistent(shouldGrandfather);
515 }
516
517 void WebResourceLoadStatisticsStore::setTimeToLiveUserInteraction(Seconds seconds)
518 {
519     ASSERT(seconds >= 0_s);
520     m_parameters.timeToLiveUserInteraction = seconds;
521 }
522
523 void WebResourceLoadStatisticsStore::setTimeToLiveCookiePartitionFree(Seconds seconds)
524 {
525     ASSERT(seconds >= 0_s);
526     m_parameters.timeToLiveCookiePartitionFree = seconds;
527 }
528
529 void WebResourceLoadStatisticsStore::setMinimumTimeBetweenDataRecordsRemoval(Seconds seconds)
530 {
531     ASSERT(seconds >= 0_s);
532     m_parameters.minimumTimeBetweenDataRecordsRemoval = seconds;
533 }
534
535 void WebResourceLoadStatisticsStore::setGrandfatheringTime(Seconds seconds)
536 {
537     ASSERT(seconds >= 0_s);
538     m_parameters.grandfatheringTime = seconds;
539 }
540
541 bool WebResourceLoadStatisticsStore::shouldRemoveDataRecords() const
542 {
543     ASSERT(!RunLoop::isMain());
544     if (m_dataRecordsBeingRemoved)
545         return false;
546
547     return !m_lastTimeDataRecordsWereRemoved || MonotonicTime::now() >= (m_lastTimeDataRecordsWereRemoved + m_parameters.minimumTimeBetweenDataRecordsRemoval);
548 }
549
550 void WebResourceLoadStatisticsStore::setDataRecordsBeingRemoved(bool value)
551 {
552     ASSERT(!RunLoop::isMain());
553     m_dataRecordsBeingRemoved = value;
554     if (m_dataRecordsBeingRemoved)
555         m_lastTimeDataRecordsWereRemoved = MonotonicTime::now();
556 }
557
558 ResourceLoadStatistics& WebResourceLoadStatisticsStore::ensureResourceStatisticsForPrimaryDomain(const String& primaryDomain)
559 {
560     ASSERT(!RunLoop::isMain());
561     return m_resourceStatisticsMap.ensure(primaryDomain, [&primaryDomain] {
562         return ResourceLoadStatistics(primaryDomain);
563     }).iterator->value;
564 }
565
566 std::unique_ptr<KeyedEncoder> WebResourceLoadStatisticsStore::createEncoderFromData() const
567 {
568     ASSERT(!RunLoop::isMain());
569     auto encoder = KeyedEncoder::encoder();
570     encoder->encodeUInt32("version", statisticsModelVersion);
571     encoder->encodeDouble("endOfGrandfatheringTimestamp", m_endOfGrandfatheringTimestamp.secondsSinceEpoch().value());
572
573     encoder->encodeObjects("browsingStatistics", m_resourceStatisticsMap.begin(), m_resourceStatisticsMap.end(), [](KeyedEncoder& encoderInner, const auto& origin) {
574         origin.value.encode(encoderInner);
575     });
576
577     encoder->encodeObjects("operatingDates", m_operatingDates.begin(), m_operatingDates.end(), [](KeyedEncoder& encoderInner, OperatingDate date) {
578         encoderInner.encodeDouble("date", date.secondsSinceEpoch().value());
579     });
580
581     return encoder;
582 }
583
584 void WebResourceLoadStatisticsStore::mergeWithDataFromDecoder(KeyedDecoder& decoder)
585 {
586     ASSERT(!RunLoop::isMain());
587
588     unsigned versionOnDisk;
589     if (!decoder.decodeUInt32("version", versionOnDisk))
590         return;
591
592     if (versionOnDisk != statisticsModelVersion)
593         return;
594
595     double endOfGrandfatheringTimestamp;
596     if (decoder.decodeDouble("endOfGrandfatheringTimestamp", endOfGrandfatheringTimestamp))
597         m_endOfGrandfatheringTimestamp = WallTime::fromRawSeconds(endOfGrandfatheringTimestamp);
598     else
599         m_endOfGrandfatheringTimestamp = { };
600
601     Vector<ResourceLoadStatistics> loadedStatistics;
602     bool succeeded = decoder.decodeObjects("browsingStatistics", loadedStatistics, [](KeyedDecoder& decoderInner, ResourceLoadStatistics& statistics) {
603         return statistics.decode(decoderInner);
604     });
605
606     if (!succeeded)
607         return;
608
609     mergeStatistics(WTFMove(loadedStatistics));
610     updateCookiePartitioning();
611
612     Vector<OperatingDate> operatingDates;
613     succeeded = decoder.decodeObjects("operatingDates", operatingDates, [](KeyedDecoder& decoder, OperatingDate& date) {
614         double value;
615         if (!decoder.decodeDouble("date", value))
616             return false;
617
618         date = OperatingDate::fromWallTime(WallTime::fromRawSeconds(value));
619         return true;
620     });
621
622     if (!succeeded)
623         return;
624
625     m_operatingDates = mergeOperatingDates(m_operatingDates, WTFMove(operatingDates));
626 }
627
628 void WebResourceLoadStatisticsStore::clearInMemory()
629 {
630     ASSERT(!RunLoop::isMain());
631     m_resourceStatisticsMap.clear();
632     m_operatingDates.clear();
633
634     updateCookiePartitioningForDomains({ }, { }, ShouldClearFirst::Yes);
635 }
636
637 void WebResourceLoadStatisticsStore::mergeStatistics(Vector<ResourceLoadStatistics>&& statistics)
638 {
639     ASSERT(!RunLoop::isMain());
640     for (auto& statistic : statistics) {
641         auto result = m_resourceStatisticsMap.ensure(statistic.highLevelDomain, [&statistic] {
642             return WTFMove(statistic);
643         });
644         if (!result.isNewEntry)
645             result.iterator->value.merge(statistic);
646     }
647 }
648
649 inline bool WebResourceLoadStatisticsStore::shouldPartitionCookies(const ResourceLoadStatistics& statistic) const
650 {
651     return statistic.isPrevalentResource && (!statistic.hadUserInteraction || WallTime::now() > statistic.mostRecentUserInteractionTime + m_parameters.timeToLiveCookiePartitionFree);
652 }
653
654 void WebResourceLoadStatisticsStore::updateCookiePartitioning()
655 {
656     ASSERT(!RunLoop::isMain());
657
658     Vector<String> domainsWithoutInteraction;
659     Vector<String> domainsWithInteraction;
660     for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
661         bool shouldPartition = shouldPartitionCookies(resourceStatistic);
662         if (resourceStatistic.isMarkedForCookiePartitioning && !shouldPartition) {
663             resourceStatistic.isMarkedForCookiePartitioning = false;
664             domainsWithInteraction.append(resourceStatistic.highLevelDomain);
665         } else if (!resourceStatistic.isMarkedForCookiePartitioning && shouldPartition) {
666             resourceStatistic.isMarkedForCookiePartitioning = true;
667             domainsWithoutInteraction.append(resourceStatistic.highLevelDomain);
668         }
669     }
670
671     if (domainsWithInteraction.isEmpty() && domainsWithoutInteraction.isEmpty())
672         return;
673
674     RunLoop::main().dispatch([this, protectedThis = makeRef(*this), domainsWithInteraction = crossThreadCopy(domainsWithInteraction), domainsWithoutInteraction = crossThreadCopy(domainsWithoutInteraction)] () {
675         m_updatePrevalentDomainsWithAndWithoutInteractionHandler(domainsWithInteraction, domainsWithoutInteraction, ShouldClearFirst::No);
676     });
677 }
678
679 void WebResourceLoadStatisticsStore::updateCookiePartitioningForDomains(const Vector<String>& domainsWithInteraction, const Vector<String>& domainsWithoutInteraction, ShouldClearFirst shouldClearFirst)
680 {
681     ASSERT(!RunLoop::isMain());
682     if (domainsWithInteraction.isEmpty() && domainsWithoutInteraction.isEmpty())
683         return;
684
685     RunLoop::main().dispatch([this, shouldClearFirst, protectedThis = makeRef(*this), domainsWithInteraction = crossThreadCopy(domainsWithInteraction), domainsWithoutInteraction = crossThreadCopy(domainsWithoutInteraction)] () {
686         m_updatePrevalentDomainsWithAndWithoutInteractionHandler(domainsWithInteraction, domainsWithoutInteraction, shouldClearFirst);
687     });
688
689     if (shouldClearFirst == ShouldClearFirst::Yes)
690         resetCookiePartitioningState();
691     else {
692         for (auto& domain : domainsWithInteraction)
693             ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookiePartitioning = false;
694     }
695
696     for (auto& domain : domainsWithoutInteraction)
697         ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookiePartitioning = true;
698 }
699
700 void WebResourceLoadStatisticsStore::clearPartitioningStateForDomains(const Vector<String>& domains)
701 {
702     ASSERT(!RunLoop::isMain());
703     if (domains.isEmpty())
704         return;
705
706     RunLoop::main().dispatch([this, protectedThis = makeRef(*this), domains = crossThreadCopy(domains)] () {
707         m_removeDomainsHandler(domains);
708     });
709
710     for (auto& domain : domains)
711         ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookiePartitioning = false;
712 }
713
714 void WebResourceLoadStatisticsStore::resetCookiePartitioningState()
715 {
716     ASSERT(!RunLoop::isMain());
717     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
718         resourceStatistic.isMarkedForCookiePartitioning = false;
719 }
720
721 void WebResourceLoadStatisticsStore::processStatistics(const WTF::Function<void (const ResourceLoadStatistics&)>& processFunction) const
722 {
723     ASSERT(!RunLoop::isMain());
724     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
725         processFunction(resourceStatistic);
726 }
727
728 bool WebResourceLoadStatisticsStore::hasHadUnexpiredRecentUserInteraction(ResourceLoadStatistics& resourceStatistic) const
729 {
730     if (resourceStatistic.hadUserInteraction && hasStatisticsExpired(resourceStatistic)) {
731         // Drop privacy sensitive data because we no longer need it.
732         // Set timestamp to 0 so that statistics merge will know
733         // it has been reset as opposed to its default -1.
734         resourceStatistic.mostRecentUserInteractionTime = { };
735         resourceStatistic.hadUserInteraction = false;
736     }
737
738     return resourceStatistic.hadUserInteraction;
739 }
740
741 Vector<String> WebResourceLoadStatisticsStore::topPrivatelyControlledDomainsToRemoveWebsiteDataFor()
742 {
743     ASSERT(!RunLoop::isMain());
744
745     bool shouldCheckForGrandfathering = m_endOfGrandfatheringTimestamp > WallTime::now();
746     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && m_endOfGrandfatheringTimestamp;
747
748     if (shouldClearGrandfathering)
749         m_endOfGrandfatheringTimestamp = { };
750
751     Vector<String> prevalentResources;
752     for (auto& statistic : m_resourceStatisticsMap.values()) {
753         if (statistic.isPrevalentResource && !hasHadUnexpiredRecentUserInteraction(statistic) && (!shouldCheckForGrandfathering || !statistic.grandfathered))
754             prevalentResources.append(statistic.highLevelDomain);
755
756         if (shouldClearGrandfathering && statistic.grandfathered)
757             statistic.grandfathered = false;
758     }
759
760     return prevalentResources;
761 }
762
763 void WebResourceLoadStatisticsStore::includeTodayAsOperatingDateIfNecessary()
764 {
765     ASSERT(!RunLoop::isMain());
766
767     auto today = OperatingDate::today();
768     if (!m_operatingDates.isEmpty() && today <= m_operatingDates.last())
769         return;
770
771     while (m_operatingDates.size() >= operatingDatesWindow)
772         m_operatingDates.remove(0);
773
774     m_operatingDates.append(today);
775 }
776
777 bool WebResourceLoadStatisticsStore::hasStatisticsExpired(const ResourceLoadStatistics& resourceStatistic) const
778 {
779     if (m_operatingDates.size() >= operatingDatesWindow) {
780         if (OperatingDate::fromWallTime(resourceStatistic.mostRecentUserInteractionTime) < m_operatingDates.first())
781             return true;
782     }
783
784     // If we don't meet the real criteria for an expired statistic, check the user setting for a tighter restriction (mainly for testing).
785     if (m_parameters.timeToLiveUserInteraction) {
786         if (WallTime::now() > resourceStatistic.mostRecentUserInteractionTime + m_parameters.timeToLiveUserInteraction.value())
787             return true;
788     }
789
790     return false;
791 }
792     
793 void WebResourceLoadStatisticsStore::setMaxStatisticsEntries(size_t maximumEntryCount)
794 {
795     m_parameters.maxStatisticsEntries = maximumEntryCount;
796 }
797     
798 void WebResourceLoadStatisticsStore::setPruneEntriesDownTo(size_t pruneTargetCount)
799 {
800     m_parameters.pruneEntriesDownTo = pruneTargetCount;
801 }
802     
803 struct StatisticsLastSeen {
804     String topPrivatelyOwnedDomain;
805     WallTime lastSeen;
806 };
807     
808 static void pruneResources(HashMap<String, WebCore::ResourceLoadStatistics>& statisticsMap, Vector<StatisticsLastSeen>& statisticsToPrune, size_t& numberOfEntriesToPrune)
809 {
810     if (statisticsToPrune.size() > numberOfEntriesToPrune) {
811         std::sort(statisticsToPrune.begin(), statisticsToPrune.end(), [](const StatisticsLastSeen& a, const StatisticsLastSeen& b) {
812             return a.lastSeen < b.lastSeen;
813         });
814     }
815
816     for (size_t i = 0, end = std::min(numberOfEntriesToPrune, statisticsToPrune.size()); i != end; ++i, --numberOfEntriesToPrune)
817         statisticsMap.remove(statisticsToPrune[i].topPrivatelyOwnedDomain);
818 }
819     
820 static unsigned computeImportance(const ResourceLoadStatistics& resourceStatistic)
821 {
822     unsigned importance = maxImportance;
823     if (!resourceStatistic.isPrevalentResource)
824         importance -= 1;
825     if (!resourceStatistic.hadUserInteraction)
826         importance -= 2;
827     return importance;
828 }
829     
830 void WebResourceLoadStatisticsStore::pruneStatisticsIfNeeded()
831 {
832     ASSERT(!RunLoop::isMain());
833     if (m_resourceStatisticsMap.size() <= m_parameters.maxStatisticsEntries)
834         return;
835
836     ASSERT(m_parameters.pruneEntriesDownTo <= m_parameters.maxStatisticsEntries);
837
838     size_t numberOfEntriesLeftToPrune = m_resourceStatisticsMap.size() - m_parameters.pruneEntriesDownTo;
839     ASSERT(numberOfEntriesLeftToPrune);
840     
841     Vector<StatisticsLastSeen> resourcesToPrunePerImportance[maxImportance + 1];
842     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
843         resourcesToPrunePerImportance[computeImportance(resourceStatistic)].append({ resourceStatistic.highLevelDomain, resourceStatistic.lastSeen });
844     
845     for (unsigned importance = 0; numberOfEntriesLeftToPrune && importance <= maxImportance; ++importance)
846         pruneResources(m_resourceStatisticsMap, resourcesToPrunePerImportance[importance], numberOfEntriesLeftToPrune);
847
848     ASSERT(!numberOfEntriesLeftToPrune);
849 }
850
851 void WebResourceLoadStatisticsStore::resetParametersToDefaultValues()
852 {
853     m_parameters = { };
854 }
855
856 void WebResourceLoadStatisticsStore::logTestingEvent(const String& event)
857 {
858     if (!m_statisticsTestingCallback)
859         return;
860
861     if (RunLoop::isMain())
862         m_statisticsTestingCallback(event);
863     else {
864         RunLoop::main().dispatch([this, protectedThis = makeRef(*this), event = event.isolatedCopy()] {
865             if (m_statisticsTestingCallback)
866                 m_statisticsTestingCallback(event);
867         });
868     }
869 }
870
871 } // namespace WebKit