Moved filesystem code out of WebResourceLoadStatisticsStore class
[WebKit.git] / Source / WebKit2 / UIProcess / WebResourceLoadStatisticsStore.cpp
1 /*
2  * Copyright (C) 2016-2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WebResourceLoadStatisticsStore.h"
28
29 #include "Logging.h"
30 #include "WebProcessMessages.h"
31 #include "WebProcessProxy.h"
32 #include "WebResourceLoadStatisticsStoreMessages.h"
33 #include "WebResourceLoadStatisticsTelemetry.h"
34 #include "WebsiteDataFetchOption.h"
35 #include "WebsiteDataStore.h"
36 #include "WebsiteDataType.h"
37 #include <WebCore/KeyedCoding.h>
38 #include <WebCore/ResourceLoadStatistics.h>
39 #include <wtf/CrossThreadCopier.h>
40 #include <wtf/MathExtras.h>
41 #include <wtf/NeverDestroyed.h>
42
43 using namespace WebCore;
44
45 namespace WebKit {
46
47 constexpr unsigned operatingDatesWindow { 30 };
48 constexpr unsigned statisticsModelVersion { 7 };
49 constexpr unsigned maxImportance { 3 };
50
51 template<typename T> static inline String isolatedPrimaryDomain(const T& value)
52 {
53     return ResourceLoadStatistics::primaryDomain(value).isolatedCopy();
54 }
55
56 static const OptionSet<WebsiteDataType>& dataTypesToRemove()
57 {
58     static NeverDestroyed<OptionSet<WebsiteDataType>> dataTypes(std::initializer_list<WebsiteDataType>({
59         WebsiteDataType::Cookies,
60         WebsiteDataType::IndexedDBDatabases,
61         WebsiteDataType::LocalStorage,
62 #if ENABLE(MEDIA_STREAM)
63         WebsiteDataType::MediaDeviceIdentifier,
64 #endif
65         WebsiteDataType::MediaKeys,
66         WebsiteDataType::OfflineWebApplicationCache,
67 #if ENABLE(NETSCAPE_PLUGIN_API)
68         WebsiteDataType::PlugInData,
69 #endif
70         WebsiteDataType::SearchFieldRecentSearches,
71         WebsiteDataType::SessionStorage,
72         WebsiteDataType::WebSQLDatabases,
73     }));
74
75     ASSERT(RunLoop::isMain());
76
77     return dataTypes;
78 }
79
80 WebResourceLoadStatisticsStore::WebResourceLoadStatisticsStore(const String& resourceLoadStatisticsDirectory, UpdateCookiePartitioningForDomainsHandler&& updateCookiePartitioningForDomainsHandler)
81     : m_statisticsQueue(WorkQueue::create("WebResourceLoadStatisticsStore Process Data Queue", WorkQueue::Type::Serial, WorkQueue::QOS::Utility))
82     , m_persistentStorage(*this, resourceLoadStatisticsDirectory)
83     , m_updateCookiePartitioningForDomainsHandler(WTFMove(updateCookiePartitioningForDomainsHandler))
84     , m_dailyTasksTimer(RunLoop::main(), this, &WebResourceLoadStatisticsStore::performDailyTasks)
85 {
86     ASSERT(RunLoop::isMain());
87
88 #if PLATFORM(COCOA)
89     registerUserDefaultsIfNeeded();
90 #endif
91
92     m_statisticsQueue->dispatchAfter(5_s, [this, protectedThis = makeRef(*this)] {
93         if (m_parameters.shouldSubmitTelemetry)
94             WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
95     });
96
97     m_dailyTasksTimer.startRepeating(24_h);
98 }
99
100 WebResourceLoadStatisticsStore::~WebResourceLoadStatisticsStore()
101 {
102 }
103     
104 void WebResourceLoadStatisticsStore::removeDataRecords()
105 {
106     ASSERT(!RunLoop::isMain());
107     
108     if (!shouldRemoveDataRecords())
109         return;
110
111     auto prevalentResourceDomains = topPrivatelyControlledDomainsToRemoveWebsiteDataFor();
112     if (prevalentResourceDomains.isEmpty())
113         return;
114     
115     setDataRecordsBeingRemoved(true);
116
117     RunLoop::main().dispatch([prevalentResourceDomains = CrossThreadCopier<Vector<String>>::copy(prevalentResourceDomains), this, protectedThis = makeRef(*this)] () mutable {
118         WebProcessProxy::deleteWebsiteDataForTopPrivatelyControlledDomainsInAllPersistentDataStores(dataTypesToRemove(), WTFMove(prevalentResourceDomains), m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, [this, protectedThis = WTFMove(protectedThis)](const HashSet<String>& domainsWithDeletedWebsiteData) mutable {
119             m_statisticsQueue->dispatch([this, protectedThis = WTFMove(protectedThis), topDomains = CrossThreadCopier<HashSet<String>>::copy(domainsWithDeletedWebsiteData)] () mutable {
120                 for (auto& prevalentResourceDomain : topDomains) {
121                     auto& statistic = ensureResourceStatisticsForPrimaryDomain(prevalentResourceDomain);
122                     ++statistic.dataRecordsRemoved;
123                 }
124                 setDataRecordsBeingRemoved(false);
125             });
126         });
127     });
128 }
129
130 void WebResourceLoadStatisticsStore::processStatisticsAndDataRecords()
131 {
132     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] () {
133         if (m_parameters.shouldClassifyResourcesBeforeDataRecordsRemoval) {
134             for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
135                 if (!resourceStatistic.isPrevalentResource && m_resourceLoadStatisticsClassifier.hasPrevalentResourceCharacteristics(resourceStatistic))
136                     resourceStatistic.isPrevalentResource = true;
137             }
138         }
139         removeDataRecords();
140         
141         pruneStatisticsIfNeeded();
142
143         if (m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned) {
144             RunLoop::main().dispatch([] {
145                 WebProcessProxy::notifyPageStatisticsAndDataRecordsProcessed();
146             });
147         }
148
149         m_persistentStorage.scheduleOrWriteMemoryStore();
150     });
151 }
152
153 void WebResourceLoadStatisticsStore::resourceLoadStatisticsUpdated(Vector<WebCore::ResourceLoadStatistics>&& origins)
154 {
155     ASSERT(!RunLoop::isMain());
156
157     mergeStatistics(WTFMove(origins));
158     // Fire before processing statistics to propagate user interaction as fast as possible to the network process.
159     updateCookiePartitioning();
160     processStatisticsAndDataRecords();
161 }
162
163 void WebResourceLoadStatisticsStore::grandfatherExistingWebsiteData()
164 {
165     ASSERT(!RunLoop::isMain());
166
167     RunLoop::main().dispatch([this, protectedThis = makeRef(*this)] () mutable {
168         WebProcessProxy::topPrivatelyControlledDomainsWithWebsiteData(dataTypesToRemove(), m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, [this, protectedThis = WTFMove(protectedThis)] (HashSet<String>&& topPrivatelyControlledDomainsWithWebsiteData) mutable {
169             m_statisticsQueue->dispatch([this, protectedThis = WTFMove(protectedThis), topDomains = CrossThreadCopier<HashSet<String>>::copy(topPrivatelyControlledDomainsWithWebsiteData)] () mutable {
170                 for (auto& topPrivatelyControlledDomain : topDomains) {
171                     auto& statistic = ensureResourceStatisticsForPrimaryDomain(topPrivatelyControlledDomain);
172                     statistic.grandfathered = true;
173                 }
174                 m_endOfGrandfatheringTimestamp = WallTime::now() + m_parameters.grandfatheringTime;
175             });
176         });
177     });
178 }
179     
180 void WebResourceLoadStatisticsStore::processWillOpenConnection(WebProcessProxy&, IPC::Connection& connection)
181 {
182     connection.addWorkQueueMessageReceiver(Messages::WebResourceLoadStatisticsStore::messageReceiverName(), m_statisticsQueue.get(), this);
183 }
184
185 void WebResourceLoadStatisticsStore::processDidCloseConnection(WebProcessProxy&, IPC::Connection& connection)
186 {
187     connection.removeWorkQueueMessageReceiver(Messages::WebResourceLoadStatisticsStore::messageReceiverName());
188 }
189
190 void WebResourceLoadStatisticsStore::applicationWillTerminate()
191 {
192     m_persistentStorage.finishAllPendingWorkSynchronously();
193 }
194
195 void WebResourceLoadStatisticsStore::performDailyTasks()
196 {
197     ASSERT(RunLoop::isMain());
198
199     includeTodayAsOperatingDateIfNecessary();
200     if (m_parameters.shouldSubmitTelemetry)
201         submitTelemetry();
202 }
203
204 void WebResourceLoadStatisticsStore::submitTelemetry()
205 {
206     ASSERT(RunLoop::isMain());
207     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
208         WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
209     });
210 }
211
212 void WebResourceLoadStatisticsStore::logUserInteraction(const URL& url)
213 {
214     if (url.isBlankURL() || url.isEmpty())
215         return;
216
217     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
218         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
219         statistics.hadUserInteraction = true;
220         statistics.mostRecentUserInteractionTime = WallTime::now();
221
222         updateCookiePartitioningForDomains({ primaryDomain }, { }, ShouldClearFirst::No);
223     });
224 }
225
226 void WebResourceLoadStatisticsStore::clearUserInteraction(const URL& url)
227 {
228     if (url.isBlankURL() || url.isEmpty())
229         return;
230
231     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
232         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
233         statistics.hadUserInteraction = false;
234         statistics.mostRecentUserInteractionTime = { };
235     });
236 }
237
238 void WebResourceLoadStatisticsStore::hasHadUserInteraction(const URL& url, WTF::Function<void (bool)>&& completionHandler)
239 {
240     if (url.isBlankURL() || url.isEmpty()) {
241         completionHandler(false);
242         return;
243     }
244
245     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), completionHandler = WTFMove(completionHandler)] () mutable {
246         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
247         bool hadUserInteraction = mapEntry == m_resourceStatisticsMap.end() ? false: hasHadUnexpiredRecentUserInteraction(mapEntry->value);
248         RunLoop::main().dispatch([hadUserInteraction, completionHandler = WTFMove(completionHandler)] {
249             completionHandler(hadUserInteraction);
250         });
251     });
252 }
253
254 void WebResourceLoadStatisticsStore::setLastSeen(const URL& url, Seconds seconds)
255 {
256     if (url.isBlankURL() || url.isEmpty())
257         return;
258     
259     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), seconds] {
260         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
261         statistics.lastSeen = WallTime::fromRawSeconds(seconds.seconds());
262     });
263 }
264     
265 void WebResourceLoadStatisticsStore::setPrevalentResource(const URL& url)
266 {
267     if (url.isBlankURL() || url.isEmpty())
268         return;
269
270     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
271         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
272         statistics.isPrevalentResource = true;
273     });
274 }
275
276 void WebResourceLoadStatisticsStore::isPrevalentResource(const URL& url, WTF::Function<void (bool)>&& completionHandler)
277 {
278     if (url.isBlankURL() || url.isEmpty()) {
279         completionHandler(false);
280         return;
281     }
282
283     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), completionHandler = WTFMove(completionHandler)] () mutable {
284         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
285         bool isPrevalentResource = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource;
286         RunLoop::main().dispatch([isPrevalentResource, completionHandler = WTFMove(completionHandler)] {
287             completionHandler(isPrevalentResource);
288         });
289     });
290 }
291
292 void WebResourceLoadStatisticsStore::clearPrevalentResource(const URL& url)
293 {
294     if (url.isBlankURL() || url.isEmpty())
295         return;
296
297     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
298         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
299         statistics.isPrevalentResource = false;
300     });
301 }
302
303 void WebResourceLoadStatisticsStore::setGrandfathered(const URL& url, bool value)
304 {
305     if (url.isBlankURL() || url.isEmpty())
306         return;
307
308     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), value] {
309         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
310         statistics.grandfathered = value;
311     });
312 }
313
314 void WebResourceLoadStatisticsStore::isGrandfathered(const URL& url, WTF::Function<void (bool)>&& completionHandler)
315 {
316     if (url.isBlankURL() || url.isEmpty()) {
317         completionHandler(false);
318         return;
319     }
320
321     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler), primaryDomain = isolatedPrimaryDomain(url)] () mutable {
322         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
323         bool isGrandFathered = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.grandfathered;
324         RunLoop::main().dispatch([isGrandFathered, completionHandler = WTFMove(completionHandler)] {
325             completionHandler(isGrandFathered);
326         });
327     });
328 }
329
330 void WebResourceLoadStatisticsStore::setSubframeUnderTopFrameOrigin(const URL& subframe, const URL& topFrame)
331 {
332     if (subframe.isBlankURL() || subframe.isEmpty() || topFrame.isBlankURL() || topFrame.isEmpty())
333         return;
334
335     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryTopFrameDomain = isolatedPrimaryDomain(topFrame), primarySubFrameDomain = isolatedPrimaryDomain(subframe)] {
336         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubFrameDomain);
337         statistics.subframeUnderTopFrameOrigins.add(primaryTopFrameDomain);
338     });
339 }
340
341 void WebResourceLoadStatisticsStore::setSubresourceUnderTopFrameOrigin(const URL& subresource, const URL& topFrame)
342 {
343     if (subresource.isBlankURL() || subresource.isEmpty() || topFrame.isBlankURL() || topFrame.isEmpty())
344         return;
345
346     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryTopFrameDomain = isolatedPrimaryDomain(topFrame), primarySubresourceDomain = isolatedPrimaryDomain(subresource)] {
347         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
348         statistics.subresourceUnderTopFrameOrigins.add(primaryTopFrameDomain);
349     });
350 }
351
352 void WebResourceLoadStatisticsStore::setSubresourceUniqueRedirectTo(const URL& subresource, const URL& hostNameRedirectedTo)
353 {
354     if (subresource.isBlankURL() || subresource.isEmpty() || hostNameRedirectedTo.isBlankURL() || hostNameRedirectedTo.isEmpty())
355         return;
356
357     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryRedirectDomain = isolatedPrimaryDomain(hostNameRedirectedTo), primarySubresourceDomain = isolatedPrimaryDomain(subresource)] {
358         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
359         statistics.subresourceUniqueRedirectsTo.add(primaryRedirectDomain);
360     });
361 }
362
363 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningUpdate()
364 {
365     // Helper function used by testing system. Should only be called from the main thread.
366     ASSERT(RunLoop::isMain());
367
368     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
369         updateCookiePartitioning();
370     });
371 }
372
373 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningUpdateForDomains(const Vector<String>& domainsToRemove, const Vector<String>& domainsToAdd, ShouldClearFirst shouldClearFirst)
374 {
375     // Helper function used by testing system. Should only be called from the main thread.
376     ASSERT(RunLoop::isMain());
377     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), domainsToRemove = CrossThreadCopier<Vector<String>>::copy(domainsToRemove), domainsToAdd = CrossThreadCopier<Vector<String>>::copy(domainsToAdd), shouldClearFirst] {
378         updateCookiePartitioningForDomains(domainsToRemove, domainsToAdd, shouldClearFirst);
379     });
380 }
381
382 #if HAVE(CFNETWORK_STORAGE_PARTITIONING)
383 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningStateReset()
384 {
385     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
386         resetCookiePartitioningState();
387     });
388 }
389 #endif
390
391 void WebResourceLoadStatisticsStore::scheduleClearInMemory()
392 {
393     ASSERT(RunLoop::isMain());
394     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
395         clearInMemory();
396     });
397 }
398
399 void WebResourceLoadStatisticsStore::scheduleClearInMemoryAndPersistent()
400 {
401     ASSERT(RunLoop::isMain());
402     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
403         clearInMemory();
404         m_persistentStorage.clear();
405         grandfatherExistingWebsiteData();
406     });
407 }
408
409 void WebResourceLoadStatisticsStore::scheduleClearInMemoryAndPersistent(std::chrono::system_clock::time_point modifiedSince)
410 {
411     // For now, be conservative and clear everything regardless of modifiedSince.
412     UNUSED_PARAM(modifiedSince);
413     scheduleClearInMemoryAndPersistent();
414 }
415
416 void WebResourceLoadStatisticsStore::setTimeToLiveUserInteraction(Seconds seconds)
417 {
418     ASSERT(seconds >= 0_s);
419     m_parameters.timeToLiveUserInteraction = seconds;
420 }
421
422 void WebResourceLoadStatisticsStore::setTimeToLiveCookiePartitionFree(Seconds seconds)
423 {
424     ASSERT(seconds >= 0_s);
425     m_parameters.timeToLiveCookiePartitionFree = seconds;
426 }
427
428 void WebResourceLoadStatisticsStore::setMinimumTimeBetweenDataRecordsRemoval(Seconds seconds)
429 {
430     ASSERT(seconds >= 0_s);
431     m_parameters.minimumTimeBetweenDataRecordsRemoval = seconds;
432 }
433
434 void WebResourceLoadStatisticsStore::setGrandfatheringTime(Seconds seconds)
435 {
436     ASSERT(seconds >= 0_s);
437     m_parameters.grandfatheringTime = seconds;
438 }
439
440 bool WebResourceLoadStatisticsStore::shouldRemoveDataRecords() const
441 {
442     ASSERT(!RunLoop::isMain());
443     if (m_dataRecordsBeingRemoved)
444         return false;
445
446     return !m_lastTimeDataRecordsWereRemoved || MonotonicTime::now() >= (m_lastTimeDataRecordsWereRemoved + m_parameters.minimumTimeBetweenDataRecordsRemoval);
447 }
448
449 void WebResourceLoadStatisticsStore::setDataRecordsBeingRemoved(bool value)
450 {
451     ASSERT(!RunLoop::isMain());
452     m_dataRecordsBeingRemoved = value;
453     if (m_dataRecordsBeingRemoved)
454         m_lastTimeDataRecordsWereRemoved = MonotonicTime::now();
455 }
456
457 ResourceLoadStatistics& WebResourceLoadStatisticsStore::ensureResourceStatisticsForPrimaryDomain(const String& primaryDomain)
458 {
459     ASSERT(!RunLoop::isMain());
460     return m_resourceStatisticsMap.ensure(primaryDomain, [&primaryDomain] {
461         return ResourceLoadStatistics(primaryDomain);
462     }).iterator->value;
463 }
464
465 std::unique_ptr<KeyedEncoder> WebResourceLoadStatisticsStore::createEncoderFromData() const
466 {
467     ASSERT(!RunLoop::isMain());
468     auto encoder = KeyedEncoder::encoder();
469     encoder->encodeUInt32("version", statisticsModelVersion);
470     encoder->encodeDouble("endOfGrandfatheringTimestamp", m_endOfGrandfatheringTimestamp.secondsSinceEpoch().value());
471
472     encoder->encodeObjects("browsingStatistics", m_resourceStatisticsMap.begin(), m_resourceStatisticsMap.end(), [](KeyedEncoder& encoderInner, const auto& origin) {
473         origin.value.encode(encoderInner);
474     });
475
476     encoder->encodeObjects("operatingDates", m_operatingDates.begin(), m_operatingDates.end(), [](KeyedEncoder& encoderInner, WallTime date) {
477         encoderInner.encodeDouble("date", date.secondsSinceEpoch().value());
478     });
479
480     return encoder;
481 }
482
483 void WebResourceLoadStatisticsStore::resetDataFromDecoder(KeyedDecoder& decoder)
484 {
485     ASSERT(!RunLoop::isMain());
486
487     clearInMemory();
488
489     unsigned versionOnDisk;
490     if (!decoder.decodeUInt32("version", versionOnDisk))
491         return;
492
493     if (versionOnDisk != statisticsModelVersion)
494         return;
495
496     double endOfGrandfatheringTimestamp;
497     if (decoder.decodeDouble("endOfGrandfatheringTimestamp", endOfGrandfatheringTimestamp))
498         m_endOfGrandfatheringTimestamp = WallTime::fromRawSeconds(endOfGrandfatheringTimestamp);
499     else
500         m_endOfGrandfatheringTimestamp = { };
501
502     Vector<ResourceLoadStatistics> loadedStatistics;
503     bool succeeded = decoder.decodeObjects("browsingStatistics", loadedStatistics, [](KeyedDecoder& decoderInner, ResourceLoadStatistics& statistics) {
504         return statistics.decode(decoderInner);
505     });
506
507     if (!succeeded)
508         return;
509
510     Vector<String> prevalentResourceDomainsWithoutUserInteraction;
511     prevalentResourceDomainsWithoutUserInteraction.reserveInitialCapacity(loadedStatistics.size());
512     for (auto& statistics : loadedStatistics) {
513         if (statistics.isPrevalentResource && !statistics.hadUserInteraction) {
514             prevalentResourceDomainsWithoutUserInteraction.uncheckedAppend(statistics.highLevelDomain);
515             statistics.isMarkedForCookiePartitioning = true;
516         }
517         m_resourceStatisticsMap.add(statistics.highLevelDomain, WTFMove(statistics));
518     }
519
520     succeeded = decoder.decodeObjects("operatingDates", m_operatingDates, [](KeyedDecoder& decoder, WallTime& wallTime) {
521         double value;
522         if (!decoder.decodeDouble("date", value))
523             return false;
524
525         wallTime = WallTime::fromRawSeconds(value);
526         return true;
527     });
528
529     if (!succeeded)
530         return;
531
532     updateCookiePartitioningForDomains({ }, prevalentResourceDomainsWithoutUserInteraction, ShouldClearFirst::Yes);
533 }
534
535 void WebResourceLoadStatisticsStore::clearInMemory()
536 {
537     ASSERT(!RunLoop::isMain());
538     m_resourceStatisticsMap.clear();
539     m_operatingDates.clear();
540
541     updateCookiePartitioningForDomains({ }, { }, ShouldClearFirst::Yes);
542 }
543
544 void WebResourceLoadStatisticsStore::mergeStatistics(Vector<ResourceLoadStatistics>&& statistics)
545 {
546     ASSERT(!RunLoop::isMain());
547     for (auto& statistic : statistics) {
548         auto result = m_resourceStatisticsMap.ensure(statistic.highLevelDomain, [&statistic] {
549             return WTFMove(statistic);
550         });
551         if (!result.isNewEntry)
552             result.iterator->value.merge(statistic);
553     }
554 }
555
556 inline bool WebResourceLoadStatisticsStore::shouldPartitionCookies(const ResourceLoadStatistics& statistic) const
557 {
558     return statistic.isPrevalentResource && (!statistic.hadUserInteraction || WallTime::now() > statistic.mostRecentUserInteractionTime + m_parameters.timeToLiveCookiePartitionFree);
559 }
560
561 void WebResourceLoadStatisticsStore::updateCookiePartitioning()
562 {
563     ASSERT(!RunLoop::isMain());
564
565     Vector<String> domainsToRemove;
566     Vector<String> domainsToAdd;
567     for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
568         bool shouldPartition = shouldPartitionCookies(resourceStatistic);
569         if (resourceStatistic.isMarkedForCookiePartitioning && !shouldPartition) {
570             resourceStatistic.isMarkedForCookiePartitioning = false;
571             domainsToRemove.append(resourceStatistic.highLevelDomain);
572         } else if (!resourceStatistic.isMarkedForCookiePartitioning && shouldPartition) {
573             resourceStatistic.isMarkedForCookiePartitioning = true;
574             domainsToAdd.append(resourceStatistic.highLevelDomain);
575         }
576     }
577
578     if (domainsToRemove.isEmpty() && domainsToAdd.isEmpty())
579         return;
580
581     RunLoop::main().dispatch([this, protectedThis = makeRef(*this), domainsToRemove = CrossThreadCopier<Vector<String>>::copy(domainsToRemove), domainsToAdd = CrossThreadCopier<Vector<String>>::copy(domainsToAdd)] () {
582         m_updateCookiePartitioningForDomainsHandler(domainsToRemove, domainsToAdd, ShouldClearFirst::No);
583     });
584 }
585
586 void WebResourceLoadStatisticsStore::updateCookiePartitioningForDomains(const Vector<String>& domainsToRemove, const Vector<String>& domainsToAdd, ShouldClearFirst shouldClearFirst)
587 {
588     ASSERT(!RunLoop::isMain());
589     if (domainsToRemove.isEmpty() && domainsToAdd.isEmpty())
590         return;
591
592     RunLoop::main().dispatch([this, shouldClearFirst, protectedThis = makeRef(*this), domainsToRemove = CrossThreadCopier<Vector<String>>::copy(domainsToRemove), domainsToAdd = CrossThreadCopier<Vector<String>>::copy(domainsToAdd)] () {
593         m_updateCookiePartitioningForDomainsHandler(domainsToRemove, domainsToAdd, shouldClearFirst);
594     });
595
596     if (shouldClearFirst == ShouldClearFirst::Yes)
597         resetCookiePartitioningState();
598     else {
599         for (auto& domain : domainsToRemove)
600             ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookiePartitioning = false;
601     }
602
603     for (auto& domain : domainsToAdd)
604         ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookiePartitioning = true;
605 }
606
607 void WebResourceLoadStatisticsStore::resetCookiePartitioningState()
608 {
609     ASSERT(!RunLoop::isMain());
610     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
611         resourceStatistic.isMarkedForCookiePartitioning = false;
612 }
613
614 void WebResourceLoadStatisticsStore::processStatistics(const WTF::Function<void (const ResourceLoadStatistics&)>& processFunction) const
615 {
616     ASSERT(!RunLoop::isMain());
617     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
618         processFunction(resourceStatistic);
619 }
620
621 bool WebResourceLoadStatisticsStore::hasHadUnexpiredRecentUserInteraction(ResourceLoadStatistics& resourceStatistic) const
622 {
623     if (resourceStatistic.hadUserInteraction && hasStatisticsExpired(resourceStatistic)) {
624         // Drop privacy sensitive data because we no longer need it.
625         // Set timestamp to 0 so that statistics merge will know
626         // it has been reset as opposed to its default -1.
627         resourceStatistic.mostRecentUserInteractionTime = { };
628         resourceStatistic.hadUserInteraction = false;
629     }
630
631     return resourceStatistic.hadUserInteraction;
632 }
633
634 Vector<String> WebResourceLoadStatisticsStore::topPrivatelyControlledDomainsToRemoveWebsiteDataFor()
635 {
636     ASSERT(!RunLoop::isMain());
637
638     bool shouldCheckForGrandfathering = m_endOfGrandfatheringTimestamp > WallTime::now();
639     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && m_endOfGrandfatheringTimestamp;
640
641     if (shouldClearGrandfathering)
642         m_endOfGrandfatheringTimestamp = { };
643
644     Vector<String> prevalentResources;
645     for (auto& statistic : m_resourceStatisticsMap.values()) {
646         if (statistic.isPrevalentResource && !hasHadUnexpiredRecentUserInteraction(statistic) && (!shouldCheckForGrandfathering || !statistic.grandfathered))
647             prevalentResources.append(statistic.highLevelDomain);
648
649         if (shouldClearGrandfathering && statistic.grandfathered)
650             statistic.grandfathered = false;
651     }
652
653     return prevalentResources;
654 }
655
656 void WebResourceLoadStatisticsStore::includeTodayAsOperatingDateIfNecessary()
657 {
658     if (!m_operatingDates.isEmpty() && (WallTime::now() - m_operatingDates.last() < 24_h))
659         return;
660
661     while (m_operatingDates.size() >= operatingDatesWindow)
662         m_operatingDates.removeFirst();
663
664     m_operatingDates.append(WallTime::now());
665 }
666
667 bool WebResourceLoadStatisticsStore::hasStatisticsExpired(const ResourceLoadStatistics& resourceStatistic) const
668 {
669     if (m_operatingDates.size() >= operatingDatesWindow) {
670         if (resourceStatistic.mostRecentUserInteractionTime < m_operatingDates.first())
671             return true;
672     }
673
674     // If we don't meet the real criteria for an expired statistic, check the user setting for a tighter restriction (mainly for testing).
675     if (m_parameters.timeToLiveUserInteraction) {
676         if (WallTime::now() > resourceStatistic.mostRecentUserInteractionTime + m_parameters.timeToLiveUserInteraction.value())
677             return true;
678     }
679
680     return false;
681 }
682     
683 void WebResourceLoadStatisticsStore::setMaxStatisticsEntries(size_t maximumEntryCount)
684 {
685     m_parameters.maxStatisticsEntries = maximumEntryCount;
686 }
687     
688 void WebResourceLoadStatisticsStore::setPruneEntriesDownTo(size_t pruneTargetCount)
689 {
690     m_parameters.pruneEntriesDownTo = pruneTargetCount;
691 }
692     
693 struct StatisticsLastSeen {
694     String topPrivatelyOwnedDomain;
695     WallTime lastSeen;
696 };
697     
698 static void pruneResources(HashMap<String, WebCore::ResourceLoadStatistics>& statisticsMap, Vector<StatisticsLastSeen>& statisticsToPrune, size_t& numberOfEntriesToPrune)
699 {
700     if (statisticsToPrune.size() > numberOfEntriesToPrune) {
701         std::sort(statisticsToPrune.begin(), statisticsToPrune.end(), [](const StatisticsLastSeen& a, const StatisticsLastSeen& b) {
702             return a.lastSeen < b.lastSeen;
703         });
704     }
705
706     for (size_t i = 0, end = std::min(numberOfEntriesToPrune, statisticsToPrune.size()); i != end; ++i, --numberOfEntriesToPrune)
707         statisticsMap.remove(statisticsToPrune[i].topPrivatelyOwnedDomain);
708 }
709     
710 static unsigned computeImportance(const ResourceLoadStatistics& resourceStatistic)
711 {
712     unsigned importance = maxImportance;
713     if (!resourceStatistic.isPrevalentResource)
714         importance -= 1;
715     if (!resourceStatistic.hadUserInteraction)
716         importance -= 2;
717     return importance;
718 }
719     
720 void WebResourceLoadStatisticsStore::pruneStatisticsIfNeeded()
721 {
722     ASSERT(!RunLoop::isMain());
723     if (m_resourceStatisticsMap.size() <= m_parameters.maxStatisticsEntries)
724         return;
725
726     ASSERT(m_parameters.pruneEntriesDownTo <= m_parameters.maxStatisticsEntries);
727
728     size_t numberOfEntriesLeftToPrune = m_resourceStatisticsMap.size() - m_parameters.pruneEntriesDownTo;
729     ASSERT(numberOfEntriesLeftToPrune);
730     
731     Vector<StatisticsLastSeen> resourcesToPrunePerImportance[maxImportance + 1];
732     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
733         resourcesToPrunePerImportance[computeImportance(resourceStatistic)].append({ resourceStatistic.highLevelDomain, resourceStatistic.lastSeen });
734     
735     for (unsigned importance = 0; numberOfEntriesLeftToPrune && importance <= maxImportance; ++importance)
736         pruneResources(m_resourceStatisticsMap, resourcesToPrunePerImportance[importance], numberOfEntriesLeftToPrune);
737
738     ASSERT(!numberOfEntriesLeftToPrune);
739 }
740
741 void WebResourceLoadStatisticsStore::resetParametersToDefaultValues()
742 {
743     m_parameters = { };
744 }
745     
746 } // namespace WebKit