WebResourceLoadStatisticsStore::m_operatingDates is unsafely modified from several...
[WebKit-https.git] / Source / WebKit / UIProcess / WebResourceLoadStatisticsStore.cpp
1 /*
2  * Copyright (C) 2016-2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WebResourceLoadStatisticsStore.h"
28
29 #include "Logging.h"
30 #include "WebProcessMessages.h"
31 #include "WebProcessProxy.h"
32 #include "WebResourceLoadStatisticsStoreMessages.h"
33 #include "WebResourceLoadStatisticsTelemetry.h"
34 #include "WebsiteDataFetchOption.h"
35 #include "WebsiteDataStore.h"
36 #include "WebsiteDataType.h"
37 #include <WebCore/KeyedCoding.h>
38 #include <WebCore/ResourceLoadStatistics.h>
39 #include <wtf/CrossThreadCopier.h>
40 #include <wtf/DateMath.h>
41 #include <wtf/MathExtras.h>
42 #include <wtf/NeverDestroyed.h>
43
44 using namespace WebCore;
45
46 namespace WebKit {
47
48 constexpr unsigned operatingDatesWindow { 30 };
49 constexpr unsigned statisticsModelVersion { 9 };
50 constexpr unsigned maxImportance { 3 };
51
52 template<typename T> static inline String isolatedPrimaryDomain(const T& value)
53 {
54     return ResourceLoadStatistics::primaryDomain(value).isolatedCopy();
55 }
56
57 static const OptionSet<WebsiteDataType>& dataTypesToRemove()
58 {
59     static NeverDestroyed<OptionSet<WebsiteDataType>> dataTypes(std::initializer_list<WebsiteDataType>({
60         WebsiteDataType::Cookies,
61         WebsiteDataType::IndexedDBDatabases,
62         WebsiteDataType::LocalStorage,
63 #if ENABLE(MEDIA_STREAM)
64         WebsiteDataType::MediaDeviceIdentifier,
65 #endif
66         WebsiteDataType::MediaKeys,
67         WebsiteDataType::OfflineWebApplicationCache,
68 #if ENABLE(NETSCAPE_PLUGIN_API)
69         WebsiteDataType::PlugInData,
70 #endif
71         WebsiteDataType::SearchFieldRecentSearches,
72         WebsiteDataType::SessionStorage,
73         WebsiteDataType::WebSQLDatabases,
74     }));
75
76     ASSERT(RunLoop::isMain());
77
78     return dataTypes;
79 }
80
81 class OperatingDate {
82 public:
83     OperatingDate() = default;
84
85     static OperatingDate fromWallTime(WallTime time)
86     {
87         double ms = time.secondsSinceEpoch().milliseconds();
88         int year = msToYear(ms);
89         int yearDay = dayInYear(ms, year);
90         int month = monthFromDayInYear(yearDay, isLeapYear(year));
91         int monthDay = dayInMonthFromDayInYear(yearDay, isLeapYear(year));
92
93         return OperatingDate { year, month, monthDay };
94     }
95
96     static OperatingDate today()
97     {
98         return OperatingDate::fromWallTime(WallTime::now());
99     }
100
101     Seconds secondsSinceEpoch() const
102     {
103         return Seconds { dateToDaysFrom1970(m_year, m_month, m_monthDay) * secondsPerDay };
104     }
105
106     bool operator==(const OperatingDate& other) const
107     {
108         return m_monthDay == other.m_monthDay && m_month == other.m_month && m_year == other.m_year;
109     }
110
111     bool operator<(const OperatingDate& other) const
112     {
113         return secondsSinceEpoch() < other.secondsSinceEpoch();
114     }
115
116     bool operator<=(const OperatingDate& other) const
117     {
118         return secondsSinceEpoch() <= other.secondsSinceEpoch();
119     }
120
121 private:
122     OperatingDate(int year, int month, int monthDay)
123         : m_year(year)
124         , m_month(month)
125         , m_monthDay(monthDay)
126     { }
127
128     int m_year { 0 };
129     int m_month { 0 }; // [0, 11].
130     int m_monthDay { 0 }; // [1, 31].
131 };
132
133 static Vector<OperatingDate> mergeOperatingDates(const Vector<OperatingDate>& existingDates, Vector<OperatingDate>&& newDates)
134 {
135     if (existingDates.isEmpty())
136         return WTFMove(newDates);
137
138     Vector<OperatingDate> mergedDates(existingDates.size() + newDates.size());
139
140     // Merge the two sorted vectors of dates.
141     std::merge(existingDates.begin(), existingDates.end(), newDates.begin(), newDates.end(), mergedDates.begin());
142     // Remove duplicate dates.
143     removeRepeatedElements(mergedDates);
144
145     // Drop old dates until the Vector size reaches operatingDatesWindow.
146     while (mergedDates.size() > operatingDatesWindow)
147         mergedDates.remove(0);
148
149     return mergedDates;
150 }
151
152 WebResourceLoadStatisticsStore::WebResourceLoadStatisticsStore(const String& resourceLoadStatisticsDirectory, UpdateCookiePartitioningForDomainsHandler&& updateCookiePartitioningForDomainsHandler)
153     : m_statisticsQueue(WorkQueue::create("WebResourceLoadStatisticsStore Process Data Queue", WorkQueue::Type::Serial, WorkQueue::QOS::Utility))
154     , m_persistentStorage(*this, resourceLoadStatisticsDirectory)
155     , m_updateCookiePartitioningForDomainsHandler(WTFMove(updateCookiePartitioningForDomainsHandler))
156     , m_dailyTasksTimer(RunLoop::main(), this, &WebResourceLoadStatisticsStore::performDailyTasks)
157 {
158     ASSERT(RunLoop::isMain());
159
160 #if PLATFORM(COCOA)
161     registerUserDefaultsIfNeeded();
162 #endif
163
164     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
165         m_persistentStorage.initialize();
166         includeTodayAsOperatingDateIfNecessary();
167     });
168
169     m_statisticsQueue->dispatchAfter(5_s, [this, protectedThis = makeRef(*this)] {
170         if (m_parameters.shouldSubmitTelemetry)
171             WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
172     });
173
174     m_dailyTasksTimer.startRepeating(24_h);
175 }
176
177 WebResourceLoadStatisticsStore::~WebResourceLoadStatisticsStore()
178 {
179 }
180     
181 void WebResourceLoadStatisticsStore::removeDataRecords()
182 {
183     ASSERT(!RunLoop::isMain());
184     
185     if (!shouldRemoveDataRecords())
186         return;
187
188     auto prevalentResourceDomains = topPrivatelyControlledDomainsToRemoveWebsiteDataFor();
189     if (prevalentResourceDomains.isEmpty())
190         return;
191     
192     setDataRecordsBeingRemoved(true);
193
194     RunLoop::main().dispatch([prevalentResourceDomains = CrossThreadCopier<Vector<String>>::copy(prevalentResourceDomains), this, protectedThis = makeRef(*this)] () mutable {
195         WebProcessProxy::deleteWebsiteDataForTopPrivatelyControlledDomainsInAllPersistentDataStores(dataTypesToRemove(), WTFMove(prevalentResourceDomains), m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, [this, protectedThis = WTFMove(protectedThis)](const HashSet<String>& domainsWithDeletedWebsiteData) mutable {
196             m_statisticsQueue->dispatch([this, protectedThis = WTFMove(protectedThis), topDomains = CrossThreadCopier<HashSet<String>>::copy(domainsWithDeletedWebsiteData)] () mutable {
197                 for (auto& prevalentResourceDomain : topDomains) {
198                     auto& statistic = ensureResourceStatisticsForPrimaryDomain(prevalentResourceDomain);
199                     ++statistic.dataRecordsRemoved;
200                 }
201                 setDataRecordsBeingRemoved(false);
202             });
203         });
204     });
205 }
206
207 void WebResourceLoadStatisticsStore::scheduleStatisticsAndDataRecordsProcessing()
208 {
209     ASSERT(RunLoop::isMain());
210     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
211         processStatisticsAndDataRecords();
212     });
213 }
214
215 void WebResourceLoadStatisticsStore::processStatisticsAndDataRecords()
216 {
217     ASSERT(!RunLoop::isMain());
218
219     if (m_parameters.shouldClassifyResourcesBeforeDataRecordsRemoval) {
220         for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
221             if (!resourceStatistic.isPrevalentResource && m_resourceLoadStatisticsClassifier.hasPrevalentResourceCharacteristics(resourceStatistic))
222                 resourceStatistic.isPrevalentResource = true;
223         }
224     }
225     removeDataRecords();
226
227     pruneStatisticsIfNeeded();
228
229     if (m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned) {
230         RunLoop::main().dispatch([] {
231             WebProcessProxy::notifyPageStatisticsAndDataRecordsProcessed();
232         });
233     }
234
235     m_persistentStorage.scheduleOrWriteMemoryStore();
236 }
237
238 void WebResourceLoadStatisticsStore::resourceLoadStatisticsUpdated(Vector<WebCore::ResourceLoadStatistics>&& origins)
239 {
240     ASSERT(!RunLoop::isMain());
241
242     mergeStatistics(WTFMove(origins));
243     // Fire before processing statistics to propagate user interaction as fast as possible to the network process.
244     updateCookiePartitioning();
245     processStatisticsAndDataRecords();
246 }
247
248 void WebResourceLoadStatisticsStore::grandfatherExistingWebsiteData()
249 {
250     ASSERT(!RunLoop::isMain());
251
252     RunLoop::main().dispatch([this, protectedThis = makeRef(*this)] () mutable {
253         WebProcessProxy::topPrivatelyControlledDomainsWithWebsiteData(dataTypesToRemove(), m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, [this, protectedThis = WTFMove(protectedThis)] (HashSet<String>&& topPrivatelyControlledDomainsWithWebsiteData) mutable {
254             m_statisticsQueue->dispatch([this, protectedThis = WTFMove(protectedThis), topDomains = CrossThreadCopier<HashSet<String>>::copy(topPrivatelyControlledDomainsWithWebsiteData)] () mutable {
255                 for (auto& topPrivatelyControlledDomain : topDomains) {
256                     auto& statistic = ensureResourceStatisticsForPrimaryDomain(topPrivatelyControlledDomain);
257                     statistic.grandfathered = true;
258                 }
259                 m_endOfGrandfatheringTimestamp = WallTime::now() + m_parameters.grandfatheringTime;
260             });
261         });
262     });
263 }
264     
265 void WebResourceLoadStatisticsStore::processWillOpenConnection(WebProcessProxy&, IPC::Connection& connection)
266 {
267     connection.addWorkQueueMessageReceiver(Messages::WebResourceLoadStatisticsStore::messageReceiverName(), m_statisticsQueue.get(), this);
268 }
269
270 void WebResourceLoadStatisticsStore::processDidCloseConnection(WebProcessProxy&, IPC::Connection& connection)
271 {
272     connection.removeWorkQueueMessageReceiver(Messages::WebResourceLoadStatisticsStore::messageReceiverName());
273 }
274
275 void WebResourceLoadStatisticsStore::applicationWillTerminate()
276 {
277     m_persistentStorage.finishAllPendingWorkSynchronously();
278 }
279
280 void WebResourceLoadStatisticsStore::performDailyTasks()
281 {
282     ASSERT(RunLoop::isMain());
283
284     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
285         includeTodayAsOperatingDateIfNecessary();
286     });
287     if (m_parameters.shouldSubmitTelemetry)
288         submitTelemetry();
289 }
290
291 void WebResourceLoadStatisticsStore::submitTelemetry()
292 {
293     ASSERT(RunLoop::isMain());
294     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
295         WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
296     });
297 }
298
299 void WebResourceLoadStatisticsStore::logUserInteraction(const URL& url)
300 {
301     if (url.isBlankURL() || url.isEmpty())
302         return;
303
304     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
305         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
306         statistics.hadUserInteraction = true;
307         statistics.mostRecentUserInteractionTime = WallTime::now();
308
309         updateCookiePartitioningForDomains({ primaryDomain }, { }, ShouldClearFirst::No);
310     });
311 }
312
313 void WebResourceLoadStatisticsStore::clearUserInteraction(const URL& url)
314 {
315     if (url.isBlankURL() || url.isEmpty())
316         return;
317
318     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
319         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
320         statistics.hadUserInteraction = false;
321         statistics.mostRecentUserInteractionTime = { };
322     });
323 }
324
325 void WebResourceLoadStatisticsStore::hasHadUserInteraction(const URL& url, WTF::Function<void (bool)>&& completionHandler)
326 {
327     if (url.isBlankURL() || url.isEmpty()) {
328         completionHandler(false);
329         return;
330     }
331
332     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), completionHandler = WTFMove(completionHandler)] () mutable {
333         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
334         bool hadUserInteraction = mapEntry == m_resourceStatisticsMap.end() ? false: hasHadUnexpiredRecentUserInteraction(mapEntry->value);
335         RunLoop::main().dispatch([hadUserInteraction, completionHandler = WTFMove(completionHandler)] {
336             completionHandler(hadUserInteraction);
337         });
338     });
339 }
340
341 void WebResourceLoadStatisticsStore::setLastSeen(const URL& url, Seconds seconds)
342 {
343     if (url.isBlankURL() || url.isEmpty())
344         return;
345     
346     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), seconds] {
347         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
348         statistics.lastSeen = WallTime::fromRawSeconds(seconds.seconds());
349     });
350 }
351     
352 void WebResourceLoadStatisticsStore::setPrevalentResource(const URL& url)
353 {
354     if (url.isBlankURL() || url.isEmpty())
355         return;
356
357     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
358         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
359         statistics.isPrevalentResource = true;
360     });
361 }
362
363 void WebResourceLoadStatisticsStore::isPrevalentResource(const URL& url, WTF::Function<void (bool)>&& completionHandler)
364 {
365     if (url.isBlankURL() || url.isEmpty()) {
366         completionHandler(false);
367         return;
368     }
369
370     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), completionHandler = WTFMove(completionHandler)] () mutable {
371         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
372         bool isPrevalentResource = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource;
373         RunLoop::main().dispatch([isPrevalentResource, completionHandler = WTFMove(completionHandler)] {
374             completionHandler(isPrevalentResource);
375         });
376     });
377 }
378
379 void WebResourceLoadStatisticsStore::clearPrevalentResource(const URL& url)
380 {
381     if (url.isBlankURL() || url.isEmpty())
382         return;
383
384     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
385         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
386         statistics.isPrevalentResource = false;
387     });
388 }
389
390 void WebResourceLoadStatisticsStore::setGrandfathered(const URL& url, bool value)
391 {
392     if (url.isBlankURL() || url.isEmpty())
393         return;
394
395     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), value] {
396         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
397         statistics.grandfathered = value;
398     });
399 }
400
401 void WebResourceLoadStatisticsStore::isGrandfathered(const URL& url, WTF::Function<void (bool)>&& completionHandler)
402 {
403     if (url.isBlankURL() || url.isEmpty()) {
404         completionHandler(false);
405         return;
406     }
407
408     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler), primaryDomain = isolatedPrimaryDomain(url)] () mutable {
409         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
410         bool isGrandFathered = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.grandfathered;
411         RunLoop::main().dispatch([isGrandFathered, completionHandler = WTFMove(completionHandler)] {
412             completionHandler(isGrandFathered);
413         });
414     });
415 }
416
417 void WebResourceLoadStatisticsStore::setSubframeUnderTopFrameOrigin(const URL& subframe, const URL& topFrame)
418 {
419     if (subframe.isBlankURL() || subframe.isEmpty() || topFrame.isBlankURL() || topFrame.isEmpty())
420         return;
421
422     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryTopFrameDomain = isolatedPrimaryDomain(topFrame), primarySubFrameDomain = isolatedPrimaryDomain(subframe)] {
423         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubFrameDomain);
424         statistics.subframeUnderTopFrameOrigins.add(primaryTopFrameDomain);
425     });
426 }
427
428 void WebResourceLoadStatisticsStore::setSubresourceUnderTopFrameOrigin(const URL& subresource, const URL& topFrame)
429 {
430     if (subresource.isBlankURL() || subresource.isEmpty() || topFrame.isBlankURL() || topFrame.isEmpty())
431         return;
432
433     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryTopFrameDomain = isolatedPrimaryDomain(topFrame), primarySubresourceDomain = isolatedPrimaryDomain(subresource)] {
434         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
435         statistics.subresourceUnderTopFrameOrigins.add(primaryTopFrameDomain);
436     });
437 }
438
439 void WebResourceLoadStatisticsStore::setSubresourceUniqueRedirectTo(const URL& subresource, const URL& hostNameRedirectedTo)
440 {
441     if (subresource.isBlankURL() || subresource.isEmpty() || hostNameRedirectedTo.isBlankURL() || hostNameRedirectedTo.isEmpty())
442         return;
443
444     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryRedirectDomain = isolatedPrimaryDomain(hostNameRedirectedTo), primarySubresourceDomain = isolatedPrimaryDomain(subresource)] {
445         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
446         statistics.subresourceUniqueRedirectsTo.add(primaryRedirectDomain);
447     });
448 }
449
450 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningUpdate()
451 {
452     // Helper function used by testing system. Should only be called from the main thread.
453     ASSERT(RunLoop::isMain());
454
455     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
456         updateCookiePartitioning();
457     });
458 }
459
460 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningUpdateForDomains(const Vector<String>& domainsToRemove, const Vector<String>& domainsToAdd, ShouldClearFirst shouldClearFirst)
461 {
462     // Helper function used by testing system. Should only be called from the main thread.
463     ASSERT(RunLoop::isMain());
464     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), domainsToRemove = CrossThreadCopier<Vector<String>>::copy(domainsToRemove), domainsToAdd = CrossThreadCopier<Vector<String>>::copy(domainsToAdd), shouldClearFirst] {
465         updateCookiePartitioningForDomains(domainsToRemove, domainsToAdd, shouldClearFirst);
466     });
467 }
468
469 #if HAVE(CFNETWORK_STORAGE_PARTITIONING)
470 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningStateReset()
471 {
472     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
473         resetCookiePartitioningState();
474     });
475 }
476 #endif
477
478 void WebResourceLoadStatisticsStore::scheduleClearInMemory()
479 {
480     ASSERT(RunLoop::isMain());
481     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
482         clearInMemory();
483     });
484 }
485
486 void WebResourceLoadStatisticsStore::scheduleClearInMemoryAndPersistent()
487 {
488     ASSERT(RunLoop::isMain());
489     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
490         clearInMemory();
491         m_persistentStorage.clear();
492         grandfatherExistingWebsiteData();
493     });
494 }
495
496 void WebResourceLoadStatisticsStore::scheduleClearInMemoryAndPersistent(std::chrono::system_clock::time_point modifiedSince)
497 {
498     // For now, be conservative and clear everything regardless of modifiedSince.
499     UNUSED_PARAM(modifiedSince);
500     scheduleClearInMemoryAndPersistent();
501 }
502
503 void WebResourceLoadStatisticsStore::setTimeToLiveUserInteraction(Seconds seconds)
504 {
505     ASSERT(seconds >= 0_s);
506     m_parameters.timeToLiveUserInteraction = seconds;
507 }
508
509 void WebResourceLoadStatisticsStore::setTimeToLiveCookiePartitionFree(Seconds seconds)
510 {
511     ASSERT(seconds >= 0_s);
512     m_parameters.timeToLiveCookiePartitionFree = seconds;
513 }
514
515 void WebResourceLoadStatisticsStore::setMinimumTimeBetweenDataRecordsRemoval(Seconds seconds)
516 {
517     ASSERT(seconds >= 0_s);
518     m_parameters.minimumTimeBetweenDataRecordsRemoval = seconds;
519 }
520
521 void WebResourceLoadStatisticsStore::setGrandfatheringTime(Seconds seconds)
522 {
523     ASSERT(seconds >= 0_s);
524     m_parameters.grandfatheringTime = seconds;
525 }
526
527 bool WebResourceLoadStatisticsStore::shouldRemoveDataRecords() const
528 {
529     ASSERT(!RunLoop::isMain());
530     if (m_dataRecordsBeingRemoved)
531         return false;
532
533     return !m_lastTimeDataRecordsWereRemoved || MonotonicTime::now() >= (m_lastTimeDataRecordsWereRemoved + m_parameters.minimumTimeBetweenDataRecordsRemoval);
534 }
535
536 void WebResourceLoadStatisticsStore::setDataRecordsBeingRemoved(bool value)
537 {
538     ASSERT(!RunLoop::isMain());
539     m_dataRecordsBeingRemoved = value;
540     if (m_dataRecordsBeingRemoved)
541         m_lastTimeDataRecordsWereRemoved = MonotonicTime::now();
542 }
543
544 ResourceLoadStatistics& WebResourceLoadStatisticsStore::ensureResourceStatisticsForPrimaryDomain(const String& primaryDomain)
545 {
546     ASSERT(!RunLoop::isMain());
547     return m_resourceStatisticsMap.ensure(primaryDomain, [&primaryDomain] {
548         return ResourceLoadStatistics(primaryDomain);
549     }).iterator->value;
550 }
551
552 std::unique_ptr<KeyedEncoder> WebResourceLoadStatisticsStore::createEncoderFromData() const
553 {
554     ASSERT(!RunLoop::isMain());
555     auto encoder = KeyedEncoder::encoder();
556     encoder->encodeUInt32("version", statisticsModelVersion);
557     encoder->encodeDouble("endOfGrandfatheringTimestamp", m_endOfGrandfatheringTimestamp.secondsSinceEpoch().value());
558
559     encoder->encodeObjects("browsingStatistics", m_resourceStatisticsMap.begin(), m_resourceStatisticsMap.end(), [](KeyedEncoder& encoderInner, const auto& origin) {
560         origin.value.encode(encoderInner);
561     });
562
563     encoder->encodeObjects("operatingDates", m_operatingDates.begin(), m_operatingDates.end(), [](KeyedEncoder& encoderInner, OperatingDate date) {
564         encoderInner.encodeDouble("date", date.secondsSinceEpoch().value());
565     });
566
567     return encoder;
568 }
569
570 void WebResourceLoadStatisticsStore::mergeWithDataFromDecoder(KeyedDecoder& decoder)
571 {
572     ASSERT(!RunLoop::isMain());
573
574     unsigned versionOnDisk;
575     if (!decoder.decodeUInt32("version", versionOnDisk))
576         return;
577
578     if (versionOnDisk != statisticsModelVersion)
579         return;
580
581     double endOfGrandfatheringTimestamp;
582     if (decoder.decodeDouble("endOfGrandfatheringTimestamp", endOfGrandfatheringTimestamp))
583         m_endOfGrandfatheringTimestamp = WallTime::fromRawSeconds(endOfGrandfatheringTimestamp);
584     else
585         m_endOfGrandfatheringTimestamp = { };
586
587     Vector<ResourceLoadStatistics> loadedStatistics;
588     bool succeeded = decoder.decodeObjects("browsingStatistics", loadedStatistics, [](KeyedDecoder& decoderInner, ResourceLoadStatistics& statistics) {
589         return statistics.decode(decoderInner);
590     });
591
592     if (!succeeded)
593         return;
594
595     mergeStatistics(WTFMove(loadedStatistics));
596     updateCookiePartitioning();
597
598     Vector<OperatingDate> operatingDates;
599     succeeded = decoder.decodeObjects("operatingDates", operatingDates, [](KeyedDecoder& decoder, OperatingDate& date) {
600         double value;
601         if (!decoder.decodeDouble("date", value))
602             return false;
603
604         date = OperatingDate::fromWallTime(WallTime::fromRawSeconds(value));
605         return true;
606     });
607
608     if (!succeeded)
609         return;
610
611     m_operatingDates = mergeOperatingDates(m_operatingDates, WTFMove(operatingDates));
612 }
613
614 void WebResourceLoadStatisticsStore::clearInMemory()
615 {
616     ASSERT(!RunLoop::isMain());
617     m_resourceStatisticsMap.clear();
618     m_operatingDates.clear();
619
620     updateCookiePartitioningForDomains({ }, { }, ShouldClearFirst::Yes);
621 }
622
623 void WebResourceLoadStatisticsStore::mergeStatistics(Vector<ResourceLoadStatistics>&& statistics)
624 {
625     ASSERT(!RunLoop::isMain());
626     for (auto& statistic : statistics) {
627         auto result = m_resourceStatisticsMap.ensure(statistic.highLevelDomain, [&statistic] {
628             return WTFMove(statistic);
629         });
630         if (!result.isNewEntry)
631             result.iterator->value.merge(statistic);
632     }
633 }
634
635 inline bool WebResourceLoadStatisticsStore::shouldPartitionCookies(const ResourceLoadStatistics& statistic) const
636 {
637     return statistic.isPrevalentResource && (!statistic.hadUserInteraction || WallTime::now() > statistic.mostRecentUserInteractionTime + m_parameters.timeToLiveCookiePartitionFree);
638 }
639
640 void WebResourceLoadStatisticsStore::updateCookiePartitioning()
641 {
642     ASSERT(!RunLoop::isMain());
643
644     Vector<String> domainsToRemove;
645     Vector<String> domainsToAdd;
646     for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
647         bool shouldPartition = shouldPartitionCookies(resourceStatistic);
648         if (resourceStatistic.isMarkedForCookiePartitioning && !shouldPartition) {
649             resourceStatistic.isMarkedForCookiePartitioning = false;
650             domainsToRemove.append(resourceStatistic.highLevelDomain);
651         } else if (!resourceStatistic.isMarkedForCookiePartitioning && shouldPartition) {
652             resourceStatistic.isMarkedForCookiePartitioning = true;
653             domainsToAdd.append(resourceStatistic.highLevelDomain);
654         }
655     }
656
657     if (domainsToRemove.isEmpty() && domainsToAdd.isEmpty())
658         return;
659
660     RunLoop::main().dispatch([this, protectedThis = makeRef(*this), domainsToRemove = CrossThreadCopier<Vector<String>>::copy(domainsToRemove), domainsToAdd = CrossThreadCopier<Vector<String>>::copy(domainsToAdd)] () {
661         m_updateCookiePartitioningForDomainsHandler(domainsToRemove, domainsToAdd, ShouldClearFirst::No);
662     });
663 }
664
665 void WebResourceLoadStatisticsStore::updateCookiePartitioningForDomains(const Vector<String>& domainsToRemove, const Vector<String>& domainsToAdd, ShouldClearFirst shouldClearFirst)
666 {
667     ASSERT(!RunLoop::isMain());
668     if (domainsToRemove.isEmpty() && domainsToAdd.isEmpty())
669         return;
670
671     RunLoop::main().dispatch([this, shouldClearFirst, protectedThis = makeRef(*this), domainsToRemove = CrossThreadCopier<Vector<String>>::copy(domainsToRemove), domainsToAdd = CrossThreadCopier<Vector<String>>::copy(domainsToAdd)] () {
672         m_updateCookiePartitioningForDomainsHandler(domainsToRemove, domainsToAdd, shouldClearFirst);
673     });
674
675     if (shouldClearFirst == ShouldClearFirst::Yes)
676         resetCookiePartitioningState();
677     else {
678         for (auto& domain : domainsToRemove)
679             ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookiePartitioning = false;
680     }
681
682     for (auto& domain : domainsToAdd)
683         ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookiePartitioning = true;
684 }
685
686 void WebResourceLoadStatisticsStore::resetCookiePartitioningState()
687 {
688     ASSERT(!RunLoop::isMain());
689     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
690         resourceStatistic.isMarkedForCookiePartitioning = false;
691 }
692
693 void WebResourceLoadStatisticsStore::processStatistics(const WTF::Function<void (const ResourceLoadStatistics&)>& processFunction) const
694 {
695     ASSERT(!RunLoop::isMain());
696     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
697         processFunction(resourceStatistic);
698 }
699
700 bool WebResourceLoadStatisticsStore::hasHadUnexpiredRecentUserInteraction(ResourceLoadStatistics& resourceStatistic) const
701 {
702     if (resourceStatistic.hadUserInteraction && hasStatisticsExpired(resourceStatistic)) {
703         // Drop privacy sensitive data because we no longer need it.
704         // Set timestamp to 0 so that statistics merge will know
705         // it has been reset as opposed to its default -1.
706         resourceStatistic.mostRecentUserInteractionTime = { };
707         resourceStatistic.hadUserInteraction = false;
708     }
709
710     return resourceStatistic.hadUserInteraction;
711 }
712
713 Vector<String> WebResourceLoadStatisticsStore::topPrivatelyControlledDomainsToRemoveWebsiteDataFor()
714 {
715     ASSERT(!RunLoop::isMain());
716
717     bool shouldCheckForGrandfathering = m_endOfGrandfatheringTimestamp > WallTime::now();
718     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && m_endOfGrandfatheringTimestamp;
719
720     if (shouldClearGrandfathering)
721         m_endOfGrandfatheringTimestamp = { };
722
723     Vector<String> prevalentResources;
724     for (auto& statistic : m_resourceStatisticsMap.values()) {
725         if (statistic.isPrevalentResource && !hasHadUnexpiredRecentUserInteraction(statistic) && (!shouldCheckForGrandfathering || !statistic.grandfathered))
726             prevalentResources.append(statistic.highLevelDomain);
727
728         if (shouldClearGrandfathering && statistic.grandfathered)
729             statistic.grandfathered = false;
730     }
731
732     return prevalentResources;
733 }
734
735 void WebResourceLoadStatisticsStore::includeTodayAsOperatingDateIfNecessary()
736 {
737     ASSERT(!RunLoop::isMain());
738
739     auto today = OperatingDate::today();
740     if (!m_operatingDates.isEmpty() && today <= m_operatingDates.last())
741         return;
742
743     while (m_operatingDates.size() >= operatingDatesWindow)
744         m_operatingDates.remove(0);
745
746     m_operatingDates.append(today);
747 }
748
749 bool WebResourceLoadStatisticsStore::hasStatisticsExpired(const ResourceLoadStatistics& resourceStatistic) const
750 {
751     if (m_operatingDates.size() >= operatingDatesWindow) {
752         if (OperatingDate::fromWallTime(resourceStatistic.mostRecentUserInteractionTime) < m_operatingDates.first())
753             return true;
754     }
755
756     // If we don't meet the real criteria for an expired statistic, check the user setting for a tighter restriction (mainly for testing).
757     if (m_parameters.timeToLiveUserInteraction) {
758         if (WallTime::now() > resourceStatistic.mostRecentUserInteractionTime + m_parameters.timeToLiveUserInteraction.value())
759             return true;
760     }
761
762     return false;
763 }
764     
765 void WebResourceLoadStatisticsStore::setMaxStatisticsEntries(size_t maximumEntryCount)
766 {
767     m_parameters.maxStatisticsEntries = maximumEntryCount;
768 }
769     
770 void WebResourceLoadStatisticsStore::setPruneEntriesDownTo(size_t pruneTargetCount)
771 {
772     m_parameters.pruneEntriesDownTo = pruneTargetCount;
773 }
774     
775 struct StatisticsLastSeen {
776     String topPrivatelyOwnedDomain;
777     WallTime lastSeen;
778 };
779     
780 static void pruneResources(HashMap<String, WebCore::ResourceLoadStatistics>& statisticsMap, Vector<StatisticsLastSeen>& statisticsToPrune, size_t& numberOfEntriesToPrune)
781 {
782     if (statisticsToPrune.size() > numberOfEntriesToPrune) {
783         std::sort(statisticsToPrune.begin(), statisticsToPrune.end(), [](const StatisticsLastSeen& a, const StatisticsLastSeen& b) {
784             return a.lastSeen < b.lastSeen;
785         });
786     }
787
788     for (size_t i = 0, end = std::min(numberOfEntriesToPrune, statisticsToPrune.size()); i != end; ++i, --numberOfEntriesToPrune)
789         statisticsMap.remove(statisticsToPrune[i].topPrivatelyOwnedDomain);
790 }
791     
792 static unsigned computeImportance(const ResourceLoadStatistics& resourceStatistic)
793 {
794     unsigned importance = maxImportance;
795     if (!resourceStatistic.isPrevalentResource)
796         importance -= 1;
797     if (!resourceStatistic.hadUserInteraction)
798         importance -= 2;
799     return importance;
800 }
801     
802 void WebResourceLoadStatisticsStore::pruneStatisticsIfNeeded()
803 {
804     ASSERT(!RunLoop::isMain());
805     if (m_resourceStatisticsMap.size() <= m_parameters.maxStatisticsEntries)
806         return;
807
808     ASSERT(m_parameters.pruneEntriesDownTo <= m_parameters.maxStatisticsEntries);
809
810     size_t numberOfEntriesLeftToPrune = m_resourceStatisticsMap.size() - m_parameters.pruneEntriesDownTo;
811     ASSERT(numberOfEntriesLeftToPrune);
812     
813     Vector<StatisticsLastSeen> resourcesToPrunePerImportance[maxImportance + 1];
814     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
815         resourcesToPrunePerImportance[computeImportance(resourceStatistic)].append({ resourceStatistic.highLevelDomain, resourceStatistic.lastSeen });
816     
817     for (unsigned importance = 0; numberOfEntriesLeftToPrune && importance <= maxImportance; ++importance)
818         pruneResources(m_resourceStatisticsMap, resourcesToPrunePerImportance[importance], numberOfEntriesLeftToPrune);
819
820     ASSERT(!numberOfEntriesLeftToPrune);
821 }
822
823 void WebResourceLoadStatisticsStore::resetParametersToDefaultValues()
824 {
825     m_parameters = { };
826 }
827     
828 } // namespace WebKit