Source/WebKit/UIProcess/WebResourceLoadStatisticsStore.cpp

   1 /*
   2  * Copyright (C) 2016-2018 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
  14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
  17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  23  * THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "WebResourceLoadStatisticsStore.h"
  28
  29 #include "Logging.h"
  30 #include "PluginProcessManager.h"
  31 #include "PluginProcessProxy.h"
  32 #include "WebProcessMessages.h"
  33 #include "WebProcessProxy.h"
  34 #include "WebResourceLoadStatisticsStoreMessages.h"
  35 #include "WebResourceLoadStatisticsTelemetry.h"
  36 #include "WebsiteDataFetchOption.h"
  37 #include "WebsiteDataStore.h"
  38 #include <WebCore/KeyedCoding.h>
  39 #include <WebCore/ResourceLoadStatistics.h>
  40 #include <wtf/CrossThreadCopier.h>
  41 #include <wtf/DateMath.h>
  42 #include <wtf/MathExtras.h>
  43 #include <wtf/NeverDestroyed.h>
  44
  45 using namespace WebCore;
  46
  47 namespace WebKit {
  48
  49 constexpr unsigned operatingDatesWindow { 30 };
  50 constexpr unsigned statisticsModelVersion { 10 };
  51 constexpr unsigned maxImportance { 3 };
  52
  53 template<typename T> static inline String isolatedPrimaryDomain(const T& value)
  54 {
  55     return ResourceLoadStatistics::primaryDomain(value).isolatedCopy();
  56 }
  57
  58 const OptionSet<WebsiteDataType>& WebResourceLoadStatisticsStore::monitoredDataTypes()
  59 {
  60     static NeverDestroyed<OptionSet<WebsiteDataType>> dataTypes(std::initializer_list<WebsiteDataType>({
  61         WebsiteDataType::Cookies,
  62         WebsiteDataType::DOMCache,
  63         WebsiteDataType::IndexedDBDatabases,
  64         WebsiteDataType::LocalStorage,
  65         WebsiteDataType::MediaKeys,
  66         WebsiteDataType::OfflineWebApplicationCache,
  67 #if ENABLE(NETSCAPE_PLUGIN_API)
  68         WebsiteDataType::PlugInData,
  69 #endif
  70         WebsiteDataType::SearchFieldRecentSearches,
  71         WebsiteDataType::SessionStorage,
  72 #if ENABLE(SERVICE_WORKER)
  73         WebsiteDataType::ServiceWorkerRegistrations,
  74 #endif
  75         WebsiteDataType::WebSQLDatabases,
  76     }));
  77
  78     ASSERT(RunLoop::isMain());
  79
  80     return dataTypes;
  81 }
  82
  83 class OperatingDate {
  84 public:
  85     OperatingDate() = default;
  86
  87     static OperatingDate fromWallTime(WallTime time)
  88     {
  89         double ms = time.secondsSinceEpoch().milliseconds();
  90         int year = msToYear(ms);
  91         int yearDay = dayInYear(ms, year);
  92         int month = monthFromDayInYear(yearDay, isLeapYear(year));
  93         int monthDay = dayInMonthFromDayInYear(yearDay, isLeapYear(year));
  94
  95         return OperatingDate { year, month, monthDay };
  96     }
  97
  98     static OperatingDate today()
  99     {
 100         return OperatingDate::fromWallTime(WallTime::now());
 101     }
 102
 103     Seconds secondsSinceEpoch() const
 104     {
 105         return Seconds { dateToDaysFrom1970(m_year, m_month, m_monthDay) * secondsPerDay };
 106     }
 107
 108     bool operator==(const OperatingDate& other) const
 109     {
 110         return m_monthDay == other.m_monthDay && m_month == other.m_month && m_year == other.m_year;
 111     }
 112
 113     bool operator<(const OperatingDate& other) const
 114     {
 115         return secondsSinceEpoch() < other.secondsSinceEpoch();
 116     }
 117
 118     bool operator<=(const OperatingDate& other) const
 119     {
 120         return secondsSinceEpoch() <= other.secondsSinceEpoch();
 121     }
 122
 123 private:
 124     OperatingDate(int year, int month, int monthDay)
 125         : m_year(year)
 126         , m_month(month)
 127         , m_monthDay(monthDay)
 128     { }
 129
 130     int m_year { 0 };
 131     int m_month { 0 }; // [0, 11].
 132     int m_monthDay { 0 }; // [1, 31].
 133 };
 134
 135 static Vector<OperatingDate> mergeOperatingDates(const Vector<OperatingDate>& existingDates, Vector<OperatingDate>&& newDates)
 136 {
 137     if (existingDates.isEmpty())
 138         return WTFMove(newDates);
 139
 140     Vector<OperatingDate> mergedDates(existingDates.size() + newDates.size());
 141
 142     // Merge the two sorted vectors of dates.
 143     std::merge(existingDates.begin(), existingDates.end(), newDates.begin(), newDates.end(), mergedDates.begin());
 144     // Remove duplicate dates.
 145     removeRepeatedElements(mergedDates);
 146
 147     // Drop old dates until the Vector size reaches operatingDatesWindow.
 148     while (mergedDates.size() > operatingDatesWindow)
 149         mergedDates.remove(0);
 150
 151     return mergedDates;
 152 }
 153
 154 WebResourceLoadStatisticsStore::WebResourceLoadStatisticsStore(const String& resourceLoadStatisticsDirectory, Function<void(const String&)>&& testingCallback, bool isEphemeral, UpdatePrevalentDomainsToPartitionOrBlockCookiesHandler&& updatePrevalentDomainsToPartitionOrBlockCookiesHandler, HasStorageAccessForFrameHandler&& hasStorageAccessForFrameHandler, GrantStorageAccessForFrameHandler&& grantStorageAccessForFrameHandler, RemovePrevalentDomainsHandler&& removeDomainsHandler)
 155     : m_statisticsQueue(WorkQueue::create("WebResourceLoadStatisticsStore Process Data Queue", WorkQueue::Type::Serial, WorkQueue::QOS::Utility))
 156     , m_persistentStorage(*this, resourceLoadStatisticsDirectory, isEphemeral ? ResourceLoadStatisticsPersistentStorage::IsReadOnly::Yes : ResourceLoadStatisticsPersistentStorage::IsReadOnly::No)
 157     , m_updatePrevalentDomainsToPartitionOrBlockCookiesHandler(WTFMove(updatePrevalentDomainsToPartitionOrBlockCookiesHandler))
 158     , m_hasStorageAccessForFrameHandler(WTFMove(hasStorageAccessForFrameHandler))
 159     , m_grantStorageAccessForFrameHandler(WTFMove(grantStorageAccessForFrameHandler))
 160     , m_removeDomainsHandler(WTFMove(removeDomainsHandler))
 161     , m_dailyTasksTimer(RunLoop::main(), this, &WebResourceLoadStatisticsStore::performDailyTasks)
 162     , m_statisticsTestingCallback(WTFMove(testingCallback))
 163 {
 164     ASSERT(RunLoop::isMain());
 165
 166 #if PLATFORM(COCOA)
 167     registerUserDefaultsIfNeeded();
 168 #endif
 169
 170     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
 171         m_persistentStorage.initialize();
 172         includeTodayAsOperatingDateIfNecessary();
 173     });
 174
 175     m_statisticsQueue->dispatchAfter(5_s, [this, protectedThis = makeRef(*this)] {
 176         if (m_parameters.shouldSubmitTelemetry)
 177             WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
 178     });
 179
 180     m_dailyTasksTimer.startRepeating(24_h);
 181 }
 182
 183 WebResourceLoadStatisticsStore::~WebResourceLoadStatisticsStore()
 184 {
 185     m_persistentStorage.finishAllPendingWorkSynchronously();
 186 }
 187
 188 void WebResourceLoadStatisticsStore::removeDataRecords()
 189 {
 190     ASSERT(!RunLoop::isMain());
 191
 192     if (!shouldRemoveDataRecords())
 193         return;
 194
 195 #if ENABLE(NETSCAPE_PLUGIN_API)
 196     m_activePluginTokens.clear();
 197     for (auto plugin : PluginProcessManager::singleton().pluginProcesses())
 198         m_activePluginTokens.add(plugin->pluginProcessToken());
 199 #endif
 200
 201     auto prevalentResourceDomains = topPrivatelyControlledDomainsToRemoveWebsiteDataFor();
 202     if (prevalentResourceDomains.isEmpty())
 203         return;
 204
 205     setDataRecordsBeingRemoved(true);
 206
 207     RunLoop::main().dispatch([prevalentResourceDomains = crossThreadCopy(prevalentResourceDomains), this, protectedThis = makeRef(*this)] () mutable {
 208         WebProcessProxy::deleteWebsiteDataForTopPrivatelyControlledDomainsInAllPersistentDataStores(WebResourceLoadStatisticsStore::monitoredDataTypes(), WTFMove(prevalentResourceDomains), m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, [this, protectedThis = WTFMove(protectedThis)](const HashSet<String>& domainsWithDeletedWebsiteData) mutable {
 209             m_statisticsQueue->dispatch([this, protectedThis = WTFMove(protectedThis), topDomains = crossThreadCopy(domainsWithDeletedWebsiteData)] () mutable {
 210                 for (auto& prevalentResourceDomain : topDomains) {
 211                     auto& statistic = ensureResourceStatisticsForPrimaryDomain(prevalentResourceDomain);
 212                     ++statistic.dataRecordsRemoved;
 213                 }
 214                 setDataRecordsBeingRemoved(false);
 215             });
 216         });
 217     });
 218 }
 219
 220 void WebResourceLoadStatisticsStore::scheduleStatisticsAndDataRecordsProcessing()
 221 {
 222     ASSERT(RunLoop::isMain());
 223     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
 224         processStatisticsAndDataRecords();
 225     });
 226 }
 227
 228 void WebResourceLoadStatisticsStore::processStatisticsAndDataRecords()
 229 {
 230     ASSERT(!RunLoop::isMain());
 231
 232     if (m_parameters.shouldClassifyResourcesBeforeDataRecordsRemoval) {
 233         for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
 234             if (!resourceStatistic.isPrevalentResource && m_resourceLoadStatisticsClassifier.hasPrevalentResourceCharacteristics(resourceStatistic))
 235                 resourceStatistic.isPrevalentResource = true;
 236         }
 237     }
 238     removeDataRecords();
 239
 240     pruneStatisticsIfNeeded();
 241
 242     if (m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned) {
 243         RunLoop::main().dispatch([] {
 244             WebProcessProxy::notifyPageStatisticsAndDataRecordsProcessed();
 245         });
 246     }
 247
 248     m_persistentStorage.scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::No);
 249 }
 250
 251 void WebResourceLoadStatisticsStore::resourceLoadStatisticsUpdated(Vector<WebCore::ResourceLoadStatistics>&& origins)
 252 {
 253     ASSERT(!RunLoop::isMain());
 254
 255     mergeStatistics(WTFMove(origins));
 256     // Fire before processing statistics to propagate user interaction as fast as possible to the network process.
 257     updateCookiePartitioning();
 258     processStatisticsAndDataRecords();
 259 }
 260
 261 void WebResourceLoadStatisticsStore::hasStorageAccess(String&& subFrameHost, String&& topFrameHost, uint64_t frameID, uint64_t pageID, WTF::CompletionHandler<void (bool)>&& callback)
 262 {
 263     ASSERT(subFrameHost != topFrameHost);
 264     ASSERT(RunLoop::isMain());
 265
 266     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), subFramePrimaryDomain = isolatedPrimaryDomain(subFrameHost), topFramePrimaryDomain = isolatedPrimaryDomain(topFrameHost), frameID, pageID, callback = WTFMove(callback)] () mutable {
 267
 268         auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
 269         if (shouldBlockCookies(subFrameStatistic)) {
 270             callback(false);
 271             return;
 272         }
 273
 274         if (!shouldPartitionCookies(subFrameStatistic)) {
 275             callback(true);
 276             return;
 277         }
 278
 279         m_hasStorageAccessForFrameHandler(subFramePrimaryDomain, topFramePrimaryDomain, frameID, pageID, WTFMove(callback));
 280     });
 281 }
 282
 283 void WebResourceLoadStatisticsStore::requestStorageAccess(String&& subFrameHost, String&& topFrameHost, uint64_t frameID, uint64_t pageID, WTF::CompletionHandler<void (bool)>&& callback)
 284 {
 285     ASSERT(subFrameHost != topFrameHost);
 286     ASSERT(RunLoop::isMain());
 287
 288     auto subFramePrimaryDomain = isolatedPrimaryDomain(subFrameHost);
 289     auto topFramePrimaryDomain = isolatedPrimaryDomain(topFrameHost);
 290     if (subFramePrimaryDomain == topFramePrimaryDomain) {
 291         callback(true);
 292         return;
 293     }
 294
 295     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), subFramePrimaryDomain = crossThreadCopy(subFramePrimaryDomain), topFramePrimaryDomain = crossThreadCopy(topFramePrimaryDomain), frameID, pageID, callback = WTFMove(callback)] () mutable {
 296
 297         auto& subFrameStatistic = ensureResourceStatisticsForPrimaryDomain(subFramePrimaryDomain);
 298         if (shouldBlockCookies(subFrameStatistic)) {
 299             callback(false);
 300             return;
 301         }
 302
 303         if (!shouldPartitionCookies(subFrameStatistic)) {
 304             callback(true);
 305             return;
 306         }
 307
 308         subFrameStatistic.timesAccessedAsFirstPartyDueToStorageAccessAPI++;
 309
 310         m_grantStorageAccessForFrameHandler(subFramePrimaryDomain, topFramePrimaryDomain, frameID, pageID, WTFMove(callback));
 311     });
 312 }
 313
 314 void WebResourceLoadStatisticsStore::grandfatherExistingWebsiteData(CompletionHandler<void()>&& callback)
 315 {
 316     ASSERT(!RunLoop::isMain());
 317
 318     RunLoop::main().dispatch([this, protectedThis = makeRef(*this), callback = WTFMove(callback)] () mutable {
 319         // FIXME: This method being a static call on WebProcessProxy is wrong.
 320         // It should be on the data store that this object belongs to.
 321         WebProcessProxy::topPrivatelyControlledDomainsWithWebsiteData(WebResourceLoadStatisticsStore::monitoredDataTypes(), m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, [this, protectedThis = WTFMove(protectedThis), callback = WTFMove(callback)] (HashSet<String>&& topPrivatelyControlledDomainsWithWebsiteData) mutable {
 322             m_statisticsQueue->dispatch([this, protectedThis = WTFMove(protectedThis), topDomains = crossThreadCopy(topPrivatelyControlledDomainsWithWebsiteData), callback = WTFMove(callback)] () mutable {
 323                 for (auto& topPrivatelyControlledDomain : topDomains) {
 324                     auto& statistic = ensureResourceStatisticsForPrimaryDomain(topPrivatelyControlledDomain);
 325                     statistic.grandfathered = true;
 326                 }
 327                 m_endOfGrandfatheringTimestamp = WallTime::now() + m_parameters.grandfatheringTime;
 328                 m_persistentStorage.scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::Yes);
 329                 callback();
 330                 logTestingEvent(ASCIILiteral("Grandfathered"));
 331             });
 332         });
 333     });
 334 }
 335
 336 void WebResourceLoadStatisticsStore::processWillOpenConnection(WebProcessProxy&, IPC::Connection& connection)
 337 {
 338     connection.addWorkQueueMessageReceiver(Messages::WebResourceLoadStatisticsStore::messageReceiverName(), m_statisticsQueue.get(), this);
 339 }
 340
 341 void WebResourceLoadStatisticsStore::processDidCloseConnection(WebProcessProxy&, IPC::Connection& connection)
 342 {
 343     connection.removeWorkQueueMessageReceiver(Messages::WebResourceLoadStatisticsStore::messageReceiverName());
 344 }
 345
 346 void WebResourceLoadStatisticsStore::applicationWillTerminate()
 347 {
 348     m_persistentStorage.finishAllPendingWorkSynchronously();
 349 }
 350
 351 void WebResourceLoadStatisticsStore::performDailyTasks()
 352 {
 353     ASSERT(RunLoop::isMain());
 354
 355     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
 356         includeTodayAsOperatingDateIfNecessary();
 357     });
 358     if (m_parameters.shouldSubmitTelemetry)
 359         submitTelemetry();
 360 }
 361
 362 void WebResourceLoadStatisticsStore::submitTelemetry()
 363 {
 364     ASSERT(RunLoop::isMain());
 365     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
 366         WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
 367     });
 368 }
 369
 370 void WebResourceLoadStatisticsStore::setResourceLoadStatisticsDebugMode(bool enable)
 371 {
 372     if (enable)
 373         setTimeToLiveCookiePartitionFree(30_s);
 374     else
 375         resetParametersToDefaultValues();
 376 }
 377
 378 void WebResourceLoadStatisticsStore::logUserInteraction(const URL& url)
 379 {
 380     if (url.isBlankURL() || url.isEmpty())
 381         return;
 382
 383     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
 384         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
 385         statistics.hadUserInteraction = true;
 386         statistics.mostRecentUserInteractionTime = WallTime::now();
 387
 388         if (statistics.isMarkedForCookiePartitioning || statistics.isMarkedForCookieBlocking)
 389             updateCookiePartitioningForDomains({ }, { }, { primaryDomain }, ShouldClearFirst::No);
 390     });
 391 }
 392
 393 void WebResourceLoadStatisticsStore::logNonRecentUserInteraction(const URL& url)
 394 {
 395     if (url.isBlankURL() || url.isEmpty())
 396         return;
 397
 398     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
 399         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
 400         statistics.hadUserInteraction = true;
 401         statistics.mostRecentUserInteractionTime = WallTime::now() - (m_parameters.timeToLiveCookiePartitionFree + Seconds::fromHours(1));
 402
 403         updateCookiePartitioningForDomains({ primaryDomain }, { }, { }, ShouldClearFirst::No);
 404     });
 405 }
 406
 407 void WebResourceLoadStatisticsStore::clearUserInteraction(const URL& url)
 408 {
 409     if (url.isBlankURL() || url.isEmpty())
 410         return;
 411
 412     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
 413         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
 414         statistics.hadUserInteraction = false;
 415         statistics.mostRecentUserInteractionTime = { };
 416     });
 417 }
 418
 419 void WebResourceLoadStatisticsStore::hasHadUserInteraction(const URL& url, WTF::Function<void (bool)>&& completionHandler)
 420 {
 421     if (url.isBlankURL() || url.isEmpty()) {
 422         completionHandler(false);
 423         return;
 424     }
 425
 426     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), completionHandler = WTFMove(completionHandler)] () mutable {
 427         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
 428         bool hadUserInteraction = mapEntry == m_resourceStatisticsMap.end() ? false: hasHadUnexpiredRecentUserInteraction(mapEntry->value);
 429         RunLoop::main().dispatch([hadUserInteraction, completionHandler = WTFMove(completionHandler)] {
 430             completionHandler(hadUserInteraction);
 431         });
 432     });
 433 }
 434
 435 void WebResourceLoadStatisticsStore::setLastSeen(const URL& url, Seconds seconds)
 436 {
 437     if (url.isBlankURL() || url.isEmpty())
 438         return;
 439
 440     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), seconds] {
 441         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
 442         statistics.lastSeen = WallTime::fromRawSeconds(seconds.seconds());
 443     });
 444 }
 445
 446 void WebResourceLoadStatisticsStore::setPrevalentResource(const URL& url)
 447 {
 448     if (url.isBlankURL() || url.isEmpty())
 449         return;
 450
 451     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
 452         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
 453         statistics.isPrevalentResource = true;
 454     });
 455 }
 456
 457 void WebResourceLoadStatisticsStore::isPrevalentResource(const URL& url, WTF::Function<void (bool)>&& completionHandler)
 458 {
 459     if (url.isBlankURL() || url.isEmpty()) {
 460         completionHandler(false);
 461         return;
 462     }
 463
 464     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), completionHandler = WTFMove(completionHandler)] () mutable {
 465         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
 466         bool isPrevalentResource = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource;
 467         RunLoop::main().dispatch([isPrevalentResource, completionHandler = WTFMove(completionHandler)] {
 468             completionHandler(isPrevalentResource);
 469         });
 470     });
 471 }
 472
 473 void WebResourceLoadStatisticsStore::isRegisteredAsSubFrameUnder(const URL& subFrame, const URL& topFrame, WTF::Function<void (bool)>&& completionHandler)
 474 {
 475     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), subFramePrimaryDomain = isolatedPrimaryDomain(subFrame), topFramePrimaryDomain = isolatedPrimaryDomain(topFrame), completionHandler = WTFMove(completionHandler)] () mutable {
 476         auto mapEntry = m_resourceStatisticsMap.find(subFramePrimaryDomain);
 477         bool isRegisteredAsSubFrameUnder = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subframeUnderTopFrameOrigins.contains(topFramePrimaryDomain);
 478         RunLoop::main().dispatch([isRegisteredAsSubFrameUnder, completionHandler = WTFMove(completionHandler)] {
 479             completionHandler(isRegisteredAsSubFrameUnder);
 480         });
 481     });
 482 }
 483
 484 void WebResourceLoadStatisticsStore::isRegisteredAsRedirectingTo(const URL& hostRedirectedFrom, const URL& hostRedirectedTo, WTF::Function<void (bool)>&& completionHandler)
 485 {
 486     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), hostRedirectedFromPrimaryDomain = isolatedPrimaryDomain(hostRedirectedFrom), hostRedirectedToPrimaryDomain = isolatedPrimaryDomain(hostRedirectedTo), completionHandler = WTFMove(completionHandler)] () mutable {
 487         auto mapEntry = m_resourceStatisticsMap.find(hostRedirectedFromPrimaryDomain);
 488         bool isRegisteredAsRedirectingTo = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.subresourceUniqueRedirectsTo.contains(hostRedirectedToPrimaryDomain);
 489         RunLoop::main().dispatch([isRegisteredAsRedirectingTo, completionHandler = WTFMove(completionHandler)] {
 490             completionHandler(isRegisteredAsRedirectingTo);
 491         });
 492     });
 493 }
 494
 495 void WebResourceLoadStatisticsStore::clearPrevalentResource(const URL& url)
 496 {
 497     if (url.isBlankURL() || url.isEmpty())
 498         return;
 499
 500     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
 501         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
 502         statistics.isPrevalentResource = false;
 503     });
 504 }
 505
 506 void WebResourceLoadStatisticsStore::setGrandfathered(const URL& url, bool value)
 507 {
 508     if (url.isBlankURL() || url.isEmpty())
 509         return;
 510
 511     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), value] {
 512         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
 513         statistics.grandfathered = value;
 514     });
 515 }
 516
 517 void WebResourceLoadStatisticsStore::isGrandfathered(const URL& url, WTF::Function<void (bool)>&& completionHandler)
 518 {
 519     if (url.isBlankURL() || url.isEmpty()) {
 520         completionHandler(false);
 521         return;
 522     }
 523
 524     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler), primaryDomain = isolatedPrimaryDomain(url)] () mutable {
 525         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
 526         bool isGrandFathered = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.grandfathered;
 527         RunLoop::main().dispatch([isGrandFathered, completionHandler = WTFMove(completionHandler)] {
 528             completionHandler(isGrandFathered);
 529         });
 530     });
 531 }
 532
 533 void WebResourceLoadStatisticsStore::setSubframeUnderTopFrameOrigin(const URL& subframe, const URL& topFrame)
 534 {
 535     if (subframe.isBlankURL() || subframe.isEmpty() || topFrame.isBlankURL() || topFrame.isEmpty())
 536         return;
 537
 538     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryTopFrameDomain = isolatedPrimaryDomain(topFrame), primarySubFrameDomain = isolatedPrimaryDomain(subframe)] {
 539         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubFrameDomain);
 540         statistics.subframeUnderTopFrameOrigins.add(primaryTopFrameDomain);
 541     });
 542 }
 543
 544 void WebResourceLoadStatisticsStore::setSubresourceUnderTopFrameOrigin(const URL& subresource, const URL& topFrame)
 545 {
 546     if (subresource.isBlankURL() || subresource.isEmpty() || topFrame.isBlankURL() || topFrame.isEmpty())
 547         return;
 548
 549     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryTopFrameDomain = isolatedPrimaryDomain(topFrame), primarySubresourceDomain = isolatedPrimaryDomain(subresource)] {
 550         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
 551         statistics.subresourceUnderTopFrameOrigins.add(primaryTopFrameDomain);
 552     });
 553 }
 554
 555 void WebResourceLoadStatisticsStore::setSubresourceUniqueRedirectTo(const URL& subresource, const URL& hostNameRedirectedTo)
 556 {
 557     if (subresource.isBlankURL() || subresource.isEmpty() || hostNameRedirectedTo.isBlankURL() || hostNameRedirectedTo.isEmpty())
 558         return;
 559
 560     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryRedirectDomain = isolatedPrimaryDomain(hostNameRedirectedTo), primarySubresourceDomain = isolatedPrimaryDomain(subresource)] {
 561         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
 562         statistics.subresourceUniqueRedirectsTo.add(primaryRedirectDomain);
 563     });
 564 }
 565
 566 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningUpdate()
 567 {
 568     // Helper function used by testing system. Should only be called from the main thread.
 569     ASSERT(RunLoop::isMain());
 570
 571     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
 572         updateCookiePartitioning();
 573     });
 574 }
 575
 576 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningUpdateForDomains(const Vector<String>& domainsToPartition, const Vector<String>& domainsToBlock, const Vector<String>& domainsToNeitherPartitionNorBlock, ShouldClearFirst shouldClearFirst)
 577 {
 578     // Helper function used by testing system. Should only be called from the main thread.
 579     ASSERT(RunLoop::isMain());
 580     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), domainsToPartition = crossThreadCopy(domainsToPartition), domainsToBlock = crossThreadCopy(domainsToBlock), domainsToNeitherPartitionNorBlock = crossThreadCopy(domainsToNeitherPartitionNorBlock), shouldClearFirst] {
 581         updateCookiePartitioningForDomains(domainsToPartition, domainsToBlock, domainsToNeitherPartitionNorBlock, shouldClearFirst);
 582     });
 583 }
 584
 585 void WebResourceLoadStatisticsStore::scheduleClearPartitioningStateForDomains(const Vector<String>& domains)
 586 {
 587     // Helper function used by testing system. Should only be called from the main thread.
 588     ASSERT(RunLoop::isMain());
 589     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), domains = crossThreadCopy(domains)] {
 590         clearPartitioningStateForDomains(domains);
 591     });
 592 }
 593
 594 #if HAVE(CFNETWORK_STORAGE_PARTITIONING)
 595 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningStateReset()
 596 {
 597     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
 598         resetCookiePartitioningState();
 599     });
 600 }
 601 #endif
 602
 603 void WebResourceLoadStatisticsStore::scheduleClearInMemory()
 604 {
 605     ASSERT(RunLoop::isMain());
 606     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
 607         clearInMemory();
 608     });
 609 }
 610
 611 void WebResourceLoadStatisticsStore::scheduleClearInMemoryAndPersistent(ShouldGrandfather shouldGrandfather, CompletionHandler<void()>&& callback)
 612 {
 613     ASSERT(RunLoop::isMain());
 614     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), shouldGrandfather, callback = WTFMove(callback)] () mutable {
 615         clearInMemory();
 616         m_persistentStorage.clear();
 617
 618         if (shouldGrandfather == ShouldGrandfather::Yes)
 619             grandfatherExistingWebsiteData([protectedThis = makeRef(*this), callback = WTFMove(callback)]() {
 620                 callback();
 621             });
 622         else {
 623             callback();
 624         }
 625     });
 626 }
 627
 628 void WebResourceLoadStatisticsStore::scheduleClearInMemoryAndPersistent(WallTime modifiedSince, ShouldGrandfather shouldGrandfather, CompletionHandler<void()>&& callback)
 629 {
 630     // For now, be conservative and clear everything regardless of modifiedSince.
 631     UNUSED_PARAM(modifiedSince);
 632     scheduleClearInMemoryAndPersistent(shouldGrandfather, WTFMove(callback));
 633 }
 634
 635 void WebResourceLoadStatisticsStore::setTimeToLiveUserInteraction(Seconds seconds)
 636 {
 637     ASSERT(seconds >= 0_s);
 638     m_parameters.timeToLiveUserInteraction = seconds;
 639 }
 640
 641 void WebResourceLoadStatisticsStore::setTimeToLiveCookiePartitionFree(Seconds seconds)
 642 {
 643     ASSERT(seconds >= 0_s);
 644     m_parameters.timeToLiveCookiePartitionFree = seconds;
 645 }
 646
 647 void WebResourceLoadStatisticsStore::setMinimumTimeBetweenDataRecordsRemoval(Seconds seconds)
 648 {
 649     ASSERT(seconds >= 0_s);
 650     m_parameters.minimumTimeBetweenDataRecordsRemoval = seconds;
 651 }
 652
 653 void WebResourceLoadStatisticsStore::setGrandfatheringTime(Seconds seconds)
 654 {
 655     ASSERT(seconds >= 0_s);
 656     m_parameters.grandfatheringTime = seconds;
 657 }
 658
 659 bool WebResourceLoadStatisticsStore::shouldRemoveDataRecords() const
 660 {
 661     ASSERT(!RunLoop::isMain());
 662     if (m_dataRecordsBeingRemoved)
 663         return false;
 664
 665 #if ENABLE(NETSCAPE_PLUGIN_API)
 666     for (auto plugin : PluginProcessManager::singleton().pluginProcesses()) {
 667         if (!m_activePluginTokens.contains(plugin->pluginProcessToken()))
 668             return true;
 669     }
 670 #endif
 671
 672     return !m_lastTimeDataRecordsWereRemoved || MonotonicTime::now() >= (m_lastTimeDataRecordsWereRemoved + m_parameters.minimumTimeBetweenDataRecordsRemoval);
 673 }
 674
 675 void WebResourceLoadStatisticsStore::setDataRecordsBeingRemoved(bool value)
 676 {
 677     ASSERT(!RunLoop::isMain());
 678     m_dataRecordsBeingRemoved = value;
 679     if (m_dataRecordsBeingRemoved)
 680         m_lastTimeDataRecordsWereRemoved = MonotonicTime::now();
 681 }
 682
 683 ResourceLoadStatistics& WebResourceLoadStatisticsStore::ensureResourceStatisticsForPrimaryDomain(const String& primaryDomain)
 684 {
 685     ASSERT(!RunLoop::isMain());
 686     return m_resourceStatisticsMap.ensure(primaryDomain, [&primaryDomain] {
 687         return ResourceLoadStatistics(primaryDomain);
 688     }).iterator->value;
 689 }
 690
 691 std::unique_ptr<KeyedEncoder> WebResourceLoadStatisticsStore::createEncoderFromData() const
 692 {
 693     ASSERT(!RunLoop::isMain());
 694     auto encoder = KeyedEncoder::encoder();
 695     encoder->encodeUInt32("version", statisticsModelVersion);
 696     encoder->encodeDouble("endOfGrandfatheringTimestamp", m_endOfGrandfatheringTimestamp.secondsSinceEpoch().value());
 697
 698     encoder->encodeObjects("browsingStatistics", m_resourceStatisticsMap.begin(), m_resourceStatisticsMap.end(), [](KeyedEncoder& encoderInner, const auto& origin) {
 699         origin.value.encode(encoderInner);
 700     });
 701
 702     encoder->encodeObjects("operatingDates", m_operatingDates.begin(), m_operatingDates.end(), [](KeyedEncoder& encoderInner, OperatingDate date) {
 703         encoderInner.encodeDouble("date", date.secondsSinceEpoch().value());
 704     });
 705
 706     return encoder;
 707 }
 708
 709 void WebResourceLoadStatisticsStore::mergeWithDataFromDecoder(KeyedDecoder& decoder)
 710 {
 711     ASSERT(!RunLoop::isMain());
 712
 713     unsigned versionOnDisk;
 714     if (!decoder.decodeUInt32("version", versionOnDisk))
 715         return;
 716
 717     if (versionOnDisk != statisticsModelVersion)
 718         return;
 719
 720     double endOfGrandfatheringTimestamp;
 721     if (decoder.decodeDouble("endOfGrandfatheringTimestamp", endOfGrandfatheringTimestamp))
 722         m_endOfGrandfatheringTimestamp = WallTime::fromRawSeconds(endOfGrandfatheringTimestamp);
 723     else
 724         m_endOfGrandfatheringTimestamp = { };
 725
 726     Vector<ResourceLoadStatistics> loadedStatistics;
 727     bool succeeded = decoder.decodeObjects("browsingStatistics", loadedStatistics, [](KeyedDecoder& decoderInner, ResourceLoadStatistics& statistics) {
 728         return statistics.decode(decoderInner);
 729     });
 730
 731     if (!succeeded)
 732         return;
 733
 734     mergeStatistics(WTFMove(loadedStatistics));
 735     updateCookiePartitioning();
 736
 737     Vector<OperatingDate> operatingDates;
 738     succeeded = decoder.decodeObjects("operatingDates", operatingDates, [](KeyedDecoder& decoder, OperatingDate& date) {
 739         double value;
 740         if (!decoder.decodeDouble("date", value))
 741             return false;
 742
 743         date = OperatingDate::fromWallTime(WallTime::fromRawSeconds(value));
 744         return true;
 745     });
 746
 747     if (!succeeded)
 748         return;
 749
 750     m_operatingDates = mergeOperatingDates(m_operatingDates, WTFMove(operatingDates));
 751 }
 752
 753 void WebResourceLoadStatisticsStore::clearInMemory()
 754 {
 755     ASSERT(!RunLoop::isMain());
 756     m_resourceStatisticsMap.clear();
 757     m_operatingDates.clear();
 758
 759     updateCookiePartitioningForDomains({ }, { }, { }, ShouldClearFirst::Yes);
 760 }
 761
 762 bool WebResourceLoadStatisticsStore::wasAccessedAsFirstPartyDueToUserInteraction(const ResourceLoadStatistics& current, const ResourceLoadStatistics& updated)
 763 {
 764     if (!current.hadUserInteraction && !updated.hadUserInteraction)
 765         return false;
 766
 767     auto mostRecentUserInteractionTime = std::max(current.mostRecentUserInteractionTime, updated.mostRecentUserInteractionTime);
 768
 769     return updated.lastSeen <= mostRecentUserInteractionTime + m_parameters.timeToLiveCookiePartitionFree;
 770 }
 771
 772 void WebResourceLoadStatisticsStore::mergeStatistics(Vector<ResourceLoadStatistics>&& statistics)
 773 {
 774     ASSERT(!RunLoop::isMain());
 775     for (auto& statistic : statistics) {
 776         auto result = m_resourceStatisticsMap.ensure(statistic.highLevelDomain, [&statistic] {
 777             return WTFMove(statistic);
 778         });
 779         if (!result.isNewEntry) {
 780             if (wasAccessedAsFirstPartyDueToUserInteraction(result.iterator->value, statistic))
 781                 result.iterator->value.timesAccessedAsFirstPartyDueToUserInteraction++;
 782             result.iterator->value.merge(statistic);
 783         }
 784     }
 785 }
 786
 787 bool WebResourceLoadStatisticsStore::shouldPartitionCookies(const ResourceLoadStatistics& statistic) const
 788 {
 789     return statistic.isPrevalentResource && statistic.hadUserInteraction && WallTime::now() > statistic.mostRecentUserInteractionTime + m_parameters.timeToLiveCookiePartitionFree;
 790 }
 791
 792 bool WebResourceLoadStatisticsStore::shouldBlockCookies(const ResourceLoadStatistics& statistic) const
 793 {
 794     return statistic.isPrevalentResource && !statistic.hadUserInteraction;
 795 }
 796
 797 void WebResourceLoadStatisticsStore::updateCookiePartitioning()
 798 {
 799     ASSERT(!RunLoop::isMain());
 800
 801     Vector<String> domainsToPartition;
 802     Vector<String> domainsToBlock;
 803     Vector<String> domainsToNeitherPartitionNorBlock;
 804     for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
 805
 806         bool shouldPartition = shouldPartitionCookies(resourceStatistic);
 807         bool shouldBlock = shouldBlockCookies(resourceStatistic);
 808
 809         if (shouldPartition && !resourceStatistic.isMarkedForCookiePartitioning) {
 810             domainsToPartition.append(resourceStatistic.highLevelDomain);
 811             resourceStatistic.isMarkedForCookiePartitioning = true;
 812         } else if (shouldBlock && !resourceStatistic.isMarkedForCookieBlocking) {
 813             domainsToBlock.append(resourceStatistic.highLevelDomain);
 814             resourceStatistic.isMarkedForCookieBlocking = true;
 815         } else if (!shouldPartition && !shouldBlock && resourceStatistic.isPrevalentResource) {
 816             domainsToNeitherPartitionNorBlock.append(resourceStatistic.highLevelDomain);
 817             resourceStatistic.isMarkedForCookiePartitioning = false;
 818             resourceStatistic.isMarkedForCookieBlocking = false;
 819         }
 820     }
 821
 822     if (domainsToPartition.isEmpty() && domainsToBlock.isEmpty() && domainsToNeitherPartitionNorBlock.isEmpty())
 823         return;
 824
 825     RunLoop::main().dispatch([this, protectedThis = makeRef(*this), domainsToPartition = crossThreadCopy(domainsToPartition), domainsToBlock = crossThreadCopy(domainsToBlock), domainsToNeitherPartitionNorBlock = crossThreadCopy(domainsToNeitherPartitionNorBlock)] () {
 826         m_updatePrevalentDomainsToPartitionOrBlockCookiesHandler(domainsToPartition, domainsToBlock, domainsToNeitherPartitionNorBlock, ShouldClearFirst::No);
 827     });
 828 }
 829
 830 void WebResourceLoadStatisticsStore::updateCookiePartitioningForDomains(const Vector<String>& domainsToPartition, const Vector<String>& domainsToBlock, const Vector<String>& domainsToNeitherPartitionNorBlock, ShouldClearFirst shouldClearFirst)
 831 {
 832     ASSERT(!RunLoop::isMain());
 833     if (domainsToPartition.isEmpty() && domainsToBlock.isEmpty() && domainsToNeitherPartitionNorBlock.isEmpty() && shouldClearFirst == ShouldClearFirst::No)
 834         return;
 835
 836     RunLoop::main().dispatch([this, shouldClearFirst, protectedThis = makeRef(*this), domainsToPartition = crossThreadCopy(domainsToPartition), domainsToBlock = crossThreadCopy(domainsToBlock), domainsToNeitherPartitionNorBlock = crossThreadCopy(domainsToNeitherPartitionNorBlock)] () {
 837         m_updatePrevalentDomainsToPartitionOrBlockCookiesHandler(domainsToPartition, domainsToBlock, domainsToNeitherPartitionNorBlock, shouldClearFirst);
 838     });
 839
 840     if (shouldClearFirst == ShouldClearFirst::Yes)
 841         resetCookiePartitioningState();
 842     else {
 843         for (auto& domain : domainsToNeitherPartitionNorBlock) {
 844             auto& statistic = ensureResourceStatisticsForPrimaryDomain(domain);
 845             statistic.isMarkedForCookiePartitioning = false;
 846             statistic.isMarkedForCookieBlocking = false;
 847         }
 848     }
 849
 850     for (auto& domain : domainsToPartition)
 851         ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookiePartitioning = true;
 852
 853     for (auto& domain : domainsToBlock)
 854         ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookieBlocking = true;
 855 }
 856
 857 void WebResourceLoadStatisticsStore::clearPartitioningStateForDomains(const Vector<String>& domains)
 858 {
 859     ASSERT(!RunLoop::isMain());
 860     if (domains.isEmpty())
 861         return;
 862
 863     RunLoop::main().dispatch([this, protectedThis = makeRef(*this), domains = crossThreadCopy(domains)] () {
 864         m_removeDomainsHandler(domains);
 865     });
 866
 867     for (auto& domain : domains) {
 868         auto& statistic = ensureResourceStatisticsForPrimaryDomain(domain);
 869         statistic.isMarkedForCookiePartitioning = false;
 870         statistic.isMarkedForCookieBlocking = false;
 871     }
 872 }
 873
 874 void WebResourceLoadStatisticsStore::resetCookiePartitioningState()
 875 {
 876     ASSERT(!RunLoop::isMain());
 877     for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
 878         resourceStatistic.isMarkedForCookiePartitioning = false;
 879         resourceStatistic.isMarkedForCookieBlocking = false;
 880     }
 881 }
 882
 883 void WebResourceLoadStatisticsStore::processStatistics(const WTF::Function<void (const ResourceLoadStatistics&)>& processFunction) const
 884 {
 885     ASSERT(!RunLoop::isMain());
 886     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
 887         processFunction(resourceStatistic);
 888 }
 889
 890 bool WebResourceLoadStatisticsStore::hasHadUnexpiredRecentUserInteraction(ResourceLoadStatistics& resourceStatistic) const
 891 {
 892     if (resourceStatistic.hadUserInteraction && hasStatisticsExpired(resourceStatistic)) {
 893         // Drop privacy sensitive data because we no longer need it.
 894         // Set timestamp to 0 so that statistics merge will know
 895         // it has been reset as opposed to its default -1.
 896         resourceStatistic.mostRecentUserInteractionTime = { };
 897         resourceStatistic.hadUserInteraction = false;
 898     }
 899
 900     return resourceStatistic.hadUserInteraction;
 901 }
 902
 903 Vector<String> WebResourceLoadStatisticsStore::topPrivatelyControlledDomainsToRemoveWebsiteDataFor()
 904 {
 905     ASSERT(!RunLoop::isMain());
 906
 907     bool shouldCheckForGrandfathering = m_endOfGrandfatheringTimestamp > WallTime::now();
 908     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && m_endOfGrandfatheringTimestamp;
 909
 910     if (shouldClearGrandfathering)
 911         m_endOfGrandfatheringTimestamp = { };
 912
 913     Vector<String> prevalentResources;
 914     for (auto& statistic : m_resourceStatisticsMap.values()) {
 915         if (statistic.isPrevalentResource && !hasHadUnexpiredRecentUserInteraction(statistic) && (!shouldCheckForGrandfathering || !statistic.grandfathered))
 916             prevalentResources.append(statistic.highLevelDomain);
 917
 918         if (shouldClearGrandfathering && statistic.grandfathered)
 919             statistic.grandfathered = false;
 920     }
 921
 922     return prevalentResources;
 923 }
 924
 925 void WebResourceLoadStatisticsStore::includeTodayAsOperatingDateIfNecessary()
 926 {
 927     ASSERT(!RunLoop::isMain());
 928
 929     auto today = OperatingDate::today();
 930     if (!m_operatingDates.isEmpty() && today <= m_operatingDates.last())
 931         return;
 932
 933     while (m_operatingDates.size() >= operatingDatesWindow)
 934         m_operatingDates.remove(0);
 935
 936     m_operatingDates.append(today);
 937 }
 938
 939 bool WebResourceLoadStatisticsStore::hasStatisticsExpired(const ResourceLoadStatistics& resourceStatistic) const
 940 {
 941     if (m_operatingDates.size() >= operatingDatesWindow) {
 942         if (OperatingDate::fromWallTime(resourceStatistic.mostRecentUserInteractionTime) < m_operatingDates.first())
 943             return true;
 944     }
 945
 946     // If we don't meet the real criteria for an expired statistic, check the user setting for a tighter restriction (mainly for testing).
 947     if (m_parameters.timeToLiveUserInteraction) {
 948         if (WallTime::now() > resourceStatistic.mostRecentUserInteractionTime + m_parameters.timeToLiveUserInteraction.value())
 949             return true;
 950     }
 951
 952     return false;
 953 }
 954
 955 void WebResourceLoadStatisticsStore::setMaxStatisticsEntries(size_t maximumEntryCount)
 956 {
 957     m_parameters.maxStatisticsEntries = maximumEntryCount;
 958 }
 959
 960 void WebResourceLoadStatisticsStore::setPruneEntriesDownTo(size_t pruneTargetCount)
 961 {
 962     m_parameters.pruneEntriesDownTo = pruneTargetCount;
 963 }
 964
 965 struct StatisticsLastSeen {
 966     String topPrivatelyOwnedDomain;
 967     WallTime lastSeen;
 968 };
 969
 970 static void pruneResources(HashMap<String, WebCore::ResourceLoadStatistics>& statisticsMap, Vector<StatisticsLastSeen>& statisticsToPrune, size_t& numberOfEntriesToPrune)
 971 {
 972     if (statisticsToPrune.size() > numberOfEntriesToPrune) {
 973         std::sort(statisticsToPrune.begin(), statisticsToPrune.end(), [](const StatisticsLastSeen& a, const StatisticsLastSeen& b) {
 974             return a.lastSeen < b.lastSeen;
 975         });
 976     }
 977
 978     for (size_t i = 0, end = std::min(numberOfEntriesToPrune, statisticsToPrune.size()); i != end; ++i, --numberOfEntriesToPrune)
 979         statisticsMap.remove(statisticsToPrune[i].topPrivatelyOwnedDomain);
 980 }
 981
 982 static unsigned computeImportance(const ResourceLoadStatistics& resourceStatistic)
 983 {
 984     unsigned importance = maxImportance;
 985     if (!resourceStatistic.isPrevalentResource)
 986         importance -= 1;
 987     if (!resourceStatistic.hadUserInteraction)
 988         importance -= 2;
 989     return importance;
 990 }
 991
 992 void WebResourceLoadStatisticsStore::pruneStatisticsIfNeeded()
 993 {
 994     ASSERT(!RunLoop::isMain());
 995     if (m_resourceStatisticsMap.size() <= m_parameters.maxStatisticsEntries)
 996         return;
 997
 998     ASSERT(m_parameters.pruneEntriesDownTo <= m_parameters.maxStatisticsEntries);
 999
1000     size_t numberOfEntriesLeftToPrune = m_resourceStatisticsMap.size() - m_parameters.pruneEntriesDownTo;
1001     ASSERT(numberOfEntriesLeftToPrune);
1002
1003     Vector<StatisticsLastSeen> resourcesToPrunePerImportance[maxImportance + 1];
1004     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
1005         resourcesToPrunePerImportance[computeImportance(resourceStatistic)].append({ resourceStatistic.highLevelDomain, resourceStatistic.lastSeen });
1006
1007     for (unsigned importance = 0; numberOfEntriesLeftToPrune && importance <= maxImportance; ++importance)
1008         pruneResources(m_resourceStatisticsMap, resourcesToPrunePerImportance[importance], numberOfEntriesLeftToPrune);
1009
1010     ASSERT(!numberOfEntriesLeftToPrune);
1011 }
1012
1013 void WebResourceLoadStatisticsStore::resetParametersToDefaultValues()
1014 {
1015     m_parameters = { };
1016 }
1017
1018 void WebResourceLoadStatisticsStore::logTestingEvent(const String& event)
1019 {
1020     if (!m_statisticsTestingCallback)
1021         return;
1022
1023     if (RunLoop::isMain())
1024         m_statisticsTestingCallback(event);
1025     else {
1026         RunLoop::main().dispatch([this, protectedThis = makeRef(*this), event = event.isolatedCopy()] {
1027             if (m_statisticsTestingCallback)
1028                 m_statisticsTestingCallback(event);
1029         });
1030     }
1031 }
1032
1033 } // namespace WebKit