Source/WebKit/UIProcess/WebResourceLoadStatisticsStore.cpp

   1 /*
   2  * Copyright (C) 2016-2017 Apple Inc. All rights reserved.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions
   6  * are met:
   7  * 1. Redistributions of source code must retain the above copyright
   8  *    notice, this list of conditions and the following disclaimer.
   9  * 2. Redistributions in binary form must reproduce the above copyright
  10  *    notice, this list of conditions and the following disclaimer in the
  11  *    documentation and/or other materials provided with the distribution.
  12  *
  13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
  14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
  17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  23  * THE POSSIBILITY OF SUCH DAMAGE.
  24  */
  25
  26 #include "config.h"
  27 #include "WebResourceLoadStatisticsStore.h"
  28
  29 #include "Logging.h"
  30 #include "WebProcessMessages.h"
  31 #include "WebProcessProxy.h"
  32 #include "WebResourceLoadStatisticsStoreMessages.h"
  33 #include "WebResourceLoadStatisticsTelemetry.h"
  34 #include "WebsiteDataFetchOption.h"
  35 #include "WebsiteDataStore.h"
  36 #include <WebCore/KeyedCoding.h>
  37 #include <WebCore/ResourceLoadStatistics.h>
  38 #include <wtf/CrossThreadCopier.h>
  39 #include <wtf/DateMath.h>
  40 #include <wtf/MathExtras.h>
  41 #include <wtf/NeverDestroyed.h>
  42
  43 using namespace WebCore;
  44
  45 namespace WebKit {
  46
  47 constexpr unsigned operatingDatesWindow { 30 };
  48 constexpr unsigned statisticsModelVersion { 9 };
  49 constexpr unsigned maxImportance { 3 };
  50
  51 template<typename T> static inline String isolatedPrimaryDomain(const T& value)
  52 {
  53     return ResourceLoadStatistics::primaryDomain(value).isolatedCopy();
  54 }
  55
  56 const OptionSet<WebsiteDataType>& WebResourceLoadStatisticsStore::monitoredDataTypes()
  57 {
  58     static NeverDestroyed<OptionSet<WebsiteDataType>> dataTypes(std::initializer_list<WebsiteDataType>({
  59         WebsiteDataType::Cookies,
  60         WebsiteDataType::IndexedDBDatabases,
  61         WebsiteDataType::LocalStorage,
  62         WebsiteDataType::MediaKeys,
  63         WebsiteDataType::OfflineWebApplicationCache,
  64 #if ENABLE(NETSCAPE_PLUGIN_API)
  65         WebsiteDataType::PlugInData,
  66 #endif
  67         WebsiteDataType::SearchFieldRecentSearches,
  68         WebsiteDataType::SessionStorage,
  69         WebsiteDataType::WebSQLDatabases,
  70     }));
  71
  72     ASSERT(RunLoop::isMain());
  73
  74     return dataTypes;
  75 }
  76
  77 class OperatingDate {
  78 public:
  79     OperatingDate() = default;
  80
  81     static OperatingDate fromWallTime(WallTime time)
  82     {
  83         double ms = time.secondsSinceEpoch().milliseconds();
  84         int year = msToYear(ms);
  85         int yearDay = dayInYear(ms, year);
  86         int month = monthFromDayInYear(yearDay, isLeapYear(year));
  87         int monthDay = dayInMonthFromDayInYear(yearDay, isLeapYear(year));
  88
  89         return OperatingDate { year, month, monthDay };
  90     }
  91
  92     static OperatingDate today()
  93     {
  94         return OperatingDate::fromWallTime(WallTime::now());
  95     }
  96
  97     Seconds secondsSinceEpoch() const
  98     {
  99         return Seconds { dateToDaysFrom1970(m_year, m_month, m_monthDay) * secondsPerDay };
 100     }
 101
 102     bool operator==(const OperatingDate& other) const
 103     {
 104         return m_monthDay == other.m_monthDay && m_month == other.m_month && m_year == other.m_year;
 105     }
 106
 107     bool operator<(const OperatingDate& other) const
 108     {
 109         return secondsSinceEpoch() < other.secondsSinceEpoch();
 110     }
 111
 112     bool operator<=(const OperatingDate& other) const
 113     {
 114         return secondsSinceEpoch() <= other.secondsSinceEpoch();
 115     }
 116
 117 private:
 118     OperatingDate(int year, int month, int monthDay)
 119         : m_year(year)
 120         , m_month(month)
 121         , m_monthDay(monthDay)
 122     { }
 123
 124     int m_year { 0 };
 125     int m_month { 0 }; // [0, 11].
 126     int m_monthDay { 0 }; // [1, 31].
 127 };
 128
 129 static Vector<OperatingDate> mergeOperatingDates(const Vector<OperatingDate>& existingDates, Vector<OperatingDate>&& newDates)
 130 {
 131     if (existingDates.isEmpty())
 132         return WTFMove(newDates);
 133
 134     Vector<OperatingDate> mergedDates(existingDates.size() + newDates.size());
 135
 136     // Merge the two sorted vectors of dates.
 137     std::merge(existingDates.begin(), existingDates.end(), newDates.begin(), newDates.end(), mergedDates.begin());
 138     // Remove duplicate dates.
 139     removeRepeatedElements(mergedDates);
 140
 141     // Drop old dates until the Vector size reaches operatingDatesWindow.
 142     while (mergedDates.size() > operatingDatesWindow)
 143         mergedDates.remove(0);
 144
 145     return mergedDates;
 146 }
 147
 148 WebResourceLoadStatisticsStore::WebResourceLoadStatisticsStore(const String& resourceLoadStatisticsDirectory, Function<void (const String&)>&& testingCallback, UpdatePrevalentDomainsWithAndWithoutInteractionHandler&& updatePrevalentDomainsWithAndWithoutInteractionHandler, RemovePrevalentDomainsHandler&& removeDomainsHandler)
 149     : m_statisticsQueue(WorkQueue::create("WebResourceLoadStatisticsStore Process Data Queue", WorkQueue::Type::Serial, WorkQueue::QOS::Utility))
 150     , m_persistentStorage(*this, resourceLoadStatisticsDirectory)
 151     , m_updatePrevalentDomainsWithAndWithoutInteractionHandler(WTFMove(updatePrevalentDomainsWithAndWithoutInteractionHandler))
 152     , m_removeDomainsHandler(WTFMove(removeDomainsHandler))
 153     , m_dailyTasksTimer(RunLoop::main(), this, &WebResourceLoadStatisticsStore::performDailyTasks)
 154     , m_statisticsTestingCallback(WTFMove(testingCallback))
 155 {
 156     ASSERT(RunLoop::isMain());
 157
 158 #if PLATFORM(COCOA)
 159     registerUserDefaultsIfNeeded();
 160 #endif
 161
 162     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
 163         m_persistentStorage.initialize();
 164         includeTodayAsOperatingDateIfNecessary();
 165     });
 166
 167     m_statisticsQueue->dispatchAfter(5_s, [this, protectedThis = makeRef(*this)] {
 168         if (m_parameters.shouldSubmitTelemetry)
 169             WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
 170     });
 171
 172     m_dailyTasksTimer.startRepeating(24_h);
 173 }
 174
 175 WebResourceLoadStatisticsStore::~WebResourceLoadStatisticsStore()
 176 {
 177 }
 178
 179 void WebResourceLoadStatisticsStore::removeDataRecords()
 180 {
 181     ASSERT(!RunLoop::isMain());
 182
 183     if (!shouldRemoveDataRecords())
 184         return;
 185
 186     auto prevalentResourceDomains = topPrivatelyControlledDomainsToRemoveWebsiteDataFor();
 187     if (prevalentResourceDomains.isEmpty())
 188         return;
 189
 190     setDataRecordsBeingRemoved(true);
 191
 192     RunLoop::main().dispatch([prevalentResourceDomains = crossThreadCopy(prevalentResourceDomains), this, protectedThis = makeRef(*this)] () mutable {
 193         WebProcessProxy::deleteWebsiteDataForTopPrivatelyControlledDomainsInAllPersistentDataStores(WebResourceLoadStatisticsStore::monitoredDataTypes(), WTFMove(prevalentResourceDomains), m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, [this, protectedThis = WTFMove(protectedThis)](const HashSet<String>& domainsWithDeletedWebsiteData) mutable {
 194             m_statisticsQueue->dispatch([this, protectedThis = WTFMove(protectedThis), topDomains = crossThreadCopy(domainsWithDeletedWebsiteData)] () mutable {
 195                 for (auto& prevalentResourceDomain : topDomains) {
 196                     auto& statistic = ensureResourceStatisticsForPrimaryDomain(prevalentResourceDomain);
 197                     ++statistic.dataRecordsRemoved;
 198                 }
 199                 setDataRecordsBeingRemoved(false);
 200             });
 201         });
 202     });
 203 }
 204
 205 void WebResourceLoadStatisticsStore::scheduleStatisticsAndDataRecordsProcessing()
 206 {
 207     ASSERT(RunLoop::isMain());
 208     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
 209         processStatisticsAndDataRecords();
 210     });
 211 }
 212
 213 void WebResourceLoadStatisticsStore::processStatisticsAndDataRecords()
 214 {
 215     ASSERT(!RunLoop::isMain());
 216
 217     if (m_parameters.shouldClassifyResourcesBeforeDataRecordsRemoval) {
 218         for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
 219             if (!resourceStatistic.isPrevalentResource && m_resourceLoadStatisticsClassifier.hasPrevalentResourceCharacteristics(resourceStatistic))
 220                 resourceStatistic.isPrevalentResource = true;
 221         }
 222     }
 223     removeDataRecords();
 224
 225     pruneStatisticsIfNeeded();
 226
 227     if (m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned) {
 228         RunLoop::main().dispatch([] {
 229             WebProcessProxy::notifyPageStatisticsAndDataRecordsProcessed();
 230         });
 231     }
 232
 233     m_persistentStorage.scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::No);
 234 }
 235
 236 void WebResourceLoadStatisticsStore::resourceLoadStatisticsUpdated(Vector<WebCore::ResourceLoadStatistics>&& origins)
 237 {
 238     ASSERT(!RunLoop::isMain());
 239
 240     mergeStatistics(WTFMove(origins));
 241     // Fire before processing statistics to propagate user interaction as fast as possible to the network process.
 242     updateCookiePartitioning();
 243     processStatisticsAndDataRecords();
 244 }
 245
 246 void WebResourceLoadStatisticsStore::grandfatherExistingWebsiteData()
 247 {
 248     ASSERT(!RunLoop::isMain());
 249
 250     RunLoop::main().dispatch([this, protectedThis = makeRef(*this)] () mutable {
 251         // FIXME: This method being a static call on WebProcessProxy is wrong.
 252         // It should be on the data store that this object belongs to.
 253         WebProcessProxy::topPrivatelyControlledDomainsWithWebsiteData(WebResourceLoadStatisticsStore::monitoredDataTypes(), m_parameters.shouldNotifyPagesWhenDataRecordsWereScanned, [this, protectedThis = WTFMove(protectedThis)] (HashSet<String>&& topPrivatelyControlledDomainsWithWebsiteData) mutable {
 254             m_statisticsQueue->dispatch([this, protectedThis = WTFMove(protectedThis), topDomains = crossThreadCopy(topPrivatelyControlledDomainsWithWebsiteData)] () mutable {
 255                 for (auto& topPrivatelyControlledDomain : topDomains) {
 256                     auto& statistic = ensureResourceStatisticsForPrimaryDomain(topPrivatelyControlledDomain);
 257                     statistic.grandfathered = true;
 258                 }
 259                 m_endOfGrandfatheringTimestamp = WallTime::now() + m_parameters.grandfatheringTime;
 260                 m_persistentStorage.scheduleOrWriteMemoryStore(ResourceLoadStatisticsPersistentStorage::ForceImmediateWrite::Yes);
 261             });
 262
 263             logTestingEvent(ASCIILiteral("Grandfathered"));
 264         });
 265     });
 266 }
 267
 268 void WebResourceLoadStatisticsStore::processWillOpenConnection(WebProcessProxy&, IPC::Connection& connection)
 269 {
 270     connection.addWorkQueueMessageReceiver(Messages::WebResourceLoadStatisticsStore::messageReceiverName(), m_statisticsQueue.get(), this);
 271 }
 272
 273 void WebResourceLoadStatisticsStore::processDidCloseConnection(WebProcessProxy&, IPC::Connection& connection)
 274 {
 275     connection.removeWorkQueueMessageReceiver(Messages::WebResourceLoadStatisticsStore::messageReceiverName());
 276 }
 277
 278 void WebResourceLoadStatisticsStore::applicationWillTerminate()
 279 {
 280     m_persistentStorage.finishAllPendingWorkSynchronously();
 281 }
 282
 283 void WebResourceLoadStatisticsStore::performDailyTasks()
 284 {
 285     ASSERT(RunLoop::isMain());
 286
 287     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
 288         includeTodayAsOperatingDateIfNecessary();
 289     });
 290     if (m_parameters.shouldSubmitTelemetry)
 291         submitTelemetry();
 292 }
 293
 294 void WebResourceLoadStatisticsStore::submitTelemetry()
 295 {
 296     ASSERT(RunLoop::isMain());
 297     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
 298         WebResourceLoadStatisticsTelemetry::calculateAndSubmit(*this);
 299     });
 300 }
 301
 302 void WebResourceLoadStatisticsStore::logUserInteraction(const URL& url)
 303 {
 304     if (url.isBlankURL() || url.isEmpty())
 305         return;
 306
 307     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
 308         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
 309         statistics.hadUserInteraction = true;
 310         statistics.mostRecentUserInteractionTime = WallTime::now();
 311
 312         updateCookiePartitioningForDomains({ primaryDomain }, { }, ShouldClearFirst::No);
 313     });
 314 }
 315
 316 void WebResourceLoadStatisticsStore::clearUserInteraction(const URL& url)
 317 {
 318     if (url.isBlankURL() || url.isEmpty())
 319         return;
 320
 321     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
 322         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
 323         statistics.hadUserInteraction = false;
 324         statistics.mostRecentUserInteractionTime = { };
 325     });
 326 }
 327
 328 void WebResourceLoadStatisticsStore::hasHadUserInteraction(const URL& url, WTF::Function<void (bool)>&& completionHandler)
 329 {
 330     if (url.isBlankURL() || url.isEmpty()) {
 331         completionHandler(false);
 332         return;
 333     }
 334
 335     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), completionHandler = WTFMove(completionHandler)] () mutable {
 336         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
 337         bool hadUserInteraction = mapEntry == m_resourceStatisticsMap.end() ? false: hasHadUnexpiredRecentUserInteraction(mapEntry->value);
 338         RunLoop::main().dispatch([hadUserInteraction, completionHandler = WTFMove(completionHandler)] {
 339             completionHandler(hadUserInteraction);
 340         });
 341     });
 342 }
 343
 344 void WebResourceLoadStatisticsStore::setLastSeen(const URL& url, Seconds seconds)
 345 {
 346     if (url.isBlankURL() || url.isEmpty())
 347         return;
 348
 349     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), seconds] {
 350         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
 351         statistics.lastSeen = WallTime::fromRawSeconds(seconds.seconds());
 352     });
 353 }
 354
 355 void WebResourceLoadStatisticsStore::setPrevalentResource(const URL& url)
 356 {
 357     if (url.isBlankURL() || url.isEmpty())
 358         return;
 359
 360     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
 361         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
 362         statistics.isPrevalentResource = true;
 363     });
 364 }
 365
 366 void WebResourceLoadStatisticsStore::isPrevalentResource(const URL& url, WTF::Function<void (bool)>&& completionHandler)
 367 {
 368     if (url.isBlankURL() || url.isEmpty()) {
 369         completionHandler(false);
 370         return;
 371     }
 372
 373     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), completionHandler = WTFMove(completionHandler)] () mutable {
 374         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
 375         bool isPrevalentResource = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.isPrevalentResource;
 376         RunLoop::main().dispatch([isPrevalentResource, completionHandler = WTFMove(completionHandler)] {
 377             completionHandler(isPrevalentResource);
 378         });
 379     });
 380 }
 381
 382 void WebResourceLoadStatisticsStore::clearPrevalentResource(const URL& url)
 383 {
 384     if (url.isBlankURL() || url.isEmpty())
 385         return;
 386
 387     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url)] {
 388         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
 389         statistics.isPrevalentResource = false;
 390     });
 391 }
 392
 393 void WebResourceLoadStatisticsStore::setGrandfathered(const URL& url, bool value)
 394 {
 395     if (url.isBlankURL() || url.isEmpty())
 396         return;
 397
 398     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryDomain = isolatedPrimaryDomain(url), value] {
 399         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primaryDomain);
 400         statistics.grandfathered = value;
 401     });
 402 }
 403
 404 void WebResourceLoadStatisticsStore::isGrandfathered(const URL& url, WTF::Function<void (bool)>&& completionHandler)
 405 {
 406     if (url.isBlankURL() || url.isEmpty()) {
 407         completionHandler(false);
 408         return;
 409     }
 410
 411     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), completionHandler = WTFMove(completionHandler), primaryDomain = isolatedPrimaryDomain(url)] () mutable {
 412         auto mapEntry = m_resourceStatisticsMap.find(primaryDomain);
 413         bool isGrandFathered = mapEntry == m_resourceStatisticsMap.end() ? false : mapEntry->value.grandfathered;
 414         RunLoop::main().dispatch([isGrandFathered, completionHandler = WTFMove(completionHandler)] {
 415             completionHandler(isGrandFathered);
 416         });
 417     });
 418 }
 419
 420 void WebResourceLoadStatisticsStore::setSubframeUnderTopFrameOrigin(const URL& subframe, const URL& topFrame)
 421 {
 422     if (subframe.isBlankURL() || subframe.isEmpty() || topFrame.isBlankURL() || topFrame.isEmpty())
 423         return;
 424
 425     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryTopFrameDomain = isolatedPrimaryDomain(topFrame), primarySubFrameDomain = isolatedPrimaryDomain(subframe)] {
 426         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubFrameDomain);
 427         statistics.subframeUnderTopFrameOrigins.add(primaryTopFrameDomain);
 428     });
 429 }
 430
 431 void WebResourceLoadStatisticsStore::setSubresourceUnderTopFrameOrigin(const URL& subresource, const URL& topFrame)
 432 {
 433     if (subresource.isBlankURL() || subresource.isEmpty() || topFrame.isBlankURL() || topFrame.isEmpty())
 434         return;
 435
 436     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryTopFrameDomain = isolatedPrimaryDomain(topFrame), primarySubresourceDomain = isolatedPrimaryDomain(subresource)] {
 437         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
 438         statistics.subresourceUnderTopFrameOrigins.add(primaryTopFrameDomain);
 439     });
 440 }
 441
 442 void WebResourceLoadStatisticsStore::setSubresourceUniqueRedirectTo(const URL& subresource, const URL& hostNameRedirectedTo)
 443 {
 444     if (subresource.isBlankURL() || subresource.isEmpty() || hostNameRedirectedTo.isBlankURL() || hostNameRedirectedTo.isEmpty())
 445         return;
 446
 447     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), primaryRedirectDomain = isolatedPrimaryDomain(hostNameRedirectedTo), primarySubresourceDomain = isolatedPrimaryDomain(subresource)] {
 448         auto& statistics = ensureResourceStatisticsForPrimaryDomain(primarySubresourceDomain);
 449         statistics.subresourceUniqueRedirectsTo.add(primaryRedirectDomain);
 450     });
 451 }
 452
 453 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningUpdate()
 454 {
 455     // Helper function used by testing system. Should only be called from the main thread.
 456     ASSERT(RunLoop::isMain());
 457
 458     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
 459         updateCookiePartitioning();
 460     });
 461 }
 462
 463 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningUpdateForDomains(const Vector<String>& domainsToRemove, const Vector<String>& domainsToAdd, ShouldClearFirst shouldClearFirst)
 464 {
 465     // Helper function used by testing system. Should only be called from the main thread.
 466     ASSERT(RunLoop::isMain());
 467     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), domainsToRemove = crossThreadCopy(domainsToRemove), domainsToAdd = crossThreadCopy(domainsToAdd), shouldClearFirst] {
 468         updateCookiePartitioningForDomains(domainsToRemove, domainsToAdd, shouldClearFirst);
 469     });
 470 }
 471
 472 void WebResourceLoadStatisticsStore::scheduleClearPartitioningStateForDomains(const Vector<String>& domains)
 473 {
 474     // Helper function used by testing system. Should only be called from the main thread.
 475     ASSERT(RunLoop::isMain());
 476     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), domains = crossThreadCopy(domains)] {
 477         clearPartitioningStateForDomains(domains);
 478     });
 479 }
 480
 481 #if HAVE(CFNETWORK_STORAGE_PARTITIONING)
 482 void WebResourceLoadStatisticsStore::scheduleCookiePartitioningStateReset()
 483 {
 484     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
 485         resetCookiePartitioningState();
 486     });
 487 }
 488 #endif
 489
 490 void WebResourceLoadStatisticsStore::scheduleClearInMemory()
 491 {
 492     ASSERT(RunLoop::isMain());
 493     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this)] {
 494         clearInMemory();
 495     });
 496 }
 497
 498 void WebResourceLoadStatisticsStore::scheduleClearInMemoryAndPersistent(ShouldGrandfather shouldGrandfather)
 499 {
 500     ASSERT(RunLoop::isMain());
 501     m_statisticsQueue->dispatch([this, protectedThis = makeRef(*this), shouldGrandfather] {
 502         clearInMemory();
 503         m_persistentStorage.clear();
 504
 505         if (shouldGrandfather == ShouldGrandfather::Yes)
 506             grandfatherExistingWebsiteData();
 507     });
 508 }
 509
 510 void WebResourceLoadStatisticsStore::scheduleClearInMemoryAndPersistent(std::chrono::system_clock::time_point modifiedSince, ShouldGrandfather shouldGrandfather)
 511 {
 512     // For now, be conservative and clear everything regardless of modifiedSince.
 513     UNUSED_PARAM(modifiedSince);
 514     scheduleClearInMemoryAndPersistent(shouldGrandfather);
 515 }
 516
 517 void WebResourceLoadStatisticsStore::setTimeToLiveUserInteraction(Seconds seconds)
 518 {
 519     ASSERT(seconds >= 0_s);
 520     m_parameters.timeToLiveUserInteraction = seconds;
 521 }
 522
 523 void WebResourceLoadStatisticsStore::setTimeToLiveCookiePartitionFree(Seconds seconds)
 524 {
 525     ASSERT(seconds >= 0_s);
 526     m_parameters.timeToLiveCookiePartitionFree = seconds;
 527 }
 528
 529 void WebResourceLoadStatisticsStore::setMinimumTimeBetweenDataRecordsRemoval(Seconds seconds)
 530 {
 531     ASSERT(seconds >= 0_s);
 532     m_parameters.minimumTimeBetweenDataRecordsRemoval = seconds;
 533 }
 534
 535 void WebResourceLoadStatisticsStore::setGrandfatheringTime(Seconds seconds)
 536 {
 537     ASSERT(seconds >= 0_s);
 538     m_parameters.grandfatheringTime = seconds;
 539 }
 540
 541 bool WebResourceLoadStatisticsStore::shouldRemoveDataRecords() const
 542 {
 543     ASSERT(!RunLoop::isMain());
 544     if (m_dataRecordsBeingRemoved)
 545         return false;
 546
 547     return !m_lastTimeDataRecordsWereRemoved || MonotonicTime::now() >= (m_lastTimeDataRecordsWereRemoved + m_parameters.minimumTimeBetweenDataRecordsRemoval);
 548 }
 549
 550 void WebResourceLoadStatisticsStore::setDataRecordsBeingRemoved(bool value)
 551 {
 552     ASSERT(!RunLoop::isMain());
 553     m_dataRecordsBeingRemoved = value;
 554     if (m_dataRecordsBeingRemoved)
 555         m_lastTimeDataRecordsWereRemoved = MonotonicTime::now();
 556 }
 557
 558 ResourceLoadStatistics& WebResourceLoadStatisticsStore::ensureResourceStatisticsForPrimaryDomain(const String& primaryDomain)
 559 {
 560     ASSERT(!RunLoop::isMain());
 561     return m_resourceStatisticsMap.ensure(primaryDomain, [&primaryDomain] {
 562         return ResourceLoadStatistics(primaryDomain);
 563     }).iterator->value;
 564 }
 565
 566 std::unique_ptr<KeyedEncoder> WebResourceLoadStatisticsStore::createEncoderFromData() const
 567 {
 568     ASSERT(!RunLoop::isMain());
 569     auto encoder = KeyedEncoder::encoder();
 570     encoder->encodeUInt32("version", statisticsModelVersion);
 571     encoder->encodeDouble("endOfGrandfatheringTimestamp", m_endOfGrandfatheringTimestamp.secondsSinceEpoch().value());
 572
 573     encoder->encodeObjects("browsingStatistics", m_resourceStatisticsMap.begin(), m_resourceStatisticsMap.end(), [](KeyedEncoder& encoderInner, const auto& origin) {
 574         origin.value.encode(encoderInner);
 575     });
 576
 577     encoder->encodeObjects("operatingDates", m_operatingDates.begin(), m_operatingDates.end(), [](KeyedEncoder& encoderInner, OperatingDate date) {
 578         encoderInner.encodeDouble("date", date.secondsSinceEpoch().value());
 579     });
 580
 581     return encoder;
 582 }
 583
 584 void WebResourceLoadStatisticsStore::mergeWithDataFromDecoder(KeyedDecoder& decoder)
 585 {
 586     ASSERT(!RunLoop::isMain());
 587
 588     unsigned versionOnDisk;
 589     if (!decoder.decodeUInt32("version", versionOnDisk))
 590         return;
 591
 592     if (versionOnDisk != statisticsModelVersion)
 593         return;
 594
 595     double endOfGrandfatheringTimestamp;
 596     if (decoder.decodeDouble("endOfGrandfatheringTimestamp", endOfGrandfatheringTimestamp))
 597         m_endOfGrandfatheringTimestamp = WallTime::fromRawSeconds(endOfGrandfatheringTimestamp);
 598     else
 599         m_endOfGrandfatheringTimestamp = { };
 600
 601     Vector<ResourceLoadStatistics> loadedStatistics;
 602     bool succeeded = decoder.decodeObjects("browsingStatistics", loadedStatistics, [](KeyedDecoder& decoderInner, ResourceLoadStatistics& statistics) {
 603         return statistics.decode(decoderInner);
 604     });
 605
 606     if (!succeeded)
 607         return;
 608
 609     mergeStatistics(WTFMove(loadedStatistics));
 610     updateCookiePartitioning();
 611
 612     Vector<OperatingDate> operatingDates;
 613     succeeded = decoder.decodeObjects("operatingDates", operatingDates, [](KeyedDecoder& decoder, OperatingDate& date) {
 614         double value;
 615         if (!decoder.decodeDouble("date", value))
 616             return false;
 617
 618         date = OperatingDate::fromWallTime(WallTime::fromRawSeconds(value));
 619         return true;
 620     });
 621
 622     if (!succeeded)
 623         return;
 624
 625     m_operatingDates = mergeOperatingDates(m_operatingDates, WTFMove(operatingDates));
 626 }
 627
 628 void WebResourceLoadStatisticsStore::clearInMemory()
 629 {
 630     ASSERT(!RunLoop::isMain());
 631     m_resourceStatisticsMap.clear();
 632     m_operatingDates.clear();
 633
 634     updateCookiePartitioningForDomains({ }, { }, ShouldClearFirst::Yes);
 635 }
 636
 637 void WebResourceLoadStatisticsStore::mergeStatistics(Vector<ResourceLoadStatistics>&& statistics)
 638 {
 639     ASSERT(!RunLoop::isMain());
 640     for (auto& statistic : statistics) {
 641         auto result = m_resourceStatisticsMap.ensure(statistic.highLevelDomain, [&statistic] {
 642             return WTFMove(statistic);
 643         });
 644         if (!result.isNewEntry)
 645             result.iterator->value.merge(statistic);
 646     }
 647 }
 648
 649 inline bool WebResourceLoadStatisticsStore::shouldPartitionCookies(const ResourceLoadStatistics& statistic) const
 650 {
 651     return statistic.isPrevalentResource && (!statistic.hadUserInteraction || WallTime::now() > statistic.mostRecentUserInteractionTime + m_parameters.timeToLiveCookiePartitionFree);
 652 }
 653
 654 void WebResourceLoadStatisticsStore::updateCookiePartitioning()
 655 {
 656     ASSERT(!RunLoop::isMain());
 657
 658     Vector<String> domainsWithoutInteraction;
 659     Vector<String> domainsWithInteraction;
 660     for (auto& resourceStatistic : m_resourceStatisticsMap.values()) {
 661         bool shouldPartition = shouldPartitionCookies(resourceStatistic);
 662         if (resourceStatistic.isMarkedForCookiePartitioning && !shouldPartition) {
 663             resourceStatistic.isMarkedForCookiePartitioning = false;
 664             domainsWithInteraction.append(resourceStatistic.highLevelDomain);
 665         } else if (!resourceStatistic.isMarkedForCookiePartitioning && shouldPartition) {
 666             resourceStatistic.isMarkedForCookiePartitioning = true;
 667             domainsWithoutInteraction.append(resourceStatistic.highLevelDomain);
 668         }
 669     }
 670
 671     if (domainsWithInteraction.isEmpty() && domainsWithoutInteraction.isEmpty())
 672         return;
 673
 674     RunLoop::main().dispatch([this, protectedThis = makeRef(*this), domainsWithInteraction = crossThreadCopy(domainsWithInteraction), domainsWithoutInteraction = crossThreadCopy(domainsWithoutInteraction)] () {
 675         m_updatePrevalentDomainsWithAndWithoutInteractionHandler(domainsWithInteraction, domainsWithoutInteraction, ShouldClearFirst::No);
 676     });
 677 }
 678
 679 void WebResourceLoadStatisticsStore::updateCookiePartitioningForDomains(const Vector<String>& domainsWithInteraction, const Vector<String>& domainsWithoutInteraction, ShouldClearFirst shouldClearFirst)
 680 {
 681     ASSERT(!RunLoop::isMain());
 682     if (domainsWithInteraction.isEmpty() && domainsWithoutInteraction.isEmpty())
 683         return;
 684
 685     RunLoop::main().dispatch([this, shouldClearFirst, protectedThis = makeRef(*this), domainsWithInteraction = crossThreadCopy(domainsWithInteraction), domainsWithoutInteraction = crossThreadCopy(domainsWithoutInteraction)] () {
 686         m_updatePrevalentDomainsWithAndWithoutInteractionHandler(domainsWithInteraction, domainsWithoutInteraction, shouldClearFirst);
 687     });
 688
 689     if (shouldClearFirst == ShouldClearFirst::Yes)
 690         resetCookiePartitioningState();
 691     else {
 692         for (auto& domain : domainsWithInteraction)
 693             ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookiePartitioning = false;
 694     }
 695
 696     for (auto& domain : domainsWithoutInteraction)
 697         ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookiePartitioning = true;
 698 }
 699
 700 void WebResourceLoadStatisticsStore::clearPartitioningStateForDomains(const Vector<String>& domains)
 701 {
 702     ASSERT(!RunLoop::isMain());
 703     if (domains.isEmpty())
 704         return;
 705
 706     RunLoop::main().dispatch([this, protectedThis = makeRef(*this), domains = crossThreadCopy(domains)] () {
 707         m_removeDomainsHandler(domains);
 708     });
 709
 710     for (auto& domain : domains)
 711         ensureResourceStatisticsForPrimaryDomain(domain).isMarkedForCookiePartitioning = false;
 712 }
 713
 714 void WebResourceLoadStatisticsStore::resetCookiePartitioningState()
 715 {
 716     ASSERT(!RunLoop::isMain());
 717     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
 718         resourceStatistic.isMarkedForCookiePartitioning = false;
 719 }
 720
 721 void WebResourceLoadStatisticsStore::processStatistics(const WTF::Function<void (const ResourceLoadStatistics&)>& processFunction) const
 722 {
 723     ASSERT(!RunLoop::isMain());
 724     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
 725         processFunction(resourceStatistic);
 726 }
 727
 728 bool WebResourceLoadStatisticsStore::hasHadUnexpiredRecentUserInteraction(ResourceLoadStatistics& resourceStatistic) const
 729 {
 730     if (resourceStatistic.hadUserInteraction && hasStatisticsExpired(resourceStatistic)) {
 731         // Drop privacy sensitive data because we no longer need it.
 732         // Set timestamp to 0 so that statistics merge will know
 733         // it has been reset as opposed to its default -1.
 734         resourceStatistic.mostRecentUserInteractionTime = { };
 735         resourceStatistic.hadUserInteraction = false;
 736     }
 737
 738     return resourceStatistic.hadUserInteraction;
 739 }
 740
 741 Vector<String> WebResourceLoadStatisticsStore::topPrivatelyControlledDomainsToRemoveWebsiteDataFor()
 742 {
 743     ASSERT(!RunLoop::isMain());
 744
 745     bool shouldCheckForGrandfathering = m_endOfGrandfatheringTimestamp > WallTime::now();
 746     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && m_endOfGrandfatheringTimestamp;
 747
 748     if (shouldClearGrandfathering)
 749         m_endOfGrandfatheringTimestamp = { };
 750
 751     Vector<String> prevalentResources;
 752     for (auto& statistic : m_resourceStatisticsMap.values()) {
 753         if (statistic.isPrevalentResource && !hasHadUnexpiredRecentUserInteraction(statistic) && (!shouldCheckForGrandfathering || !statistic.grandfathered))
 754             prevalentResources.append(statistic.highLevelDomain);
 755
 756         if (shouldClearGrandfathering && statistic.grandfathered)
 757             statistic.grandfathered = false;
 758     }
 759
 760     return prevalentResources;
 761 }
 762
 763 void WebResourceLoadStatisticsStore::includeTodayAsOperatingDateIfNecessary()
 764 {
 765     ASSERT(!RunLoop::isMain());
 766
 767     auto today = OperatingDate::today();
 768     if (!m_operatingDates.isEmpty() && today <= m_operatingDates.last())
 769         return;
 770
 771     while (m_operatingDates.size() >= operatingDatesWindow)
 772         m_operatingDates.remove(0);
 773
 774     m_operatingDates.append(today);
 775 }
 776
 777 bool WebResourceLoadStatisticsStore::hasStatisticsExpired(const ResourceLoadStatistics& resourceStatistic) const
 778 {
 779     if (m_operatingDates.size() >= operatingDatesWindow) {
 780         if (OperatingDate::fromWallTime(resourceStatistic.mostRecentUserInteractionTime) < m_operatingDates.first())
 781             return true;
 782     }
 783
 784     // If we don't meet the real criteria for an expired statistic, check the user setting for a tighter restriction (mainly for testing).
 785     if (m_parameters.timeToLiveUserInteraction) {
 786         if (WallTime::now() > resourceStatistic.mostRecentUserInteractionTime + m_parameters.timeToLiveUserInteraction.value())
 787             return true;
 788     }
 789
 790     return false;
 791 }
 792
 793 void WebResourceLoadStatisticsStore::setMaxStatisticsEntries(size_t maximumEntryCount)
 794 {
 795     m_parameters.maxStatisticsEntries = maximumEntryCount;
 796 }
 797
 798 void WebResourceLoadStatisticsStore::setPruneEntriesDownTo(size_t pruneTargetCount)
 799 {
 800     m_parameters.pruneEntriesDownTo = pruneTargetCount;
 801 }
 802
 803 struct StatisticsLastSeen {
 804     String topPrivatelyOwnedDomain;
 805     WallTime lastSeen;
 806 };
 807
 808 static void pruneResources(HashMap<String, WebCore::ResourceLoadStatistics>& statisticsMap, Vector<StatisticsLastSeen>& statisticsToPrune, size_t& numberOfEntriesToPrune)
 809 {
 810     if (statisticsToPrune.size() > numberOfEntriesToPrune) {
 811         std::sort(statisticsToPrune.begin(), statisticsToPrune.end(), [](const StatisticsLastSeen& a, const StatisticsLastSeen& b) {
 812             return a.lastSeen < b.lastSeen;
 813         });
 814     }
 815
 816     for (size_t i = 0, end = std::min(numberOfEntriesToPrune, statisticsToPrune.size()); i != end; ++i, --numberOfEntriesToPrune)
 817         statisticsMap.remove(statisticsToPrune[i].topPrivatelyOwnedDomain);
 818 }
 819
 820 static unsigned computeImportance(const ResourceLoadStatistics& resourceStatistic)
 821 {
 822     unsigned importance = maxImportance;
 823     if (!resourceStatistic.isPrevalentResource)
 824         importance -= 1;
 825     if (!resourceStatistic.hadUserInteraction)
 826         importance -= 2;
 827     return importance;
 828 }
 829
 830 void WebResourceLoadStatisticsStore::pruneStatisticsIfNeeded()
 831 {
 832     ASSERT(!RunLoop::isMain());
 833     if (m_resourceStatisticsMap.size() <= m_parameters.maxStatisticsEntries)
 834         return;
 835
 836     ASSERT(m_parameters.pruneEntriesDownTo <= m_parameters.maxStatisticsEntries);
 837
 838     size_t numberOfEntriesLeftToPrune = m_resourceStatisticsMap.size() - m_parameters.pruneEntriesDownTo;
 839     ASSERT(numberOfEntriesLeftToPrune);
 840
 841     Vector<StatisticsLastSeen> resourcesToPrunePerImportance[maxImportance + 1];
 842     for (auto& resourceStatistic : m_resourceStatisticsMap.values())
 843         resourcesToPrunePerImportance[computeImportance(resourceStatistic)].append({ resourceStatistic.highLevelDomain, resourceStatistic.lastSeen });
 844
 845     for (unsigned importance = 0; numberOfEntriesLeftToPrune && importance <= maxImportance; ++importance)
 846         pruneResources(m_resourceStatisticsMap, resourcesToPrunePerImportance[importance], numberOfEntriesLeftToPrune);
 847
 848     ASSERT(!numberOfEntriesLeftToPrune);
 849 }
 850
 851 void WebResourceLoadStatisticsStore::resetParametersToDefaultValues()
 852 {
 853     m_parameters = { };
 854 }
 855
 856 void WebResourceLoadStatisticsStore::logTestingEvent(const String& event)
 857 {
 858     if (!m_statisticsTestingCallback)
 859         return;
 860
 861     if (RunLoop::isMain())
 862         m_statisticsTestingCallback(event);
 863     else {
 864         RunLoop::main().dispatch([this, protectedThis = makeRef(*this), event = event.isolatedCopy()] {
 865             if (m_statisticsTestingCallback)
 866                 m_statisticsTestingCallback(event);
 867         });
 868     }
 869 }
 870
 871 } // namespace WebKit