[WK2] Use a rolling 30-day uptime for processing statistics
authorbfulgham@apple.com <bfulgham@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Sat, 8 Jul 2017 00:07:13 +0000 (00:07 +0000)
committerbfulgham@apple.com <bfulgham@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Sat, 8 Jul 2017 00:07:13 +0000 (00:07 +0000)
https://bugs.webkit.org/show_bug.cgi?id=174235
<rdar://problem/33164381>

Reviewed by Chris Dumez.

Source/WebCore:

Add a KeyedDecoder specialization for Deque.

* platform/KeyedCoding.h:
(WebCore::KeyedDecoder::decodeObjects):

Source/WebKit2:

* UIProcess/API/Cocoa/WKWebsiteDataStore.cpp:
(WebKit::WKWebsiteDataStore::_resourceLoadStatisticsResetToConsistentState): Initialize time-to-live to zero by default.
* UIProcess/Storage/ResourceLoadStatisticsStore.cpp:
(WebKit::ResourceLoadStatisticsStore::createEncoderFromData): Write out vector
of operating dates.
(WebKit::ResourceLoadStatisticsStore::readDataFromDecoder): Read in vector of
operating dates.
(WebKit::ResourceLoadStatisticsStore::hasHadRecentUserInteraction): Check new
convenience method.
(WebKit::ResourceLoadStatisticsStore::markTodayAsOperatingDate): Added.
(WebKit::ResourceLoadStatisticsStore::hasStatisticsExpired): Added.
* UIProcess/Storage/ResourceLoadStatisticsStore.h:
* UIProcess/WebResourceLoadStatisticsStore.cpp:
(WebKit::WebResourceLoadStatisticsStore::readDataFromDiskIfNeeded): When reading
a new data file, update the current operating date (if needed).
(WebKit::WebResourceLoadStatisticsStore::handleDailyTasks): Roll uptime dates as
needed, then handle telemetry.
* UIProcess/WebResourceLoadStatisticsStore.h:

Source/WTF:

Modify Deque to allow it to be used in a template specialization in KeyedDecoder.

* wtf/Deque.h:

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@219274 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WTF/ChangeLog
Source/WTF/wtf/Deque.h
Source/WebCore/ChangeLog
Source/WebCore/platform/KeyedCoding.h
Source/WebKit2/ChangeLog
Source/WebKit2/UIProcess/API/Cocoa/WKWebsiteDataStore.mm
Source/WebKit2/UIProcess/Storage/ResourceLoadStatisticsStore.cpp
Source/WebKit2/UIProcess/Storage/ResourceLoadStatisticsStore.h
Source/WebKit2/UIProcess/WebResourceLoadStatisticsStore.cpp
Source/WebKit2/UIProcess/WebResourceLoadStatisticsStore.h

index 8531763..0211a20 100644 (file)
@@ -1,3 +1,15 @@
+2017-07-07  Brent Fulgham  <bfulgham@apple.com>
+
+        [WK2] Use a rolling 30-day uptime for processing statistics
+        https://bugs.webkit.org/show_bug.cgi?id=174235
+        <rdar://problem/33164381>
+
+        Reviewed by Chris Dumez.
+
+        Modify Deque to allow it to be used in a template specialization in KeyedDecoder.
+
+        * wtf/Deque.h:
+
 2017-07-07  Commit Queue  <commit-queue@webkit.org>
 
         Unreviewed, rolling out r219238, r219239, and r219241.
index 2c95085..252b081 100644 (file)
@@ -46,6 +46,8 @@ template<typename T, size_t inlineCapacity = 0>
 class Deque {
     WTF_MAKE_FAST_ALLOCATED;
 public:
+    typedef T ValueType;
+
     typedef DequeIterator<T, inlineCapacity> iterator;
     typedef DequeConstIterator<T, inlineCapacity> const_iterator;
     typedef std::reverse_iterator<iterator> reverse_iterator;
index 9473ab4..50dfdb4 100644 (file)
@@ -1,3 +1,16 @@
+2017-07-07  Brent Fulgham  <bfulgham@apple.com>
+
+        [WK2] Use a rolling 30-day uptime for processing statistics
+        https://bugs.webkit.org/show_bug.cgi?id=174235
+        <rdar://problem/33164381>
+
+        Reviewed by Chris Dumez.
+
+        Add a KeyedDecoder specialization for Deque.
+
+        * platform/KeyedCoding.h:
+        (WebCore::KeyedDecoder::decodeObjects):
+
 2017-07-07  Daniel Bates  <dabates@apple.com>
 
         [AppCache] Ignore fallback entries whose namespace is not prefixed with manifest path
index b11164f..3e1fd83 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2013 Apple Inc. All rights reserved.
+ * Copyright (C) 2013-2017 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -26,6 +26,7 @@
 #pragma once
 
 #include <functional>
+#include <wtf/Deque.h>
 #include <wtf/Forward.h>
 #include <wtf/Vector.h>
 
@@ -104,15 +105,15 @@ public:
         return result;
     }
 
-    template<typename T, typename F>
-    bool decodeObjects(const String& key, Vector<T>& objects, F&& function)
+    template<typename ContainerType, typename F>
+    bool decodeObjects(const String& key, ContainerType& objects, F&& function)
     {
         if (!beginArray(key))
             return false;
 
         bool result = true;
         while (beginArrayElement()) {
-            T element;
+            typename ContainerType::ValueType element;
             if (!function(*this, element)) {
                 result = false;
                 break;
index 55990e0..5a54ad9 100644 (file)
@@ -1,3 +1,30 @@
+2017-07-07  Brent Fulgham  <bfulgham@apple.com>
+
+        [WK2] Use a rolling 30-day uptime for processing statistics
+        https://bugs.webkit.org/show_bug.cgi?id=174235
+        <rdar://problem/33164381>
+
+        Reviewed by Chris Dumez.
+
+        * UIProcess/API/Cocoa/WKWebsiteDataStore.cpp:
+        (WebKit::WKWebsiteDataStore::_resourceLoadStatisticsResetToConsistentState): Initialize time-to-live to zero by default.
+        * UIProcess/Storage/ResourceLoadStatisticsStore.cpp:
+        (WebKit::ResourceLoadStatisticsStore::createEncoderFromData): Write out vector
+        of operating dates.
+        (WebKit::ResourceLoadStatisticsStore::readDataFromDecoder): Read in vector of
+        operating dates.
+        (WebKit::ResourceLoadStatisticsStore::hasHadRecentUserInteraction): Check new
+        convenience method.
+        (WebKit::ResourceLoadStatisticsStore::markTodayAsOperatingDate): Added.
+        (WebKit::ResourceLoadStatisticsStore::hasStatisticsExpired): Added.
+        * UIProcess/Storage/ResourceLoadStatisticsStore.h:
+        * UIProcess/WebResourceLoadStatisticsStore.cpp:
+        (WebKit::WebResourceLoadStatisticsStore::readDataFromDiskIfNeeded): When reading
+        a new data file, update the current operating date (if needed).
+        (WebKit::WebResourceLoadStatisticsStore::handleDailyTasks): Roll uptime dates as
+        needed, then handle telemetry.
+        * UIProcess/WebResourceLoadStatisticsStore.h:
+
 2017-07-07  Wenson Hsieh  <wenson_hsieh@apple.com>
 
         [iOS DnD] For cross-app drags, 'drop' event handlers are never invoked if dataTransfer.dropEffect is not set while dragging
index f92246a..492af2b 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2014, 2016 Apple Inc. All rights reserved.
+ * Copyright (C) 2014-2017 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -440,7 +440,7 @@ static Vector<WebKit::WebsiteDataRecord> toWebsiteDataRecords(NSArray *dataRecor
     if (!store)
         return;
 
-    store->setTimeToLiveUserInteraction(24_h * 30.);
+    store->setTimeToLiveUserInteraction(0_s);
     store->setTimeToLiveCookiePartitionFree(24_h);
     store->setMinimumTimeBetweenDataRecordsRemoval(1_h);
     store->setGrandfatheringTime(1_h);
index 3a57062..1a73601 100644 (file)
@@ -39,8 +39,9 @@ namespace WebKit {
 
 using namespace WebCore;
 
-const unsigned statisticsModelVersion { 5 };
-
+const unsigned statisticsModelVersion { 6 };
+const unsigned operatingDatesWindow { 30 };
+    
 Ref<ResourceLoadStatisticsStore> ResourceLoadStatisticsStore::create()
 {
     return adoptRef(*new ResourceLoadStatisticsStore());
@@ -89,6 +90,10 @@ std::unique_ptr<KeyedEncoder> ResourceLoadStatisticsStore::createEncoderFromData
         origin.value.encode(encoderInner);
     });
 
+    encoder->encodeObjects("operatingDates", m_operatingDates.begin(), m_operatingDates.end(), [](KeyedEncoder& encoderInner, WallTime date) {
+        encoderInner.encodeDouble("date", date.secondsSinceEpoch().value());
+    });
+    
     return encoder;
 }
 
@@ -128,6 +133,18 @@ void ResourceLoadStatisticsStore::readDataFromDecoder(KeyedDecoder& decoder)
         }
         m_resourceStatisticsMap.set(statistics.highLevelDomain, WTFMove(statistics));
     }
+
+    succeeded = decoder.decodeObjects("operatingDates", m_operatingDates, [](KeyedDecoder& decoder, WallTime& wallTime) {
+        double value;
+        if (!decoder.decodeDouble("date", value))
+            return false;
+        
+        wallTime = WallTime::fromRawSeconds(value);
+        return true;
+    });
+
+    if (!succeeded)
+        return;
     
     fireShouldPartitionCookiesHandler({ }, prevalentResourceDomainsWithoutUserInteraction, true);
 }
@@ -136,7 +153,8 @@ void ResourceLoadStatisticsStore::clearInMemory()
 {
     ASSERT(!RunLoop::isMain());
     m_resourceStatisticsMap.clear();
-    
+    m_operatingDates.clear();
+
     fireShouldPartitionCookiesHandler({ }, { }, true);
 }
 
@@ -293,7 +311,7 @@ bool ResourceLoadStatisticsStore::hasHadRecentUserInteraction(ResourceLoadStatis
     if (!resourceStatistic.hadUserInteraction)
         return false;
 
-    if (WallTime::now() > resourceStatistic.mostRecentUserInteractionTime + m_timeToLiveUserInteraction) {
+    if (hasStatisticsExpired(resourceStatistic)) {
         // Drop privacy sensitive data because we no longer need it.
         // Set timestamp to 0 so that statistics merge will know
         // it has been reset as opposed to its default -1.
@@ -403,5 +421,33 @@ void ResourceLoadStatisticsStore::dataRecordsWereRemoved()
     ASSERT(!RunLoop::isMain());
     m_dataRecordsRemovalPending = false;
 }
+
+void ResourceLoadStatisticsStore::includeTodayAsOperatingDateIfNecessary()
+{
+    if (!m_operatingDates.isEmpty() && (WallTime::now() - m_operatingDates.last() < 24_h))
+        return;
+
+    while (m_operatingDates.size() >= operatingDatesWindow)
+        m_operatingDates.removeFirst();
+
+    m_operatingDates.append(WallTime::now());
+}
+    
+bool ResourceLoadStatisticsStore::hasStatisticsExpired(const ResourceLoadStatistics& resourceStatistic) const
+{
+    if (m_operatingDates.size() >= operatingDatesWindow) {
+        if (resourceStatistic.mostRecentUserInteractionTime < m_operatingDates.first())
+            return true;
+    }
+
+    // If we don't meet the real criteria for an expired statistic, check the user
+    // setting for a tighter restriction (mainly for testing).
+    if (m_timeToLiveUserInteraction) {
+        if (WallTime::now() > resourceStatistic.mostRecentUserInteractionTime + m_timeToLiveUserInteraction)
+            return true;
+    }
+    
+    return false;
+}
     
 }
index 23bb742..62b8fea 100644 (file)
@@ -25,6 +25,7 @@
 
 #pragma once
 
+#include <wtf/Deque.h>
 #include <wtf/Function.h>
 #include <wtf/HashMap.h>
 #include <wtf/HashSet.h>
@@ -99,12 +100,16 @@ public:
     void dataRecordsBeingRemoved();
     void dataRecordsWereRemoved();
 
+    void includeTodayAsOperatingDateIfNecessary();
+
 private:
     ResourceLoadStatisticsStore() = default;
 
     bool shouldPartitionCookies(const WebCore::ResourceLoadStatistics&) const;
+    bool hasStatisticsExpired(const WebCore::ResourceLoadStatistics&) const;
 
     HashMap<String, WebCore::ResourceLoadStatistics> m_resourceStatisticsMap;
+    Deque<WTF::WallTime> m_operatingDates;
 
     WTF::Function<void()> m_dataAddedHandler;
     WTF::Function<void(const Vector<String>&, const Vector<String>&, bool clearFirst)> m_shouldPartitionCookiesForDomainsHandler;
@@ -112,7 +117,7 @@ private:
     WTF::Function<void()> m_deletePersistentStoreHandler;
     WTF::Function<void()> m_fireTelemetryHandler;
 
-    Seconds m_timeToLiveUserInteraction { 24_h * 30. };
+    Seconds m_timeToLiveUserInteraction { 0_s };
     Seconds m_timeToLiveCookiePartitionFree { 24_h };
     Seconds m_grandfatheringTime { 1_h };
     Seconds m_minimumTimeBetweenDataRecordsRemoval { 1_h };
index 1344882..583231d 100644 (file)
@@ -94,8 +94,8 @@ WebResourceLoadStatisticsStore::WebResourceLoadStatisticsStore(const String& res
     : m_resourceLoadStatisticsStore(ResourceLoadStatisticsStore::create())
     , m_statisticsQueue(WorkQueue::create("WebResourceLoadStatisticsStore Process Data Queue", WorkQueue::Type::Serial, WorkQueue::QOS::Utility))
     , m_statisticsStoragePath(resourceLoadStatisticsDirectory)
-    , m_telemetryOneShotTimer(RunLoop::main(), this, &WebResourceLoadStatisticsStore::telemetryTimerFired)
-    , m_telemetryRepeatedTimer(RunLoop::main(), this, &WebResourceLoadStatisticsStore::telemetryTimerFired)
+    , m_telemetryOneShotTimer(RunLoop::main(), this, &WebResourceLoadStatisticsStore::submitTelemetryIfNecessary)
+    , m_telemetryRepeatedTimer(RunLoop::main(), this, &WebResourceLoadStatisticsStore::performDailyTasks)
 {
     ASSERT(RunLoop::isMain());
 
@@ -272,6 +272,8 @@ void WebResourceLoadStatisticsStore::readDataFromDiskIfNeeded()
 
     if (coreStore().isEmpty())
         grandfatherExistingWebsiteData();
+
+    coreStore().includeTodayAsOperatingDateIfNecessary();
 }
     
 void WebResourceLoadStatisticsStore::refreshFromDisk()
@@ -526,7 +528,16 @@ std::unique_ptr<KeyedDecoder> WebResourceLoadStatisticsStore::createDecoderFromD
     return KeyedDecoder::decoder(reinterpret_cast<const uint8_t*>(buffer.data()), buffer.size());
 }
 
-void WebResourceLoadStatisticsStore::telemetryTimerFired()
+void WebResourceLoadStatisticsStore::performDailyTasks()
+{
+    ASSERT(RunLoop::isMain());
+
+    coreStore().includeTodayAsOperatingDateIfNecessary();
+
+    submitTelemetryIfNecessary();
+}
+
+void WebResourceLoadStatisticsStore::submitTelemetryIfNecessary()
 {
     ASSERT(RunLoop::isMain());
     
index 55982b9..efb0caa 100644 (file)
@@ -132,9 +132,10 @@ private:
     void clearInMemoryData();
     void syncWithExistingStatisticsStorageIfNeeded();
     void refreshFromDisk();
-    void telemetryTimerFired();
+    void submitTelemetryIfNecessary();
     void submitTelemetry();
     bool hasStatisticsFileChangedSinceLastSync(const String& path);
+    void performDailyTasks();
 
 #if PLATFORM(COCOA)
     void registerUserDefaultsIfNeeded();