Check notifyPagesWhenTelemetryWasCaptured before notifying pages if there is no WebPa...
[WebKit-https.git] / Source / WebKit / UIProcess / WebResourceLoadStatisticsTelemetry.cpp
1 /*
2  * Copyright (C) 2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "WebResourceLoadStatisticsTelemetry.h"
28
29 #include "WebProcessPool.h"
30 #include "WebProcessProxy.h"
31 #include "WebResourceLoadStatisticsStore.h"
32 #include <WebCore/DiagnosticLoggingKeys.h>
33 #include <WebCore/ResourceLoadStatistics.h>
34 #include <wtf/MainThread.h>
35 #include <wtf/NeverDestroyed.h>
36 #include <wtf/RunLoop.h>
37 #include <wtf/text/StringBuilder.h>
38
39 using namespace WebCore;
40
41 namespace WebKit {
42
43 const unsigned minimumPrevalentResourcesForTelemetry = 3;
44 const unsigned significantFiguresForLoggedValues = 3;
45 static bool notifyPagesWhenTelemetryWasCaptured = false;
46
47 struct PrevalentResourceTelemetry {
48     unsigned numberOfTimesDataRecordsRemoved;
49     bool hasHadUserInteraction;
50     unsigned daysSinceUserInteraction;
51     unsigned subframeUnderTopFrameOrigins;
52     unsigned subresourceUnderTopFrameOrigins;
53     unsigned subresourceUniqueRedirectsTo;
54 };
55
56 static Vector<PrevalentResourceTelemetry> sortedPrevalentResourceTelemetry(const WebResourceLoadStatisticsStore& store)
57 {
58     ASSERT(!RunLoop::isMain());
59     Vector<PrevalentResourceTelemetry> sorted;
60     store.processStatistics([&sorted] (auto& statistic) {
61         if (!statistic.isPrevalentResource)
62             return;
63
64         unsigned daysSinceUserInteraction = statistic.mostRecentUserInteractionTime <= WallTime() ? 0 : std::floor((WallTime::now() - statistic.mostRecentUserInteractionTime) / 24_h);
65         sorted.append(PrevalentResourceTelemetry {
66             statistic.dataRecordsRemoved,
67             statistic.hadUserInteraction,
68             daysSinceUserInteraction,
69             statistic.subframeUnderTopFrameOrigins.size(),
70             statistic.subresourceUnderTopFrameOrigins.size(),
71             statistic.subresourceUniqueRedirectsTo.size()
72         });
73     });
74
75     if (sorted.size() < minimumPrevalentResourcesForTelemetry)
76         return { };
77
78     std::sort(sorted.begin(), sorted.end(), [](const PrevalentResourceTelemetry& a, const PrevalentResourceTelemetry& b) {
79         return a.subframeUnderTopFrameOrigins + a.subresourceUnderTopFrameOrigins + a.subresourceUniqueRedirectsTo >
80         b.subframeUnderTopFrameOrigins + b.subresourceUnderTopFrameOrigins + b.subresourceUniqueRedirectsTo;
81     });
82
83     return sorted;
84 }
85
86 static unsigned numberOfResourcesWithUserInteraction(const Vector<PrevalentResourceTelemetry>& resources, size_t begin, size_t end)
87 {
88     if (resources.isEmpty() || resources.size() < begin + 1 || resources.size() < end + 1)
89         return 0;
90     
91     unsigned result = 0;
92     for (size_t i = begin; i < end; ++i) {
93         if (resources[i].hasHadUserInteraction)
94             ++result;
95     }
96     
97     return result;
98 }
99     
100 static unsigned median(const Vector<unsigned>& v)
101 {
102     if (v.isEmpty())
103         return 0;
104     if (v.size() == 1)
105         return v[0];
106     
107     auto size = v.size();
108     auto middle = size / 2;
109     if (size % 2)
110         return v[middle];
111     return (v[middle - 1] + v[middle]) / 2;
112 }
113     
114 static unsigned median(const Vector<PrevalentResourceTelemetry>& v, unsigned begin, unsigned end, const WTF::Function<unsigned(const PrevalentResourceTelemetry& telemetry)>& statisticGetter)
115 {
116     if (v.isEmpty() || v.size() < begin + 1 || v.size() < end + 1)
117         return 0;
118     
119     Vector<unsigned> part;
120     part.reserveInitialCapacity(end - begin + 1);
121     for (unsigned i = begin; i <= end; ++i)
122         part.uncheckedAppend(statisticGetter(v[i]));
123     
124     return median(part);
125 }
126     
127 static WebPageProxy* nonEphemeralWebPageProxy()
128 {
129     auto processPools = WebProcessPool::allProcessPools();
130     if (processPools.isEmpty())
131         return nullptr;
132     
133     auto processPool = processPools[0];
134     if (!processPool)
135         return nullptr;
136     
137     for (auto& webProcess : processPool->processes()) {
138         for (auto& page : webProcess->pages()) {
139             if (page->sessionID().isEphemeral())
140                 continue;
141             return page;
142         }
143     }
144     return nullptr;
145 }
146     
147 static void submitTopList(unsigned numberOfResourcesFromTheTop, const Vector<PrevalentResourceTelemetry>& sortedPrevalentResources, const Vector<PrevalentResourceTelemetry>& sortedPrevalentResourcesWithoutUserInteraction, WebPageProxy& webPageProxy)
148 {
149     WTF::Function<unsigned(const PrevalentResourceTelemetry& telemetry)> subframeUnderTopFrameOriginsGetter = [] (const PrevalentResourceTelemetry& t) {
150         return t.subframeUnderTopFrameOrigins;
151     };
152     WTF::Function<unsigned(const PrevalentResourceTelemetry& telemetry)> subresourceUnderTopFrameOriginsGetter = [] (const PrevalentResourceTelemetry& t) {
153         return t.subresourceUnderTopFrameOrigins;
154     };
155     WTF::Function<unsigned(const PrevalentResourceTelemetry& telemetry)> subresourceUniqueRedirectsToGetter = [] (const PrevalentResourceTelemetry& t) {
156         return t.subresourceUniqueRedirectsTo;
157     };
158     WTF::Function<unsigned(const PrevalentResourceTelemetry& telemetry)> numberOfTimesDataRecordsRemovedGetter = [] (const PrevalentResourceTelemetry& t) {
159         return t.numberOfTimesDataRecordsRemoved;
160     };
161     
162     unsigned topPrevalentResourcesWithUserInteraction = numberOfResourcesWithUserInteraction(sortedPrevalentResources, 0, numberOfResourcesFromTheTop - 1);
163     unsigned topSubframeUnderTopFrameOrigins = median(sortedPrevalentResourcesWithoutUserInteraction, 0, numberOfResourcesFromTheTop - 1, subframeUnderTopFrameOriginsGetter);
164     unsigned topSubresourceUnderTopFrameOrigins = median(sortedPrevalentResourcesWithoutUserInteraction, 0, numberOfResourcesFromTheTop - 1, subresourceUnderTopFrameOriginsGetter);
165     unsigned topSubresourceUniqueRedirectsTo = median(sortedPrevalentResourcesWithoutUserInteraction, 0, numberOfResourcesFromTheTop - 1, subresourceUniqueRedirectsToGetter);
166     unsigned topNumberOfTimesDataRecordsRemoved = median(sortedPrevalentResourcesWithoutUserInteraction, 0, numberOfResourcesFromTheTop - 1, numberOfTimesDataRecordsRemovedGetter);
167     
168     StringBuilder preambleBuilder;
169     preambleBuilder.appendLiteral("top");
170     preambleBuilder.appendNumber(numberOfResourcesFromTheTop);
171     String descriptionPreamble = preambleBuilder.toString();
172     
173     webPageProxy.logDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), descriptionPreamble + "PrevalentResourcesWithUserInteraction",
174         topPrevalentResourcesWithUserInteraction, significantFiguresForLoggedValues, ShouldSample::No);
175     webPageProxy.logDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), descriptionPreamble + "SubframeUnderTopFrameOrigins",
176         topSubframeUnderTopFrameOrigins, significantFiguresForLoggedValues, ShouldSample::No);
177     webPageProxy.logDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), descriptionPreamble + "SubresourceUnderTopFrameOrigins",
178         topSubresourceUnderTopFrameOrigins, significantFiguresForLoggedValues, ShouldSample::No);
179     webPageProxy.logDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), descriptionPreamble + "SubresourceUniqueRedirectsTo",
180         topSubresourceUniqueRedirectsTo, significantFiguresForLoggedValues, ShouldSample::No);
181     webPageProxy.logDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), descriptionPreamble + "NumberOfTimesDataRecordsRemoved",
182         topNumberOfTimesDataRecordsRemoved, significantFiguresForLoggedValues, ShouldSample::No);
183 }
184     
185 static void submitTopLists(const Vector<PrevalentResourceTelemetry>& sortedPrevalentResources, const Vector<PrevalentResourceTelemetry>& sortedPrevalentResourcesWithoutUserInteraction, WebPageProxy& webPageProxy)
186 {
187     submitTopList(1, sortedPrevalentResources, sortedPrevalentResourcesWithoutUserInteraction, webPageProxy);
188     
189     if (sortedPrevalentResourcesWithoutUserInteraction.size() < 3)
190         return;
191     submitTopList(3, sortedPrevalentResources, sortedPrevalentResourcesWithoutUserInteraction, webPageProxy);
192     
193     if (sortedPrevalentResourcesWithoutUserInteraction.size() < 10)
194         return;
195     submitTopList(10, sortedPrevalentResources, sortedPrevalentResourcesWithoutUserInteraction, webPageProxy);
196     
197     if (sortedPrevalentResourcesWithoutUserInteraction.size() < 50)
198         return;
199     submitTopList(50, sortedPrevalentResources, sortedPrevalentResourcesWithoutUserInteraction, webPageProxy);
200     
201     if (sortedPrevalentResourcesWithoutUserInteraction.size() < 100)
202         return;
203     submitTopList(100, sortedPrevalentResources, sortedPrevalentResourcesWithoutUserInteraction, webPageProxy);
204 }
205     
206 // This function is for testing purposes.
207 void static notifyPages(unsigned totalPrevalentResources, unsigned totalPrevalentResourcesWithUserInteraction, unsigned top3SubframeUnderTopFrameOrigins)
208 {
209     RunLoop::main().dispatch([totalPrevalentResources, totalPrevalentResourcesWithUserInteraction, top3SubframeUnderTopFrameOrigins] {
210         API::Dictionary::MapType messageBody;
211         messageBody.set(ASCIILiteral("TotalPrevalentResources"), API::UInt64::create(totalPrevalentResources));
212         messageBody.set(ASCIILiteral("TotalPrevalentResourcesWithUserInteraction"), API::UInt64::create(totalPrevalentResourcesWithUserInteraction));
213         messageBody.set(ASCIILiteral("Top3SubframeUnderTopFrameOrigins"), API::UInt64::create(top3SubframeUnderTopFrameOrigins));
214         WebProcessProxy::notifyPageStatisticsTelemetryFinished(API::Dictionary::create(messageBody).ptr());
215     });
216 }
217     
218 // This function is for testing purposes.
219 void static notifyPages(const Vector<PrevalentResourceTelemetry>& sortedPrevalentResources, const Vector<PrevalentResourceTelemetry>& sortedPrevalentResourcesWithoutUserInteraction, unsigned totalNumberOfPrevalentResourcesWithUserInteraction)
220 {
221     WTF::Function<unsigned(const PrevalentResourceTelemetry& telemetry)> subframeUnderTopFrameOriginsGetter = [] (const PrevalentResourceTelemetry& t) {
222         return t.subframeUnderTopFrameOrigins;
223     };
224     
225     notifyPages(sortedPrevalentResources.size(), totalNumberOfPrevalentResourcesWithUserInteraction, median(sortedPrevalentResourcesWithoutUserInteraction, 0, 2, subframeUnderTopFrameOriginsGetter));
226 }
227     
228 void WebResourceLoadStatisticsTelemetry::calculateAndSubmit(const WebResourceLoadStatisticsStore& resourceLoadStatisticsStore)
229 {
230     ASSERT(!RunLoop::isMain());
231     
232     auto sortedPrevalentResources = sortedPrevalentResourceTelemetry(resourceLoadStatisticsStore);
233     if (notifyPagesWhenTelemetryWasCaptured && sortedPrevalentResources.isEmpty()) {
234         notifyPages(0, 0, 0);
235         return;
236     }
237     
238     Vector<PrevalentResourceTelemetry> sortedPrevalentResourcesWithoutUserInteraction;
239     sortedPrevalentResourcesWithoutUserInteraction.reserveInitialCapacity(sortedPrevalentResources.size());
240     Vector<unsigned> prevalentResourcesDaysSinceUserInteraction;
241     
242     for (auto& prevalentResource : sortedPrevalentResources) {
243         if (prevalentResource.hasHadUserInteraction)
244             prevalentResourcesDaysSinceUserInteraction.append(prevalentResource.daysSinceUserInteraction);
245         else
246             sortedPrevalentResourcesWithoutUserInteraction.uncheckedAppend(prevalentResource);
247     }
248     
249     auto webPageProxy = nonEphemeralWebPageProxy();
250     if (!webPageProxy) {
251         if (notifyPagesWhenTelemetryWasCaptured)
252             notifyPages(0, 0, 0);
253         return;
254     }
255     
256     if (notifyPagesWhenTelemetryWasCaptured) {
257         notifyPages(sortedPrevalentResources, sortedPrevalentResourcesWithoutUserInteraction, prevalentResourcesDaysSinceUserInteraction.size());
258         // The notify pages function is for testing so we don't need to do an actual submission.
259         return;
260     }
261     
262     webPageProxy->logDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), ASCIILiteral("totalNumberOfPrevalentResources"), sortedPrevalentResources.size(), significantFiguresForLoggedValues, ShouldSample::No);
263     webPageProxy->logDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), ASCIILiteral("totalNumberOfPrevalentResourcesWithUserInteraction"), prevalentResourcesDaysSinceUserInteraction.size(), significantFiguresForLoggedValues, ShouldSample::No);
264     
265     if (prevalentResourcesDaysSinceUserInteraction.size() > 0)
266         webPageProxy->logDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), ASCIILiteral("topPrevalentResourceWithUserInteractionDaysSinceUserInteraction"), prevalentResourcesDaysSinceUserInteraction[0], significantFiguresForLoggedValues, ShouldSample::No);
267     if (prevalentResourcesDaysSinceUserInteraction.size() > 1)
268         webPageProxy->logDiagnosticMessageWithValue(DiagnosticLoggingKeys::resourceLoadStatisticsTelemetryKey(), ASCIILiteral("medianPrevalentResourcesWithUserInteractionDaysSinceUserInteraction"), median(prevalentResourcesDaysSinceUserInteraction), significantFiguresForLoggedValues, ShouldSample::No);
269     
270     submitTopLists(sortedPrevalentResources, sortedPrevalentResourcesWithoutUserInteraction, *webPageProxy);
271 }
272     
273 void WebResourceLoadStatisticsTelemetry::setNotifyPagesWhenTelemetryWasCaptured(bool always)
274 {
275     notifyPagesWhenTelemetryWasCaptured = always;
276 }
277     
278 }