7eae1a7c891419d41b6052545246dfe14aed6fda
[WebKit-https.git] / Source / WebKit / NetworkProcess / Classifier / WebResourceLoadStatisticsStore.h
1 /*
2  * Copyright (C) 2016-2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #pragma once
27
28 #if ENABLE(RESOURCE_LOAD_STATISTICS)
29
30 #include "Connection.h"
31 #include "StorageAccessStatus.h"
32 #include "WebPageProxyIdentifier.h"
33 #include "WebsiteDataType.h"
34 #include <WebCore/FrameIdentifier.h>
35 #include <WebCore/PageIdentifier.h>
36 #include <WebCore/RegistrableDomain.h>
37 #include <wtf/CompletionHandler.h>
38 #include <wtf/RunLoop.h>
39 #include <wtf/ThreadSafeRefCounted.h>
40 #include <wtf/Vector.h>
41 #include <wtf/WallTime.h>
42 #include <wtf/WeakPtr.h>
43 #include <wtf/text/WTFString.h>
44
45 namespace WTF {
46 class WorkQueue;
47 }
48
49 namespace WebCore {
50 class ResourceRequest;
51 struct ResourceLoadStatistics;
52 enum class ShouldSample : bool;
53 enum class IncludeHttpOnlyCookies : bool;
54 enum class StorageAccessPromptWasShown : bool;
55 enum class StorageAccessWasGranted : bool;
56 }
57
58 namespace WebKit {
59
60 class NetworkSession;
61 class ResourceLoadStatisticsStore;
62 class ResourceLoadStatisticsPersistentStorage;
63 class WebFrameProxy;
64 class WebProcessProxy;
65 enum class ShouldGrandfatherStatistics : bool;
66 enum class ShouldIncludeLocalhost : bool { No, Yes };
67 enum class EnableResourceLoadStatisticsDebugMode : bool { No, Yes };
68 enum class EnableResourceLoadStatisticsNSURLSessionSwitching : bool { No, Yes };
69 enum class WebsiteDataToRemove : uint8_t {
70     All,
71     AllButHttpOnlyCookies,
72     AllButCookies
73 };
74 struct RegistrableDomainsToBlockCookiesFor {
75     Vector<WebCore::RegistrableDomain> domainsToBlockAndDeleteCookiesFor;
76     Vector<WebCore::RegistrableDomain> domainsToBlockButKeepCookiesFor;
77     RegistrableDomainsToBlockCookiesFor isolatedCopy() const { return { domainsToBlockAndDeleteCookiesFor.isolatedCopy(), domainsToBlockButKeepCookiesFor.isolatedCopy() }; }
78 };
79
80 class WebResourceLoadStatisticsStore final : public ThreadSafeRefCounted<WebResourceLoadStatisticsStore, WTF::DestructionThread::Main> {
81 public:
82     using ResourceLoadStatistics = WebCore::ResourceLoadStatistics;
83     using RegistrableDomain = WebCore::RegistrableDomain;
84     using TopFrameDomain = WebCore::RegistrableDomain;
85     using SubFrameDomain = WebCore::RegistrableDomain;
86     using SubResourceDomain = WebCore::RegistrableDomain;
87     using RedirectDomain = WebCore::RegistrableDomain;
88     using RedirectedFromDomain = WebCore::RegistrableDomain;
89     using RedirectedToDomain = WebCore::RegistrableDomain;
90     using NavigatedFromDomain = WebCore::RegistrableDomain;
91     using NavigatedToDomain = WebCore::RegistrableDomain;
92     using DomainInNeedOfStorageAccess = WebCore::RegistrableDomain;
93     using OpenerDomain = WebCore::RegistrableDomain;
94     using StorageAccessWasGranted = WebCore::StorageAccessWasGranted;
95     using StorageAccessPromptWasShown = WebCore::StorageAccessPromptWasShown;
96
97     static Ref<WebResourceLoadStatisticsStore> create(NetworkSession& networkSession, const String& resourceLoadStatisticsDirectory, ShouldIncludeLocalhost shouldIncludeLocalhost)
98     {
99         return adoptRef(*new WebResourceLoadStatisticsStore(networkSession, resourceLoadStatisticsDirectory, shouldIncludeLocalhost));
100     }
101
102     ~WebResourceLoadStatisticsStore();
103
104     void didDestroyNetworkSession();
105
106     static const OptionSet<WebsiteDataType>& monitoredDataTypes();
107
108     WorkQueue& statisticsQueue() { return m_statisticsQueue.get(); }
109
110     void setNotifyPagesWhenDataRecordsWereScanned(bool);
111     void setNotifyPagesWhenTelemetryWasCaptured(bool, CompletionHandler<void()>&&);
112     void setShouldClassifyResourcesBeforeDataRecordsRemoval(bool, CompletionHandler<void()>&&);
113     void setShouldSubmitTelemetry(bool);
114
115     void grantStorageAccess(const SubFrameDomain&, const TopFrameDomain&, WebCore::FrameIdentifier, WebCore::PageIdentifier, StorageAccessPromptWasShown, CompletionHandler<void(StorageAccessWasGranted, StorageAccessPromptWasShown)>&&);
116
117     void applicationWillTerminate();
118
119     void logFrameNavigation(const WebFrameProxy&, const URL& pageURL, const WebCore::ResourceRequest&, const URL& redirectURL);
120     void logFrameNavigation(const NavigatedToDomain&, const TopFrameDomain&, const NavigatedFromDomain&, bool isRedirect, bool isMainFrame);
121     void logUserInteraction(const TopFrameDomain&, CompletionHandler<void()>&&);
122     void logCrossSiteLoadWithLinkDecoration(const NavigatedFromDomain&, const NavigatedToDomain&, CompletionHandler<void()>&&);
123     void clearUserInteraction(const TopFrameDomain&, CompletionHandler<void()>&&);
124     void deleteWebsiteDataForRegistrableDomains(OptionSet<WebsiteDataType>, Vector<std::pair<RegistrableDomain, WebsiteDataToRemove>>&&, bool shouldNotifyPage, CompletionHandler<void(const HashSet<RegistrableDomain>&)>&&);
125     void registrableDomainsWithWebsiteData(OptionSet<WebsiteDataType>, bool shouldNotifyPage, CompletionHandler<void(HashSet<RegistrableDomain>&&)>&&);
126     StorageAccessWasGranted grantStorageAccess(const SubFrameDomain&, const TopFrameDomain&, Optional<WebCore::FrameIdentifier>, WebCore::PageIdentifier);
127     void hasHadUserInteraction(const RegistrableDomain&, CompletionHandler<void(bool)>&&);
128     void hasStorageAccess(const SubFrameDomain&, const TopFrameDomain&, Optional<WebCore::FrameIdentifier>, WebCore::PageIdentifier, CompletionHandler<void(bool)>&&);
129     bool hasStorageAccessForFrame(const SubFrameDomain&, const TopFrameDomain&, WebCore::FrameIdentifier, WebCore::PageIdentifier);
130     void requestStorageAccess(const SubFrameDomain&, const TopFrameDomain&, WebCore::FrameIdentifier, WebCore::PageIdentifier, WebPageProxyIdentifier, CompletionHandler<void(StorageAccessWasGranted, StorageAccessPromptWasShown)>&&);
131     void setLastSeen(const RegistrableDomain&, Seconds, CompletionHandler<void()>&&);
132     void setPrevalentResource(const RegistrableDomain&, CompletionHandler<void()>&&);
133     void setVeryPrevalentResource(const RegistrableDomain&, CompletionHandler<void()>&&);
134     void dumpResourceLoadStatistics(CompletionHandler<void(String)>&&);
135     void isPrevalentResource(const RegistrableDomain&, CompletionHandler<void(bool)>&&);
136     void isVeryPrevalentResource(const RegistrableDomain&, CompletionHandler<void(bool)>&&);
137     void isRegisteredAsSubresourceUnder(const SubResourceDomain&, const TopFrameDomain&, CompletionHandler<void(bool)>&&);
138     void isRegisteredAsSubFrameUnder(const SubFrameDomain&, const TopFrameDomain&, CompletionHandler<void(bool)>&&);
139     void isRegisteredAsRedirectingTo(const RedirectedFromDomain&, const RedirectedToDomain&, CompletionHandler<void(bool)>&&);
140     void clearPrevalentResource(const RegistrableDomain&, CompletionHandler<void()>&&);
141     void setGrandfathered(const RegistrableDomain&, bool, CompletionHandler<void()>&&);
142     void isGrandfathered(const RegistrableDomain&, CompletionHandler<void(bool)>&&);
143     void removePrevalentDomains(const Vector<RegistrableDomain>&);
144     void setNotifyPagesWhenDataRecordsWereScanned(bool, CompletionHandler<void()>&&);
145     void setIsRunningTest(bool, CompletionHandler<void()>&&);
146     void setSubframeUnderTopFrameDomain(const SubFrameDomain&, const TopFrameDomain&, CompletionHandler<void()>&&);
147     void setSubresourceUnderTopFrameDomain(const SubResourceDomain&, const TopFrameDomain&, CompletionHandler<void()>&&);
148     void setSubresourceUniqueRedirectTo(const SubResourceDomain&, const RedirectedToDomain&, CompletionHandler<void()>&&);
149     void setSubresourceUniqueRedirectFrom(const SubResourceDomain&, const RedirectedFromDomain&, CompletionHandler<void()>&&);
150     void setTopFrameUniqueRedirectTo(const TopFrameDomain&, const RedirectedToDomain&, CompletionHandler<void()>&&);
151     void setTopFrameUniqueRedirectFrom(const TopFrameDomain&, const RedirectedFromDomain&, CompletionHandler<void()>&&);
152     void scheduleCookieBlockingUpdate(CompletionHandler<void()>&&);
153     void scheduleCookieBlockingUpdateForDomains(const Vector<RegistrableDomain>&, CompletionHandler<void()>&&);
154     void scheduleClearBlockingStateForDomains(const Vector<RegistrableDomain>&, CompletionHandler<void()>&&);
155     void scheduleStatisticsAndDataRecordsProcessing(CompletionHandler<void()>&&);
156     void submitTelemetry(CompletionHandler<void()>&&);
157     void scheduleClearInMemoryAndPersistent(ShouldGrandfatherStatistics, CompletionHandler<void()>&&);
158     void scheduleClearInMemoryAndPersistent(WallTime modifiedSince, ShouldGrandfatherStatistics, CompletionHandler<void()>&&);
159
160     void setTimeToLiveUserInteraction(Seconds, CompletionHandler<void()>&&);
161     void setMinimumTimeBetweenDataRecordsRemoval(Seconds, CompletionHandler<void()>&&);
162     void setGrandfatheringTime(Seconds, CompletionHandler<void()>&&);
163     void setCacheMaxAgeCap(Seconds, CompletionHandler<void()>&&);
164     void setMaxStatisticsEntries(size_t, CompletionHandler<void()>&&);
165     void setPruneEntriesDownTo(size_t, CompletionHandler<void()>&&);
166
167     void resetParametersToDefaultValues(CompletionHandler<void()>&&);
168
169     void setResourceLoadStatisticsDebugMode(bool, CompletionHandler<void()>&&);
170     void setPrevalentResourceForDebugMode(const RegistrableDomain&, CompletionHandler<void()>&&);
171
172     void logTestingEvent(const String&);
173     void callGrantStorageAccessHandler(const SubFrameDomain&, const TopFrameDomain&, Optional<WebCore::FrameIdentifier>, WebCore::PageIdentifier, CompletionHandler<void(StorageAccessWasGranted)>&&);
174     void removeAllStorageAccess(CompletionHandler<void()>&&);
175     void callUpdatePrevalentDomainsToBlockCookiesForHandler(const RegistrableDomainsToBlockCookiesFor&, CompletionHandler<void()>&&);
176     void callRemoveDomainsHandler(const Vector<RegistrableDomain>&);
177     void callHasStorageAccessForFrameHandler(const SubFrameDomain&, const TopFrameDomain&, WebCore::FrameIdentifier, WebCore::PageIdentifier, CompletionHandler<void(bool)>&&);
178
179     void hasCookies(const RegistrableDomain&, CompletionHandler<void(bool)>&&);
180
181     void didCreateNetworkProcess();
182
183     void notifyResourceLoadStatisticsProcessed();
184
185     NetworkSession* networkSession();
186     void invalidateAndCancel();
187
188     void sendDiagnosticMessageWithValue(const String& message, const String& description, unsigned value, unsigned sigDigits, WebCore::ShouldSample) const;
189     void notifyPageStatisticsTelemetryFinished(unsigned totalPrevalentResources, unsigned totalPrevalentResourcesWithUserInteraction, unsigned top3SubframeUnderTopFrameOrigins) const;
190
191     void resourceLoadStatisticsUpdated(Vector<WebCore::ResourceLoadStatistics>&&);
192     void requestStorageAccessUnderOpener(DomainInNeedOfStorageAccess&&, WebCore::PageIdentifier openerID, OpenerDomain&&);
193
194 private:
195     explicit WebResourceLoadStatisticsStore(NetworkSession&, const String&, ShouldIncludeLocalhost);
196
197     void postTask(WTF::Function<void()>&&);
198     static void postTaskReply(WTF::Function<void()>&&);
199
200     void performDailyTasks();
201
202     StorageAccessStatus storageAccessStatus(const String& subFramePrimaryDomain, const String& topFramePrimaryDomain);
203
204     void flushAndDestroyPersistentStore();
205
206     WeakPtr<NetworkSession> m_networkSession;
207     Ref<WorkQueue> m_statisticsQueue;
208     std::unique_ptr<ResourceLoadStatisticsStore> m_statisticsStore;
209     std::unique_ptr<ResourceLoadStatisticsPersistentStorage> m_persistentStorage;
210
211     RunLoop::Timer<WebResourceLoadStatisticsStore> m_dailyTasksTimer;
212
213     bool m_hasScheduledProcessStats { false };
214
215     bool m_firstNetworkProcessCreated { false };
216 };
217
218 } // namespace WebKit
219
220 #endif