2c304e794f237133fb5b2c97ad88ed7a03aa7af7
[WebKit-https.git] / Source / WebKit / NetworkProcess / Classifier / ResourceLoadStatisticsDatabaseStore.cpp
1 /*
2  * Copyright (C) 2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ResourceLoadStatisticsDatabaseStore.h"
28
29 #if ENABLE(RESOURCE_LOAD_STATISTICS)
30
31 #include "Logging.h"
32 #include "NetworkSession.h"
33 #include "PluginProcessManager.h"
34 #include "PluginProcessProxy.h"
35 #include "ResourceLoadStatisticsMemoryStore.h"
36 #include "StorageAccessStatus.h"
37 #include "WebProcessProxy.h"
38 #include "WebResourceLoadStatisticsTelemetry.h"
39 #include "WebsiteDataStore.h"
40 #include <WebCore/DocumentStorageAccess.h>
41 #include <WebCore/KeyedCoding.h>
42 #include <WebCore/NetworkStorageSession.h>
43 #include <WebCore/ResourceLoadStatistics.h>
44 #include <WebCore/SQLiteDatabase.h>
45 #include <WebCore/SQLiteStatement.h>
46 #include <WebCore/UserGestureIndicator.h>
47 #include <wtf/CallbackAggregator.h>
48 #include <wtf/DateMath.h>
49 #include <wtf/HashMap.h>
50 #include <wtf/MathExtras.h>
51 #include <wtf/StdSet.h>
52 #include <wtf/text/StringBuilder.h>
53
54 namespace WebKit {
55 using namespace WebCore;
56
57 #if PLATFORM(COCOA)
58 #define RELEASE_LOG_IF_ALLOWED(sessionID, fmt, ...) RELEASE_LOG_IF(sessionID.isAlwaysOnLoggingAllowed(), Network, "%p - ResourceLoadStatisticsDatabaseStore::" fmt, this, ##__VA_ARGS__)
59 #define RELEASE_LOG_ERROR_IF_ALLOWED(sessionID, fmt, ...) RELEASE_LOG_ERROR_IF(sessionID.isAlwaysOnLoggingAllowed(), Network, "%p - ResourceLoadStatisticsDatabaseStore::" fmt, this, ##__VA_ARGS__)
60 #else
61 #define RELEASE_LOG_IF_ALLOWED(sessionID, fmt, ...)  ((void)0)
62 #define RELEASE_LOG_ERROR_IF_ALLOWED(sessionID, fmt, ...)  ((void)0)
63 #endif
64
65 constexpr auto observedDomainCountQuery = "SELECT COUNT(*) FROM ObservedDomains"_s;
66 constexpr auto insertObservedDomainQuery = "INSERT INTO ObservedDomains (registrableDomain, lastSeen, hadUserInteraction,"
67     "mostRecentUserInteractionTime, grandfathered, isPrevalent, isVeryPrevalent, dataRecordsRemoved, timesAccessedAsFirstPartyDueToUserInteraction,"
68     "timesAccessedAsFirstPartyDueToStorageAccessAPI) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"_s;
69 constexpr auto insertTopLevelDomainQuery = "INSERT INTO TopLevelDomains VALUES (?)"_s;
70 constexpr auto domainIDFromStringQuery = "SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
71 constexpr auto storageAccessUnderTopFrameDomainsQuery = "INSERT INTO StorageAccessUnderTopFrameDomains (domainID, topLevelDomainID) "
72     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
73 constexpr auto topFrameUniqueRedirectsToQuery = "INSERT INTO TopFrameUniqueRedirectsTo (sourceDomainID, toDomainID) SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
74 constexpr auto topFrameUniqueRedirectsToExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameUniqueRedirectsTo WHERE sourceDomainID = ? "
75     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
76 constexpr auto topFrameUniqueRedirectsFromQuery = "INSERT INTO TopFrameUniqueRedirectsFrom (targetDomainID, fromDomainID) "
77     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
78 constexpr auto topFrameUniqueRedirectsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameUniqueRedirectsFrom WHERE targetDomainID = ? "
79     "AND fromDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
80 constexpr auto topFrameLinkDecorationsFromQuery = "INSERT INTO TopFrameLinkDecorationsFrom (fromDomainID, toDomainID) "
81     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
82 constexpr auto topFrameLinkDecorationsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameLinkDecorationsFrom WHERE fromDomainID = ? "
83     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
84 constexpr auto subframeUnderTopFrameDomainsQuery = "INSERT INTO SubframeUnderTopFrameDomains (subFrameDomainID, topFrameDomainID) "
85     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
86 constexpr auto subframeUnderTopFrameDomainExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubframeUnderTopFrameDomains WHERE subFrameDomainID = ? "
87     "AND topFrameDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
88 constexpr auto subresourceUnderTopFrameDomainsQuery = "INSERT INTO SubresourceUnderTopFrameDomains (subresourceDomainID, topFrameDomainID) "
89     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
90 constexpr auto subresourceUnderTopFrameDomainExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUnderTopFrameDomains "
91     "WHERE subresourceDomainID = ? AND topFrameDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
92 constexpr auto subresourceUniqueRedirectsToQuery = "INSERT INTO SubresourceUniqueRedirectsTo (subresourceDomainID, toDomainID) "
93     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
94 constexpr auto subresourceUniqueRedirectsToExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUniqueRedirectsTo WHERE subresourceDomainID = ? "
95     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
96 constexpr auto subresourceUniqueRedirectsFromQuery = "INSERT INTO SubresourceUniqueRedirectsFrom (subresourceDomainID, fromDomainID) "
97     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
98 constexpr auto subresourceUniqueRedirectsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUniqueRedirectsFrom WHERE subresourceDomainID = ? "
99     "AND fromDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
100 constexpr auto mostRecentUserInteractionQuery = "UPDATE ObservedDomains SET hadUserInteraction = ?, mostRecentUserInteractionTime = ? "
101     "WHERE registrableDomain = ?"_s;
102 constexpr auto updateLastSeenQuery = "UPDATE ObservedDomains SET lastSeen = ? WHERE registrableDomain = ?"_s;
103 constexpr auto updatePrevalentResourceQuery = "UPDATE ObservedDomains SET isPrevalent = ? WHERE registrableDomain = ?"_s;
104 constexpr auto isPrevalentResourceQuery = "SELECT isPrevalent FROM ObservedDomains WHERE registrableDomain = ?"_s;
105 constexpr auto updateVeryPrevalentResourceQuery = "UPDATE ObservedDomains SET isVeryPrevalent = ? WHERE registrableDomain = ?"_s;
106 constexpr auto isVeryPrevalentResourceQuery = "SELECT isVeryPrevalent FROM ObservedDomains WHERE registrableDomain = ?"_s;
107 constexpr auto clearPrevalentResourceQuery = "UPDATE ObservedDomains SET isPrevalent = 0, isVeryPrevalent = 0 WHERE registrableDomain = ?"_s;
108 constexpr auto hadUserInteractionQuery = "SELECT hadUserInteraction, mostRecentUserInteractionTime FROM ObservedDomains WHERE registrableDomain = ?"_s;
109 constexpr auto updateGrandfatheredQuery = "UPDATE ObservedDomains SET grandfathered = ? WHERE registrableDomain = ?"_s;
110 constexpr auto isGrandfatheredQuery = "SELECT grandfathered FROM ObservedDomains WHERE registrableDomain = ?"_s;
111 constexpr auto findExpiredUserInteractionQuery = "SELECT domainID FROM ObservedDomains WHERE hadUserInteraction = 1 AND mostRecentUserInteractionTime < ?"_s;
112
113 constexpr auto createObservedDomain = "CREATE TABLE ObservedDomains ("
114     "domainID INTEGER PRIMARY KEY, registrableDomain TEXT NOT NULL UNIQUE ON CONFLICT FAIL, lastSeen REAL NOT NULL, "
115     "hadUserInteraction INTEGER NOT NULL, mostRecentUserInteractionTime REAL NOT NULL, grandfathered INTEGER NOT NULL, "
116     "isPrevalent INTEGER NOT NULL, isVeryPrevalent INTEGER NOT NULL, dataRecordsRemoved INTEGER NOT NULL,"
117     "timesAccessedAsFirstPartyDueToUserInteraction INTEGER NOT NULL, timesAccessedAsFirstPartyDueToStorageAccessAPI INTEGER NOT NULL);"_s;
118     
119 constexpr auto createTopLevelDomains = "CREATE TABLE TopLevelDomains ("
120     "topLevelDomainID INTEGER PRIMARY KEY, CONSTRAINT fkDomainID FOREIGN KEY(topLevelDomainID) "
121     "REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
122     
123 constexpr auto createStorageAccessUnderTopFrameDomains = "CREATE TABLE StorageAccessUnderTopFrameDomains ("
124     "domainID INTEGER NOT NULL, topLevelDomainID INTEGER NOT NULL ON CONFLICT FAIL, "
125     "CONSTRAINT fkDomainID FOREIGN KEY(domainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
126     "FOREIGN KEY(topLevelDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
127     
128 constexpr auto createTopFrameUniqueRedirectsTo = "CREATE TABLE TopFrameUniqueRedirectsTo ("
129     "sourceDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
130     "FOREIGN KEY(sourceDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
131     "FOREIGN KEY(toDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
132     
133 constexpr auto createTopFrameUniqueRedirectsFrom = "CREATE TABLE TopFrameUniqueRedirectsFrom ("
134     "targetDomainID INTEGER NOT NULL, fromDomainID INTEGER NOT NULL, "
135     "FOREIGN KEY(targetDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
136     "FOREIGN KEY(fromDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
137
138 constexpr auto createTopFrameLinkDecorationsFrom = "CREATE TABLE TopFrameLinkDecorationsFrom ("
139     "fromDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
140     "FOREIGN KEY(fromDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
141     "FOREIGN KEY(toDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
142
143 constexpr auto createSubframeUnderTopFrameDomains = "CREATE TABLE SubframeUnderTopFrameDomains ("
144     "subFrameDomainID INTEGER NOT NULL, topFrameDomainID INTEGER NOT NULL, "
145     "FOREIGN KEY(subFrameDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
146     "FOREIGN KEY(topFrameDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
147     
148 constexpr auto createSubresourceUnderTopFrameDomains = "CREATE TABLE SubresourceUnderTopFrameDomains ("
149     "subresourceDomainID INTEGER NOT NULL, topFrameDomainID INTEGER NOT NULL, "
150     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
151     "FOREIGN KEY(topFrameDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
152     
153 constexpr auto createSubresourceUniqueRedirectsTo = "CREATE TABLE SubresourceUniqueRedirectsTo ("
154     "subresourceDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
155     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
156     "FOREIGN KEY(toDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
157     
158 constexpr auto createSubresourceUniqueRedirectsFrom = "CREATE TABLE SubresourceUniqueRedirectsFrom ("
159     "subresourceDomainID INTEGER NOT NULL, fromDomainID INTEGER NOT NULL, "
160     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
161     "FOREIGN KEY(fromDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
162     
163 ResourceLoadStatisticsDatabaseStore::ResourceLoadStatisticsDatabaseStore(WebResourceLoadStatisticsStore& store, WorkQueue& workQueue, ShouldIncludeLocalhost shouldIncludeLocalhost, const String& storageDirectoryPath, PAL::SessionID sessionID)
164     : ResourceLoadStatisticsStore(store, workQueue, shouldIncludeLocalhost)
165     , m_storageDirectoryPath(storageDirectoryPath + "/observations.db")
166     , m_observedDomainCount(m_database, observedDomainCountQuery)
167     , m_insertObservedDomainStatement(m_database, insertObservedDomainQuery)
168     , m_insertTopLevelDomainStatement(m_database, insertTopLevelDomainQuery)
169     , m_domainIDFromStringStatement(m_database, domainIDFromStringQuery)
170     , m_storageAccessUnderTopFrameDomainsStatement(m_database, storageAccessUnderTopFrameDomainsQuery)
171     , m_topFrameUniqueRedirectsTo(m_database, topFrameUniqueRedirectsToQuery)
172     , m_topFrameUniqueRedirectsToExists(m_database, topFrameUniqueRedirectsToExistsQuery)
173     , m_topFrameUniqueRedirectsFrom(m_database, topFrameUniqueRedirectsFromQuery)
174     , m_topFrameUniqueRedirectsFromExists(m_database, topFrameUniqueRedirectsFromExistsQuery)
175     , m_topFrameLinkDecorationsFrom(m_database, topFrameLinkDecorationsFromQuery)
176     , m_topFrameLinkDecorationsFromExists(m_database, topFrameLinkDecorationsFromExistsQuery)
177     , m_subframeUnderTopFrameDomains(m_database, subframeUnderTopFrameDomainsQuery)
178     , m_subframeUnderTopFrameDomainExists(m_database, subframeUnderTopFrameDomainExistsQuery)
179     , m_subresourceUnderTopFrameDomains(m_database, subresourceUnderTopFrameDomainsQuery)
180     , m_subresourceUnderTopFrameDomainExists(m_database, subresourceUnderTopFrameDomainExistsQuery)
181     , m_subresourceUniqueRedirectsTo(m_database, subresourceUniqueRedirectsToQuery)
182     , m_subresourceUniqueRedirectsToExists(m_database, subresourceUniqueRedirectsToExistsQuery)
183     , m_subresourceUniqueRedirectsFrom(m_database, subresourceUniqueRedirectsFromQuery)
184     , m_subresourceUniqueRedirectsFromExists(m_database, subresourceUniqueRedirectsFromExistsQuery)
185     , m_mostRecentUserInteractionStatement(m_database, mostRecentUserInteractionQuery)
186     , m_updateLastSeenStatement(m_database, updateLastSeenQuery)
187     , m_updatePrevalentResourceStatement(m_database, updatePrevalentResourceQuery)
188     , m_isPrevalentResourceStatement(m_database, isPrevalentResourceQuery)
189     , m_updateVeryPrevalentResourceStatement(m_database, updateVeryPrevalentResourceQuery)
190     , m_isVeryPrevalentResourceStatement(m_database, isVeryPrevalentResourceQuery)
191     , m_clearPrevalentResourceStatement(m_database, clearPrevalentResourceQuery)
192     , m_hadUserInteractionStatement(m_database, hadUserInteractionQuery)
193     , m_updateGrandfatheredStatement(m_database, updateGrandfatheredQuery)
194     , m_isGrandfatheredStatement(m_database, isGrandfatheredQuery)
195     , m_findExpiredUserInteractionStatement(m_database, findExpiredUserInteractionQuery)
196     , m_sessionID(sessionID)
197 {
198     ASSERT(!RunLoop::isMain());
199
200     if (!m_database.open(m_storageDirectoryPath)) {
201         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::open failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
202         ASSERT_NOT_REACHED();
203         return;
204     }
205     
206     // Since we are using a workerQueue, the sequential dispatch blocks may be called by different threads.
207     m_database.disableThreadingChecks();
208
209     if (!m_database.tableExists("ObservedDomains"_s)) {
210         if (!createSchema()) {
211             RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::createSchema failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
212             ASSERT_NOT_REACHED();
213             return;
214         }
215     }
216     
217     if (!m_database.turnOnIncrementalAutoVacuum())
218         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::turnOnIncrementalAutoVacuum failed, error message: %{public}s", this, m_database.lastErrorMsg());
219
220     if (!prepareStatements()) {
221         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::prepareStatements failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
222         ASSERT_NOT_REACHED();
223         return;
224     }
225
226     workQueue.dispatchAfter(5_s, [weakThis = makeWeakPtr(*this)] {
227         if (weakThis)
228             weakThis->calculateAndSubmitTelemetry();
229     });
230 }
231
232 bool ResourceLoadStatisticsDatabaseStore::isEmpty() const
233 {
234     ASSERT(!RunLoop::isMain());
235     
236     bool result = false;
237     if (m_observedDomainCount.step() == SQLITE_ROW)
238         result = !m_observedDomainCount.getColumnInt(0);
239     
240     int resetResult = m_observedDomainCount.reset();
241     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
242     
243     return result;
244 }
245
246 bool ResourceLoadStatisticsDatabaseStore::createSchema()
247 {
248     ASSERT(!RunLoop::isMain());
249
250     if (!m_database.executeCommand(createObservedDomain)) {
251         LOG_ERROR("Could not create ObservedDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
252         return false;
253     }
254
255     if (!m_database.executeCommand(createTopLevelDomains)) {
256         LOG_ERROR("Could not create TopLevelDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
257         return false;
258     }
259
260     if (!m_database.executeCommand(createStorageAccessUnderTopFrameDomains)) {
261         LOG_ERROR("Could not create StorageAccessUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
262         return false;
263     }
264
265     if (!m_database.executeCommand(createTopFrameUniqueRedirectsTo)) {
266         LOG_ERROR("Could not create TopFrameUniqueRedirectsTo table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
267         return false;
268     }
269
270     if (!m_database.executeCommand(createTopFrameUniqueRedirectsFrom)) {
271         LOG_ERROR("Could not create TopFrameUniqueRedirectsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
272         return false;
273     }
274
275     if (!m_database.executeCommand(createTopFrameLinkDecorationsFrom)) {
276         LOG_ERROR("Could not create TopFrameLinkDecorationsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
277         return false;
278     }
279     
280     if (!m_database.executeCommand(createSubframeUnderTopFrameDomains)) {
281         LOG_ERROR("Could not create SubframeUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
282         return false;
283     }
284
285     if (!m_database.executeCommand(createSubresourceUnderTopFrameDomains)) {
286         LOG_ERROR("Could not create SubresourceUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
287         return false;
288     }
289
290     if (!m_database.executeCommand(createSubresourceUniqueRedirectsTo)) {
291         LOG_ERROR("Could not create SubresourceUniqueRedirectsTo table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
292         return false;
293     }
294
295     if (!m_database.executeCommand(createSubresourceUniqueRedirectsFrom)) {
296         LOG_ERROR("Could not create SubresourceUniqueRedirectsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
297         return false;
298     }
299
300     return true;
301 }
302
303 bool ResourceLoadStatisticsDatabaseStore::prepareStatements()
304 {
305     ASSERT(!RunLoop::isMain());
306         
307     if (m_observedDomainCount.prepare() != SQLITE_OK
308         || m_insertObservedDomainStatement.prepare() != SQLITE_OK
309         || m_insertTopLevelDomainStatement.prepare() != SQLITE_OK
310         || m_domainIDFromStringStatement.prepare() != SQLITE_OK
311         || m_storageAccessUnderTopFrameDomainsStatement.prepare() != SQLITE_OK
312         || m_topFrameUniqueRedirectsTo.prepare() != SQLITE_OK
313         || m_topFrameUniqueRedirectsToExists.prepare() != SQLITE_OK
314         || m_topFrameUniqueRedirectsFrom.prepare() != SQLITE_OK
315         || m_topFrameUniqueRedirectsFromExists.prepare() != SQLITE_OK
316         || m_subframeUnderTopFrameDomains.prepare() != SQLITE_OK
317         || m_subframeUnderTopFrameDomainExists.prepare() != SQLITE_OK
318         || m_subresourceUnderTopFrameDomains.prepare() != SQLITE_OK
319         || m_subresourceUnderTopFrameDomainExists.prepare() != SQLITE_OK
320         || m_subresourceUniqueRedirectsTo.prepare() != SQLITE_OK
321         || m_subresourceUniqueRedirectsToExists.prepare() != SQLITE_OK
322         || m_subresourceUniqueRedirectsFrom.prepare() != SQLITE_OK
323         || m_subresourceUniqueRedirectsFromExists.prepare() != SQLITE_OK
324         || m_updateLastSeenStatement.prepare() != SQLITE_OK
325         || m_mostRecentUserInteractionStatement.prepare() != SQLITE_OK
326         || m_updatePrevalentResourceStatement.prepare() != SQLITE_OK
327         || m_isPrevalentResourceStatement.prepare() != SQLITE_OK
328         || m_updateVeryPrevalentResourceStatement.prepare() != SQLITE_OK
329         || m_isVeryPrevalentResourceStatement.prepare() != SQLITE_OK
330         || m_clearPrevalentResourceStatement.prepare() != SQLITE_OK
331         || m_hadUserInteractionStatement.prepare() != SQLITE_OK
332         || m_updateGrandfatheredStatement.prepare() != SQLITE_OK
333         || m_isGrandfatheredStatement.prepare() != SQLITE_OK
334         || m_findExpiredUserInteractionStatement.prepare() != SQLITE_OK
335         || m_topFrameLinkDecorationsFrom.prepare() != SQLITE_OK
336         || m_topFrameLinkDecorationsFromExists.prepare() != SQLITE_OK) {
337         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::prepareStatements failed to prepare, error message: %{public}s", this, m_database.lastErrorMsg());
338         ASSERT_NOT_REACHED();
339         return false;
340     }
341
342     return true;
343 }
344
345 bool ResourceLoadStatisticsDatabaseStore::insertObservedDomain(const ResourceLoadStatistics& loadStatistics)
346 {
347     ASSERT(!RunLoop::isMain());
348
349 #ifndef NDEBUG
350     ASSERT(confirmDomainDoesNotExist(loadStatistics.registrableDomain));
351 #endif
352
353     if (m_insertObservedDomainStatement.bindText(1, loadStatistics.registrableDomain.string()) != SQLITE_OK
354         || m_insertObservedDomainStatement.bindDouble(2, loadStatistics.lastSeen.secondsSinceEpoch().value()) != SQLITE_OK
355         || m_insertObservedDomainStatement.bindInt(3, loadStatistics.hadUserInteraction) != SQLITE_OK
356         || m_insertObservedDomainStatement.bindDouble(4, loadStatistics.mostRecentUserInteractionTime.secondsSinceEpoch().value()) != SQLITE_OK
357         || m_insertObservedDomainStatement.bindInt(5, loadStatistics.grandfathered) != SQLITE_OK
358         || m_insertObservedDomainStatement.bindInt(6, loadStatistics.isPrevalentResource) != SQLITE_OK
359         || m_insertObservedDomainStatement.bindInt(7, loadStatistics.isVeryPrevalentResource) != SQLITE_OK
360         || m_insertObservedDomainStatement.bindInt(8, loadStatistics.dataRecordsRemoved) != SQLITE_OK
361         || m_insertObservedDomainStatement.bindInt(9, loadStatistics.timesAccessedAsFirstPartyDueToUserInteraction) != SQLITE_OK
362         || m_insertObservedDomainStatement.bindInt(10, loadStatistics.timesAccessedAsFirstPartyDueToStorageAccessAPI) != SQLITE_OK) {
363         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::insertObservedDomain failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
364         ASSERT_NOT_REACHED();
365         return false;
366     }
367
368     if (m_insertObservedDomainStatement.step() != SQLITE_DONE) {
369         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::insertObservedDomain failed to commit, error message: %{public}s", this, m_database.lastErrorMsg());
370         ASSERT_NOT_REACHED();
371         return false;
372     }
373
374     int resetResult = m_insertObservedDomainStatement.reset();
375     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
376
377     return true;
378 }
379
380 bool ResourceLoadStatisticsDatabaseStore::relationshipExists(WebCore::SQLiteStatement& statement, Optional<unsigned> firstDomainID, const RegistrableDomain& secondDomain) const
381 {
382     if (!firstDomainID)
383         return false;
384     
385     ASSERT(!RunLoop::isMain());
386
387     if (statement.bindInt(1, *firstDomainID) != SQLITE_OK
388         || statement.bindText(2, secondDomain.string()) != SQLITE_OK
389         || statement.step() != SQLITE_ROW) {
390         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_insertDomainRelationshipStatement failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
391         ASSERT_NOT_REACHED();
392         return false;
393     }
394     bool relationshipExists = !!statement.getColumnInt(0);
395
396     int resetResult = statement.reset();
397     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
398     
399     return relationshipExists;
400 }
401
402 bool ResourceLoadStatisticsDatabaseStore::insertDomainRelationship(WebCore::SQLiteStatement& statement, unsigned domainID, const RegistrableDomain& topFrame)
403 {
404     ASSERT(!RunLoop::isMain());
405
406     if (statement.bindInt(1, domainID) != SQLITE_OK
407         || statement.bindText(2, topFrame.string()) != SQLITE_OK
408         || statement.step() != SQLITE_DONE) {
409         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_insertDomainRelationshipStatement failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
410         ASSERT_NOT_REACHED();
411         return false;
412     }
413
414     int resetResult = statement.reset();
415     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
416     
417     return true;
418 }
419
420 #ifndef NDEBUG
421 bool ResourceLoadStatisticsDatabaseStore::confirmDomainDoesNotExist(const RegistrableDomain& domain) const
422 {
423     ASSERT(!RunLoop::isMain());
424
425     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK) {
426         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::confirmDomainDoesNotExist failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
427         ASSERT_NOT_REACHED();
428         return false;
429     }
430         
431     if (m_domainIDFromStringStatement.step() == SQLITE_ROW) {
432         int resetResult = m_domainIDFromStringStatement.reset();
433         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
434         return false;
435     }
436
437     int resetResult = m_domainIDFromStringStatement.reset();
438     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
439     
440     return true;
441 }
442 #endif
443
444 Optional<unsigned> ResourceLoadStatisticsDatabaseStore::domainID(const RegistrableDomain& domain) const
445 {
446     ASSERT(!RunLoop::isMain());
447
448     unsigned domainID = 0;
449
450     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK) {
451         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::domainIDFromString failed, error message: %{public}s", this, m_database.lastErrorMsg());
452         return WTF::nullopt;
453     }
454     
455     if (m_domainIDFromStringStatement.step() != SQLITE_ROW) {
456         int resetResult = m_domainIDFromStringStatement.reset();
457         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
458         return WTF::nullopt;
459     }
460
461     domainID = m_domainIDFromStringStatement.getColumnInt(0);
462
463     int resetResult = m_domainIDFromStringStatement.reset();
464     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
465     
466     return domainID;
467 }
468
469 void ResourceLoadStatisticsDatabaseStore::insertDomainRelationships(const ResourceLoadStatistics& loadStatistics)
470 {
471     ASSERT(!RunLoop::isMain());
472     auto registrableDomainID = domainID(loadStatistics.registrableDomain);
473     
474     if (!registrableDomainID)
475         return;
476     
477     for (auto& topFrameDomain : loadStatistics.storageAccessUnderTopFrameDomains)
478         insertDomainRelationship(m_storageAccessUnderTopFrameDomainsStatement, registrableDomainID.value(), topFrameDomain);
479
480     for (auto& toDomain : loadStatistics.topFrameUniqueRedirectsTo)
481         insertDomainRelationship(m_topFrameUniqueRedirectsTo, registrableDomainID.value(), toDomain);
482     
483     for (auto& fromDomain : loadStatistics.topFrameUniqueRedirectsFrom)
484         insertDomainRelationship(m_topFrameUniqueRedirectsFrom, registrableDomainID.value(), fromDomain);
485     
486     for (auto& topFrameDomain : loadStatistics.subframeUnderTopFrameDomains)
487         insertDomainRelationship(m_subframeUnderTopFrameDomains, registrableDomainID.value(), topFrameDomain);
488     
489     for (auto& topFrameDomain : loadStatistics.subresourceUnderTopFrameDomains)
490         insertDomainRelationship(m_subresourceUnderTopFrameDomains, registrableDomainID.value(), topFrameDomain);
491     
492     for (auto& toDomain : loadStatistics.subresourceUniqueRedirectsTo)
493         insertDomainRelationship(m_subresourceUniqueRedirectsTo, registrableDomainID.value(), toDomain);
494     
495     for (auto& fromDomain : loadStatistics.subresourceUniqueRedirectsFrom)
496         insertDomainRelationship(m_subresourceUniqueRedirectsFrom, registrableDomainID.value(), fromDomain);
497 }
498
499 void ResourceLoadStatisticsDatabaseStore::populateFromMemoryStore(const ResourceLoadStatisticsMemoryStore& memoryStore)
500 {
501     ASSERT(!RunLoop::isMain());
502
503     if (!isEmpty())
504         return;
505
506     auto& statisticsMap = memoryStore.data();
507     for (const auto& statistic : statisticsMap)
508         insertObservedDomain(statistic.value);
509
510     // Make a separate pass for inter-domain relationships so we
511     // can refer to the ObservedDomain table entries
512     for (auto& statistic : statisticsMap)
513         insertDomainRelationships(statistic.value);
514 }
515
516 void ResourceLoadStatisticsDatabaseStore::calculateAndSubmitTelemetry() const
517 {
518     ASSERT(!RunLoop::isMain());
519
520     // FIXME(195088): Implement for Database version.
521 }
522
523 static String domainsToString(const HashSet<RegistrableDomain>& domains)
524 {
525     StringBuilder builder;
526     for (const auto& domainName : domains) {
527         if (!builder.isEmpty())
528             builder.appendLiteral(", ");
529         builder.append('"');
530         builder.append(domainName.string());
531         builder.append('"');
532     }
533
534     return builder.toString();
535 }
536
537 void ResourceLoadStatisticsDatabaseStore::incrementRecordsDeletedCountForDomains(HashSet<RegistrableDomain>&& domains)
538 {
539     ASSERT(!RunLoop::isMain());
540
541     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET dataRecordsRemoved = dataRecordsRemoved + 1 WHERE registrableDomain IN (", domainsToString(domains), ")"));
542     if (domainsToUpdateStatement.prepare() != SQLITE_OK
543         || domainsToUpdateStatement.step() != SQLITE_DONE) {
544         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::incrementStatisticsForDomains failed, error message: %{public}s", this, m_database.lastErrorMsg());
545         ASSERT_NOT_REACHED();
546     }
547 }
548
549 unsigned ResourceLoadStatisticsDatabaseStore::recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(unsigned primaryDomainID, StdSet<unsigned>& nonPrevalentRedirectionSources, unsigned numberOfRecursiveCalls)
550 {
551     ASSERT(!RunLoop::isMain());
552
553     if (numberOfRecursiveCalls >= maxNumberOfRecursiveCallsInRedirectTraceBack) {
554         RELEASE_LOG(ResourceLoadStatistics, "Hit %u recursive calls in redirect backtrace. Returning early.", maxNumberOfRecursiveCallsInRedirectTraceBack);
555         return numberOfRecursiveCalls;
556     }
557     
558     ++numberOfRecursiveCalls;
559
560     StdSet<unsigned> newlyIdentifiedDomains;
561     SQLiteStatement findSubresources(m_database, "SELECT SubresourceUniqueRedirectsFrom.fromDomainID from SubresourceUniqueRedirectsFrom INNER JOIN ObservedDomains ON ObservedDomains.domainID = SubresourceUniqueRedirectsFrom.fromDomainID WHERE subresourceDomainID = ? AND ObservedDomains.isPrevalent = 0"_s);
562     if (findSubresources.prepare() != SQLITE_OK
563         || findSubresources.bindInt(1, primaryDomainID) != SQLITE_OK) {
564         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain failed, error message: %{public}s", this, m_database.lastErrorMsg());
565         ASSERT_NOT_REACHED();
566         return 0;
567     }
568     
569     while (findSubresources.step() == SQLITE_ROW) {
570         int newDomainID = findSubresources.getColumnInt(0);
571         auto insertResult = nonPrevalentRedirectionSources.insert(newDomainID);
572         if (insertResult.second)
573             newlyIdentifiedDomains.insert(newDomainID);
574     }
575
576     SQLiteStatement findTopFrames(m_database, "SELECT TopFrameUniqueRedirectsFrom.fromDomainID from TopFrameUniqueRedirectsFrom INNER JOIN ObservedDomains ON ObservedDomains.domainID = TopFrameUniqueRedirectsFrom.fromDomainID WHERE targetDomainID = ? AND ObservedDomains.isPrevalent = 0"_s);
577     if (findTopFrames.prepare() != SQLITE_OK
578         || findTopFrames.bindInt(1, primaryDomainID) != SQLITE_OK) {
579         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain failed, error message: %{public}s", this, m_database.lastErrorMsg());
580         ASSERT_NOT_REACHED();
581         return 0;
582     }
583     
584     while (findTopFrames.step() == SQLITE_ROW) {
585         int newDomainID = findTopFrames.getColumnInt(0);
586         auto insertResult = nonPrevalentRedirectionSources.insert(newDomainID);
587         if (insertResult.second)
588             newlyIdentifiedDomains.insert(newDomainID);
589     }
590
591     if (newlyIdentifiedDomains.empty())
592         return numberOfRecursiveCalls;
593
594     for (auto domainID : newlyIdentifiedDomains)
595         numberOfRecursiveCalls = recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(domainID, nonPrevalentRedirectionSources, numberOfRecursiveCalls);
596
597     return numberOfRecursiveCalls;
598 }
599
600 template <typename IteratorType>
601 static String buildList(const WTF::IteratorRange<IteratorType>& values)
602 {
603     StringBuilder builder;
604     for (auto domainID : values) {
605         if (!builder.isEmpty())
606             builder.appendLiteral(", ");
607         builder.appendNumber(domainID);
608     }
609     return builder.toString();
610 }
611
612 void ResourceLoadStatisticsDatabaseStore::markAsPrevalentIfHasRedirectedToPrevalent()
613 {
614     ASSERT(!RunLoop::isMain());
615
616     StdSet<unsigned> prevalentDueToRedirect;
617     SQLiteStatement subresourceRedirectStatement(m_database, "SELECT DISTINCT SubresourceUniqueRedirectsTo.subresourceDomainID FROM SubresourceUniqueRedirectsTo JOIN ObservedDomains ON ObservedDomains.domainID = SubresourceUniqueRedirectsTo.toDomainID AND ObservedDomains.isPrevalent = 1"_s);
618     if (subresourceRedirectStatement.prepare() == SQLITE_OK) {
619         while (subresourceRedirectStatement.step() == SQLITE_ROW)
620             prevalentDueToRedirect.insert(subresourceRedirectStatement.getColumnInt(0));
621     }
622
623     SQLiteStatement topFrameRedirectStatement(m_database, "SELECT DISTINCT TopFrameUniqueRedirectsTo.sourceDomainID FROM TopFrameUniqueRedirectsTo JOIN ObservedDomains ON ObservedDomains.domainID = TopFrameUniqueRedirectsTo.toDomainID AND ObservedDomains.isPrevalent = 1"_s);
624     if (topFrameRedirectStatement.prepare() == SQLITE_OK) {
625         while (topFrameRedirectStatement.step() == SQLITE_ROW)
626             prevalentDueToRedirect.insert(topFrameRedirectStatement.getColumnInt(0));
627     }
628
629     SQLiteStatement markPrevalentStatement(m_database, makeString("UPDATE ObservedDomains SET isPrevalent = 1 WHERE domainID IN (", buildList(WTF::IteratorRange<StdSet<unsigned>::iterator>(prevalentDueToRedirect.begin(), prevalentDueToRedirect.end())), ")"));
630     if (markPrevalentStatement.prepare() != SQLITE_OK
631         || markPrevalentStatement.step() != SQLITE_DONE) {
632         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::markAsPrevalentIfHasRedirectedToPrevalent failed to execute, error message: %{public}s", this, m_database.lastErrorMsg());
633         ASSERT_NOT_REACHED();
634     }
635 }
636
637 HashMap<unsigned, ResourceLoadStatisticsDatabaseStore::NotVeryPrevalentResources> ResourceLoadStatisticsDatabaseStore::findNotVeryPrevalentResources()
638 {
639     ASSERT(!RunLoop::isMain());
640
641     HashMap<unsigned, NotVeryPrevalentResources> results;
642     SQLiteStatement notVeryPrevalentResourcesStatement(m_database, "SELECT domainID, registrableDomain, isPrevalent FROM ObservedDomains WHERE isVeryPrevalent = 0"_s);
643     if (notVeryPrevalentResourcesStatement.prepare() == SQLITE_OK) {
644         while (notVeryPrevalentResourcesStatement.step() == SQLITE_ROW) {
645             unsigned key = static_cast<unsigned>(notVeryPrevalentResourcesStatement.getColumnInt(0));
646             NotVeryPrevalentResources value({ RegistrableDomain::uncheckedCreateFromRegistrableDomainString(notVeryPrevalentResourcesStatement.getColumnText(1))
647                 , notVeryPrevalentResourcesStatement.getColumnInt(2) ? ResourceLoadPrevalence::High : ResourceLoadPrevalence::Low
648                 , 0, 0, 0, 0 });
649             results.add(key, value);
650         }
651     }
652
653     StringBuilder builder;
654     for (auto value : results.keys()) {
655         if (!builder.isEmpty())
656             builder.appendLiteral(", ");
657         builder.appendNumber(value);
658     }
659
660     auto domainIDsOfInterest = builder.toString();
661
662     SQLiteStatement subresourceUnderTopFrameDomainsStatement(m_database, makeString("SELECT subresourceDomainID, COUNT(topFrameDomainID) FROM SubresourceUnderTopFrameDomains WHERE subresourceDomainID IN (", domainIDsOfInterest, ") GROUP BY subresourceDomainID"));
663     if (subresourceUnderTopFrameDomainsStatement.prepare() == SQLITE_OK) {
664         while (subresourceUnderTopFrameDomainsStatement.step() == SQLITE_ROW) {
665             unsigned domainID = static_cast<unsigned>(subresourceUnderTopFrameDomainsStatement.getColumnInt(0));
666             auto result = results.find(domainID);
667             if (result != results.end())
668                 result->value.subresourceUnderTopFrameDomainsCount = static_cast<unsigned>(subresourceUnderTopFrameDomainsStatement.getColumnInt(1));
669         }
670     }
671
672     SQLiteStatement subresourceUniqueRedirectsToCountStatement(m_database, makeString("SELECT subresourceDomainID, COUNT(toDomainID) FROM SubresourceUniqueRedirectsTo WHERE subresourceDomainID IN (", domainIDsOfInterest, ") GROUP BY subresourceDomainID"));
673     if (subresourceUniqueRedirectsToCountStatement.prepare() == SQLITE_OK) {
674         while (subresourceUniqueRedirectsToCountStatement.step() == SQLITE_ROW) {
675             unsigned domainID = static_cast<unsigned>(subresourceUniqueRedirectsToCountStatement.getColumnInt(0));
676             auto result = results.find(domainID);
677             if (result != results.end())
678                 result->value.subresourceUniqueRedirectsToCount = static_cast<unsigned>(subresourceUniqueRedirectsToCountStatement.getColumnInt(1));
679         }
680     }
681
682     SQLiteStatement subframeUnderTopFrameDomainsCountStatement(m_database, makeString("SELECT subframeDomainID, COUNT(topFrameDomainID) FROM SubframeUnderTopFrameDomains WHERE subframeDomainID IN (", domainIDsOfInterest, ") GROUP BY subframeDomainID"));
683     if (subframeUnderTopFrameDomainsCountStatement.prepare() == SQLITE_OK) {
684         while (subframeUnderTopFrameDomainsCountStatement.step() == SQLITE_ROW) {
685             unsigned domainID = static_cast<unsigned>(subframeUnderTopFrameDomainsCountStatement.getColumnInt(0));
686             auto result = results.find(domainID);
687             if (result != results.end())
688                 result->value.subframeUnderTopFrameDomainsCount = static_cast<unsigned>(subframeUnderTopFrameDomainsCountStatement.getColumnInt(1));
689         }
690     }
691
692     SQLiteStatement topFrameUniqueRedirectsToCountStatement(m_database, makeString("SELECT sourceDomainID, COUNT(toDomainID) FROM TopFrameUniqueRedirectsTo WHERE sourceDomainID IN (", domainIDsOfInterest, ") GROUP BY sourceDomainID"));
693     if (topFrameUniqueRedirectsToCountStatement.prepare() == SQLITE_OK) {
694         while (topFrameUniqueRedirectsToCountStatement.step() == SQLITE_ROW) {
695             unsigned domainID = static_cast<unsigned>(topFrameUniqueRedirectsToCountStatement.getColumnInt(0));
696             auto result = results.find(domainID);
697             if (result != results.end())
698                 result->value.topFrameUniqueRedirectsToCount = static_cast<unsigned>(topFrameUniqueRedirectsToCountStatement.getColumnInt(1));
699         }
700     }
701
702     return results;
703 }
704
705 void ResourceLoadStatisticsDatabaseStore::reclassifyResources()
706 {
707     ASSERT(!RunLoop::isMain());
708
709     auto notVeryPrevalentResources = findNotVeryPrevalentResources();
710
711     for (auto& resourceStatistic : notVeryPrevalentResources.values()) {
712         if (shouldSkip(resourceStatistic.registrableDomain))
713             continue;
714
715         auto newPrevalence = classifier().calculateResourcePrevalence(resourceStatistic.subresourceUnderTopFrameDomainsCount, resourceStatistic.subresourceUniqueRedirectsToCount, resourceStatistic.subframeUnderTopFrameDomainsCount, resourceStatistic.topFrameUniqueRedirectsToCount, resourceStatistic.prevalence);
716         if (newPrevalence != resourceStatistic.prevalence)
717             setPrevalentResource(resourceStatistic.registrableDomain, newPrevalence);
718     }
719 }
720
721 void ResourceLoadStatisticsDatabaseStore::classifyPrevalentResources()
722 {
723     ASSERT(!RunLoop::isMain());
724     ensurePrevalentResourcesForDebugMode();
725     markAsPrevalentIfHasRedirectedToPrevalent();
726     reclassifyResources();
727 }
728
729 void ResourceLoadStatisticsDatabaseStore::syncStorageIfNeeded()
730 {
731     ASSERT(!RunLoop::isMain());
732     m_database.runVacuumCommand();
733 }
734
735 void ResourceLoadStatisticsDatabaseStore::syncStorageImmediately()
736 {
737     ASSERT(!RunLoop::isMain());
738     m_database.runVacuumCommand();
739 }
740
741 void ResourceLoadStatisticsDatabaseStore::hasStorageAccess(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain, Optional<FrameIdentifier> frameID, PageIdentifier pageID, CompletionHandler<void(bool)>&& completionHandler)
742 {
743     ASSERT(!RunLoop::isMain());
744
745     ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
746
747     switch (cookieTreatmentForOrigin(subFrameDomain)) {
748     case CookieTreatmentResult::BlockAndPurge:
749         completionHandler(false);
750         return;
751     case CookieTreatmentResult::Allow:
752         // We should only return true if the context has asked for and been granted access.
753         completionHandler(false);
754         return;
755     case CookieTreatmentResult::BlockAndKeep:
756         // Do nothing. The below dispatch will complete the task.
757         break;
758     };
759
760     RunLoop::main().dispatch([store = makeRef(store()), subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, completionHandler = WTFMove(completionHandler)]() mutable {
761         store->callHasStorageAccessForFrameHandler(subFrameDomain, topFrameDomain, frameID.value(), pageID, [store = store.copyRef(), completionHandler = WTFMove(completionHandler)](bool result) mutable {
762             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler), result] () mutable {
763                 completionHandler(result);
764             });
765         });
766     });
767 }
768
769 void ResourceLoadStatisticsDatabaseStore::requestStorageAccess(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, FrameIdentifier frameID, PageIdentifier pageID, CompletionHandler<void(StorageAccessStatus)>&& completionHandler)
770 {
771     ASSERT(!RunLoop::isMain());
772
773     auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
774
775     switch (cookieTreatmentForOrigin(subFrameDomain)) {
776     case CookieTreatmentResult::BlockAndPurge: {
777         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "Cannot grant storage access to %{public}s since its cookies are blocked in third-party contexts and it has not received user interaction as first-party.", subFrameDomain.string().utf8().data());
778         completionHandler(StorageAccessStatus::CannotRequestAccess);
779         }
780         return;
781     case CookieTreatmentResult::BlockAndKeep: {
782         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "No need to grant storage access to %{public}s since its cookies are not blocked in third-party contexts.", subFrameDomain.string().utf8().data());
783         completionHandler(StorageAccessStatus::HasAccess);
784         }
785         return;
786     case CookieTreatmentResult::Allow:
787         // Do nothing
788         break;
789     };
790
791     auto userWasPromptedEarlier = hasUserGrantedStorageAccessThroughPrompt(subFrameStatus.second, topFrameDomain);
792     if (userWasPromptedEarlier == StorageAccessPromptWasShown::No) {
793         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "About to ask the user whether they want to grant storage access to %{public}s under %{public}s or not.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
794         completionHandler(StorageAccessStatus::RequiresUserPrompt);
795         return;
796     }
797
798     if (userWasPromptedEarlier == StorageAccessPromptWasShown::Yes)
799         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "Storage access was granted to %{public}s under %{public}s.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
800
801     SQLiteStatement incrementStorageAccess(m_database, "UPDATE ObservedDomains SET timesAccessedAsFirstPartyDueToStorageAccessAPI = timesAccessedAsFirstPartyDueToStorageAccessAPI + 1 WHERE domainID = ?");
802     if (incrementStorageAccess.prepare() != SQLITE_OK
803         || incrementStorageAccess.bindInt(1, subFrameStatus.second) != SQLITE_OK
804         || incrementStorageAccess.step() != SQLITE_DONE) {
805         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::requestStorageAccess failed, error message: %{public}s", this, m_database.lastErrorMsg());
806         ASSERT_NOT_REACHED();
807         return;
808     }
809     
810     grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, userWasPromptedEarlier, [completionHandler = WTFMove(completionHandler)] (StorageAccessWasGranted wasGranted) mutable {
811         completionHandler(wasGranted == StorageAccessWasGranted::Yes ? StorageAccessStatus::HasAccess : StorageAccessStatus::CannotRequestAccess);
812     });
813 }
814
815 void ResourceLoadStatisticsDatabaseStore::requestStorageAccessUnderOpener(DomainInNeedOfStorageAccess&& domainInNeedOfStorageAccess, PageIdentifier openerPageID, OpenerDomain&& openerDomain)
816 {
817     ASSERT(domainInNeedOfStorageAccess != openerDomain);
818     ASSERT(!RunLoop::isMain());
819
820     if (domainInNeedOfStorageAccess == openerDomain)
821         return;
822
823     ensureResourceStatisticsForRegistrableDomain(domainInNeedOfStorageAccess);
824
825     if (cookieTreatmentForOrigin(domainInNeedOfStorageAccess) != CookieTreatmentResult::Allow)
826         return;
827
828     RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "[Temporary combatibility fix] Storage access was granted for %{public}s under opener page from %{public}s, with user interaction in the opened window.", domainInNeedOfStorageAccess.string().utf8().data(), openerDomain.string().utf8().data());
829     grantStorageAccessInternal(WTFMove(domainInNeedOfStorageAccess), WTFMove(openerDomain), WTF::nullopt, openerPageID, StorageAccessPromptWasShown::No, [](StorageAccessWasGranted) { });
830 }
831
832 void ResourceLoadStatisticsDatabaseStore::grantStorageAccess(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, FrameIdentifier frameID, PageIdentifier pageID, StorageAccessPromptWasShown promptWasShown, CompletionHandler<void(StorageAccessWasGranted)>&& completionHandler)
833 {
834     ASSERT(!RunLoop::isMain());
835
836     if (promptWasShown == StorageAccessPromptWasShown::Yes) {
837         auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
838         ASSERT(subFrameStatus.first == AddedRecord::No);
839         ASSERT(hasHadUserInteraction(subFrameDomain, OperatingDatesWindow::Long));
840         insertDomainRelationship(m_storageAccessUnderTopFrameDomainsStatement, subFrameStatus.second, topFrameDomain);
841     }
842
843     grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, promptWasShown, WTFMove(completionHandler));
844 }
845
846 void ResourceLoadStatisticsDatabaseStore::grantStorageAccessInternal(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, Optional<FrameIdentifier> frameID, PageIdentifier pageID, StorageAccessPromptWasShown promptWasShownNowOrEarlier, CompletionHandler<void(StorageAccessWasGranted)>&& completionHandler)
847 {
848     ASSERT(!RunLoop::isMain());
849
850     if (subFrameDomain == topFrameDomain) {
851         completionHandler(StorageAccessWasGranted::Yes);
852         return;
853     }
854
855     if (promptWasShownNowOrEarlier == StorageAccessPromptWasShown::Yes) {
856 #ifndef NDEBUG
857         auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
858         ASSERT(subFrameStatus.first == AddedRecord::No);
859         ASSERT(hasHadUserInteraction(subFrameDomain, OperatingDatesWindow::Long));
860         ASSERT(hasUserGrantedStorageAccessThroughPrompt(subFrameStatus.second, topFrameDomain) == StorageAccessPromptWasShown::Yes);
861 #endif
862         setUserInteraction(subFrameDomain, true, WallTime::now());
863     }
864
865     RunLoop::main().dispatch([subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, store = makeRef(store()), completionHandler = WTFMove(completionHandler)]() mutable {
866         store->callGrantStorageAccessHandler(subFrameDomain, topFrameDomain, frameID, pageID, [completionHandler = WTFMove(completionHandler), store = store.copyRef()](StorageAccessWasGranted wasGranted) mutable {
867             store->statisticsQueue().dispatch([wasGranted, completionHandler = WTFMove(completionHandler)] () mutable {
868                 completionHandler(wasGranted);
869             });
870         });
871     });
872
873 }
874
875 void ResourceLoadStatisticsDatabaseStore::grandfatherDataForDomains(const HashSet<RegistrableDomain>& domains)
876 {
877     ASSERT(!RunLoop::isMain());
878
879     for (auto& registrableDomain : domains)
880         ensureResourceStatisticsForRegistrableDomain(registrableDomain);
881
882     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET grandfathered = 1 WHERE registrableDomain IN (", domainsToString(domains), ")"));
883     if (domainsToUpdateStatement.prepare() != SQLITE_OK
884         || domainsToUpdateStatement.step() != SQLITE_DONE) {
885         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::grandfatherDataForDomains failed, error message: %{public}s", this, m_database.lastErrorMsg());
886         ASSERT_NOT_REACHED();
887     }
888 }
889
890 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::ensurePrevalentResourcesForDebugMode()
891 {
892     ASSERT(!RunLoop::isMain());
893
894     if (!debugModeEnabled())
895         return { };
896
897     Vector<RegistrableDomain> primaryDomainsToBlock;
898     primaryDomainsToBlock.reserveInitialCapacity(2);
899
900     ensureResourceStatisticsForRegistrableDomain(debugStaticPrevalentResource());
901     setPrevalentResource(debugStaticPrevalentResource(), ResourceLoadPrevalence::High);
902     primaryDomainsToBlock.uncheckedAppend(debugStaticPrevalentResource());
903
904     if (!debugManualPrevalentResource().isEmpty()) {
905         ensureResourceStatisticsForRegistrableDomain(debugManualPrevalentResource());
906         setPrevalentResource(debugManualPrevalentResource(), ResourceLoadPrevalence::High);
907         primaryDomainsToBlock.uncheckedAppend(debugManualPrevalentResource());
908         RELEASE_LOG_INFO(ITPDebug, "Did set %{public}s as prevalent resource for the purposes of ITP Debug Mode.", debugManualPrevalentResource().string().utf8().data());
909     }
910
911     return primaryDomainsToBlock;
912 }
913
914 void ResourceLoadStatisticsDatabaseStore::logFrameNavigation(const RegistrableDomain& targetDomain, const RegistrableDomain& topFrameDomain, const RegistrableDomain& sourceDomain, bool isRedirect, bool isMainFrame)
915 {
916     ASSERT(!RunLoop::isMain());
917
918     bool areTargetAndTopFrameDomainsSameSite = targetDomain == topFrameDomain;
919     bool areTargetAndSourceDomainsSameSite = targetDomain == sourceDomain;
920
921     bool statisticsWereUpdated = false;
922     if (!isMainFrame && !(areTargetAndTopFrameDomainsSameSite || areTargetAndSourceDomainsSameSite)) {
923         auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
924         updateLastSeen(targetDomain, ResourceLoadStatistics::reduceTimeResolution(WallTime::now()));
925         ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
926         if (!relationshipExists(m_subframeUnderTopFrameDomainExists, targetResult.second, topFrameDomain)) {
927             insertDomainRelationship(m_subframeUnderTopFrameDomains, targetResult.second, topFrameDomain);
928             statisticsWereUpdated = true;
929         }
930     }
931
932     if (isRedirect && !areTargetAndSourceDomainsSameSite) {
933         if (isMainFrame) {
934             auto redirectingDomainResult = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
935             auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
936
937             if (!relationshipExists(m_topFrameUniqueRedirectsToExists, redirectingDomainResult.second, targetDomain)) {
938                 insertDomainRelationship(m_topFrameUniqueRedirectsTo, redirectingDomainResult.second, targetDomain);
939                 statisticsWereUpdated = true;
940             }
941
942             if (!relationshipExists(m_topFrameUniqueRedirectsFromExists, targetResult.second, sourceDomain)) {
943                 insertDomainRelationship(m_topFrameUniqueRedirectsFrom, targetResult.second, sourceDomain);
944                 statisticsWereUpdated = true;
945             }
946         } else {
947             auto redirectingDomainResult = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
948             auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
949             if (!relationshipExists(m_subresourceUniqueRedirectsToExists, redirectingDomainResult.second, targetDomain)) {
950                 insertDomainRelationship(m_subresourceUniqueRedirectsTo, redirectingDomainResult.second, targetDomain);
951                 statisticsWereUpdated = true;
952             }
953
954             if (!relationshipExists(m_subresourceUniqueRedirectsFromExists, targetResult.second, sourceDomain)) {
955                 insertDomainRelationship(m_subresourceUniqueRedirectsFrom, targetResult.second, sourceDomain);
956                 statisticsWereUpdated = true;
957             }
958         }
959     }
960
961     if (statisticsWereUpdated)
962         scheduleStatisticsProcessingRequestIfNecessary();
963 }
964
965 void ResourceLoadStatisticsDatabaseStore::logCrossSiteLoadWithLinkDecoration(const NavigatedFromDomain& fromDomain, const NavigatedToDomain& toDomain)
966 {
967     ASSERT(!RunLoop::isMain());
968     ASSERT(fromDomain != toDomain);
969
970     auto fromDomainResult = ensureResourceStatisticsForRegistrableDomain(fromDomain);
971
972     if (!relationshipExists(m_topFrameLinkDecorationsFromExists, fromDomainResult.second, toDomain)) {
973         insertDomainRelationship(m_topFrameLinkDecorationsFrom, fromDomainResult.second, toDomain);
974         scheduleStatisticsProcessingRequestIfNecessary();
975     }
976 }
977
978 void ResourceLoadStatisticsDatabaseStore::setUserInteraction(const RegistrableDomain& domain, bool hadUserInteraction, WallTime mostRecentInteraction)
979 {
980     ASSERT(!RunLoop::isMain());
981
982     if (m_mostRecentUserInteractionStatement.bindInt(1, hadUserInteraction) != SQLITE_OK
983         || m_mostRecentUserInteractionStatement.bindDouble(2, mostRecentInteraction.secondsSinceEpoch().value()) != SQLITE_OK
984         || m_mostRecentUserInteractionStatement.bindText(3, domain.string()) != SQLITE_OK
985         || m_mostRecentUserInteractionStatement.step() != SQLITE_DONE) {
986         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::setUserInteraction, error message: %{public}s", this, m_database.lastErrorMsg());
987         ASSERT_NOT_REACHED();
988         return;
989     }
990
991     int resetResult = m_mostRecentUserInteractionStatement.reset();
992     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
993 }
994
995 void ResourceLoadStatisticsDatabaseStore::logUserInteraction(const TopFrameDomain& domain)
996 {
997     ASSERT(!RunLoop::isMain());
998
999     ensureResourceStatisticsForRegistrableDomain(domain);
1000     setUserInteraction(domain, true, WallTime::now());
1001 }
1002
1003 void ResourceLoadStatisticsDatabaseStore::clearUserInteraction(const RegistrableDomain& domain)
1004 {
1005     ASSERT(!RunLoop::isMain());
1006
1007     auto targetResult = ensureResourceStatisticsForRegistrableDomain(domain);
1008     setUserInteraction(domain, false, { });
1009
1010     SQLiteStatement removeStorageAccess(m_database, "DELETE FROM StorageAccessUnderTopFrameDomains WHERE domainID = ?");
1011     if (removeStorageAccess.prepare() != SQLITE_OK
1012         || removeStorageAccess.bindInt(1, targetResult.second) != SQLITE_OK
1013         || removeStorageAccess.step() != SQLITE_DONE) {
1014         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::logUserInteraction failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1015         ASSERT_NOT_REACHED();
1016     }
1017 }
1018
1019 bool ResourceLoadStatisticsDatabaseStore::hasHadUserInteraction(const RegistrableDomain& domain, OperatingDatesWindow operatingDatesWindow)
1020 {
1021     ASSERT(!RunLoop::isMain());
1022
1023     if (m_hadUserInteractionStatement.bindText(1, domain.string()) != SQLITE_OK
1024         || m_hadUserInteractionStatement.step() != SQLITE_ROW) {
1025         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_hadUserInteractionStatement failed, error message: %{public}s", this, m_database.lastErrorMsg());
1026
1027         int resetResult = m_hadUserInteractionStatement.reset();
1028         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1029         return false;
1030     }
1031
1032     bool hadUserInteraction = !!m_hadUserInteractionStatement.getColumnInt(0);
1033     if (!hadUserInteraction) {
1034         int resetResult = m_hadUserInteractionStatement.reset();
1035         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1036         return false;
1037     }
1038     
1039     WallTime mostRecentUserInteractionTime = WallTime::fromRawSeconds(m_hadUserInteractionStatement.getColumnDouble(1));
1040
1041     if (hasStatisticsExpired(mostRecentUserInteractionTime, operatingDatesWindow)) {
1042         // Drop privacy sensitive data because we no longer need it.
1043         // Set timestamp to 0 so that statistics merge will know
1044         // it has been reset as opposed to its default -1.
1045         clearUserInteraction(domain);
1046         hadUserInteraction = false;
1047     }
1048     
1049     int resetResult = m_hadUserInteractionStatement.reset();
1050     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1051
1052     return hadUserInteraction;
1053 }
1054
1055 void ResourceLoadStatisticsDatabaseStore::setPrevalentResource(const RegistrableDomain& domain, ResourceLoadPrevalence newPrevalence)
1056 {
1057     ASSERT(!RunLoop::isMain());
1058     if (shouldSkip(domain))
1059         return;
1060
1061     if (m_updatePrevalentResourceStatement.bindInt(1, 1) != SQLITE_OK
1062         || m_updatePrevalentResourceStatement.bindText(2, domain.string()) != SQLITE_OK
1063         || m_updatePrevalentResourceStatement.step() != SQLITE_DONE) {
1064         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_updatePrevalentResourceStatement failed, error message: %{public}s", this, m_database.lastErrorMsg());
1065         ASSERT_NOT_REACHED();
1066         return;
1067     }
1068     
1069     int resetResult = m_updatePrevalentResourceStatement.reset();
1070     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1071
1072     if (newPrevalence == ResourceLoadPrevalence::VeryHigh) {
1073         if (m_updateVeryPrevalentResourceStatement.bindInt(1, 1) != SQLITE_OK
1074             || m_updateVeryPrevalentResourceStatement.bindText(2, domain.string()) != SQLITE_OK
1075             || m_updateVeryPrevalentResourceStatement.step() != SQLITE_DONE) {
1076             RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_updateVeryPrevalentResourceStatement failed, error message: %{public}s", this, m_database.lastErrorMsg());
1077             ASSERT_NOT_REACHED();
1078             return;
1079         }
1080
1081         int resetResult = m_updateVeryPrevalentResourceStatement.reset();
1082         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1083     }
1084
1085     StdSet<unsigned> nonPrevalentRedirectionSources;
1086     recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(domainID(domain).value(), nonPrevalentRedirectionSources, 0);
1087     setDomainsAsPrevalent(WTFMove(nonPrevalentRedirectionSources));
1088 }
1089
1090 void ResourceLoadStatisticsDatabaseStore::setDomainsAsPrevalent(StdSet<unsigned>&& domains)
1091 {
1092     ASSERT(!RunLoop::isMain());
1093
1094     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET isPrevalent = 1 WHERE domainID IN (", buildList(WTF::IteratorRange<StdSet<unsigned>::iterator>(domains.begin(), domains.end())), ")"));
1095     if (domainsToUpdateStatement.prepare() != SQLITE_OK
1096         || domainsToUpdateStatement.step() != SQLITE_DONE) {
1097         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::setDomainsAsPrevalent failed, error message: %{public}s", this, m_database.lastErrorMsg());
1098         ASSERT_NOT_REACHED();
1099         return;
1100     }
1101 }
1102
1103 void ResourceLoadStatisticsDatabaseStore::dumpResourceLoadStatistics(CompletionHandler<void(const String&)>&& completionHandler)
1104 {
1105     ASSERT(!RunLoop::isMain());
1106
1107     // FIXME(195088): Implement SQLite-based dumping routines.
1108     ASSERT_NOT_REACHED();
1109     completionHandler({ });
1110 }
1111
1112 bool ResourceLoadStatisticsDatabaseStore::predicateValueForDomain(WebCore::SQLiteStatement& predicateStatement, const RegistrableDomain& domain) const
1113 {
1114     ASSERT(!RunLoop::isMain());
1115     
1116     if (predicateStatement.bindText(1, domain.string()) != SQLITE_OK
1117         || predicateStatement.step() != SQLITE_ROW) {
1118
1119         int resetResult = predicateStatement.reset();
1120         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1121
1122         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::predicateValueForDomain failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1123         return false;
1124     }
1125
1126     bool result = !!predicateStatement.getColumnInt(0);
1127     int resetResult = predicateStatement.reset();
1128     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1129     return result;
1130 }
1131
1132 bool ResourceLoadStatisticsDatabaseStore::isPrevalentResource(const RegistrableDomain& domain) const
1133 {
1134     ASSERT(!RunLoop::isMain());
1135
1136     if (shouldSkip(domain))
1137         return false;
1138
1139     return predicateValueForDomain(m_isPrevalentResourceStatement, domain);
1140 }
1141
1142 bool ResourceLoadStatisticsDatabaseStore::isVeryPrevalentResource(const RegistrableDomain& domain) const
1143 {
1144     ASSERT(!RunLoop::isMain());
1145
1146     if (shouldSkip(domain))
1147         return false;
1148
1149     return predicateValueForDomain(m_isVeryPrevalentResourceStatement, domain);
1150 }
1151
1152 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsSubresourceUnder(const SubResourceDomain& subresourceDomain, const TopFrameDomain& topFrameDomain) const
1153 {
1154     ASSERT(!RunLoop::isMain());
1155
1156     return relationshipExists(m_subresourceUnderTopFrameDomainExists, domainID(subresourceDomain), topFrameDomain);
1157 }
1158
1159 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsSubFrameUnder(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain) const
1160 {
1161     ASSERT(!RunLoop::isMain());
1162
1163     return relationshipExists(m_subframeUnderTopFrameDomainExists, domainID(subFrameDomain), topFrameDomain);
1164 }
1165
1166 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsRedirectingTo(const RedirectedFromDomain& redirectedFromDomain, const RedirectedToDomain& redirectedToDomain) const
1167 {
1168     ASSERT(!RunLoop::isMain());
1169
1170     return relationshipExists(m_subresourceUniqueRedirectsToExists, domainID(redirectedFromDomain), redirectedToDomain);
1171 }
1172
1173 void ResourceLoadStatisticsDatabaseStore::clearPrevalentResource(const RegistrableDomain& domain)
1174 {
1175     ASSERT(!RunLoop::isMain());
1176
1177     ensureResourceStatisticsForRegistrableDomain(domain);
1178     
1179     if (m_clearPrevalentResourceStatement.bindText(1, domain.string()) != SQLITE_OK
1180         || m_clearPrevalentResourceStatement.step() != SQLITE_DONE) {
1181         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::clearPrevalentResource, error message: %{public}s", this, m_database.lastErrorMsg());
1182         ASSERT_NOT_REACHED();
1183         return;
1184     }
1185     
1186     int resetResult = m_clearPrevalentResourceStatement.reset();
1187     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1188 }
1189
1190 void ResourceLoadStatisticsDatabaseStore::setGrandfathered(const RegistrableDomain& domain, bool value)
1191 {
1192     ASSERT(!RunLoop::isMain());
1193
1194     ensureResourceStatisticsForRegistrableDomain(domain);
1195     
1196     if (m_updateGrandfatheredStatement.bindInt(1, value) != SQLITE_OK
1197         || m_updateGrandfatheredStatement.bindText(1, domain.string()) != SQLITE_OK
1198         || m_updateGrandfatheredStatement.step() != SQLITE_DONE) {
1199         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::setGrandfathered failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1200         ASSERT_NOT_REACHED();
1201         return;
1202     }
1203     
1204     int resetResult = m_updateGrandfatheredStatement.reset();
1205     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1206 }
1207
1208 bool ResourceLoadStatisticsDatabaseStore::isGrandfathered(const RegistrableDomain& domain) const
1209 {
1210     ASSERT(!RunLoop::isMain());
1211
1212     return predicateValueForDomain(m_isGrandfatheredStatement, domain);
1213 }
1214
1215 void ResourceLoadStatisticsDatabaseStore::setSubframeUnderTopFrameDomain(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain)
1216 {
1217     ASSERT(!RunLoop::isMain());
1218
1219     auto result = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
1220
1221     // For consistency, make sure we also have a statistics entry for the top frame domain.
1222     ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1223
1224     insertDomainRelationship(m_subframeUnderTopFrameDomains, result.second, topFrameDomain);
1225 }
1226
1227 void ResourceLoadStatisticsDatabaseStore::setSubresourceUnderTopFrameDomain(const SubResourceDomain& subresourceDomain, const TopFrameDomain& topFrameDomain)
1228 {
1229     ASSERT(!RunLoop::isMain());
1230
1231     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1232
1233     // For consistency, make sure we also have a statistics entry for the top frame domain.
1234     ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1235
1236     insertDomainRelationship(m_subresourceUnderTopFrameDomains, result.second, topFrameDomain);
1237 }
1238
1239 void ResourceLoadStatisticsDatabaseStore::setSubresourceUniqueRedirectTo(const SubResourceDomain& subresourceDomain, const RedirectDomain& redirectDomain)
1240 {
1241     ASSERT(!RunLoop::isMain());
1242
1243     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1244
1245     // For consistency, make sure we also have a statistics entry for the redirect domain.
1246     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1247
1248     insertDomainRelationship(m_subresourceUniqueRedirectsTo, result.second, redirectDomain);
1249 }
1250
1251 void ResourceLoadStatisticsDatabaseStore::setSubresourceUniqueRedirectFrom(const SubResourceDomain& subresourceDomain, const RedirectDomain& redirectDomain)
1252 {
1253     ASSERT(!RunLoop::isMain());
1254
1255     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1256
1257     // For consistency, make sure we also have a statistics entry for the redirect domain.
1258     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1259
1260     insertDomainRelationship(m_subresourceUniqueRedirectsFrom, result.second, redirectDomain);
1261 }
1262
1263 void ResourceLoadStatisticsDatabaseStore::setTopFrameUniqueRedirectTo(const TopFrameDomain& topFrameDomain, const RedirectDomain& redirectDomain)
1264 {
1265     ASSERT(!RunLoop::isMain());
1266
1267     auto result = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1268
1269     // For consistency, make sure we also have a statistics entry for the redirect domain.
1270     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1271
1272     insertDomainRelationship(m_topFrameUniqueRedirectsTo, result.second, redirectDomain);
1273 }
1274
1275 void ResourceLoadStatisticsDatabaseStore::setTopFrameUniqueRedirectFrom(const TopFrameDomain& topFrameDomain, const RedirectDomain& redirectDomain)
1276 {
1277     ASSERT(!RunLoop::isMain());
1278
1279     auto result = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1280
1281     // For consistency, make sure we also have a statistics entry for the redirect domain.
1282     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1283
1284     insertDomainRelationship(m_topFrameUniqueRedirectsFrom, result.second, redirectDomain);
1285 }
1286
1287 std::pair<ResourceLoadStatisticsDatabaseStore::AddedRecord, unsigned> ResourceLoadStatisticsDatabaseStore::ensureResourceStatisticsForRegistrableDomain(const RegistrableDomain& domain)
1288 {
1289     ASSERT(!RunLoop::isMain());
1290
1291     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK) {
1292         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::ensureResourceStatisticsForRegistrableDomain failed, error message: %{public}s", this, m_database.lastErrorMsg());
1293         ASSERT_NOT_REACHED();
1294         return { AddedRecord::No, 0 };
1295     }
1296     
1297     if (m_domainIDFromStringStatement.step() == SQLITE_ROW) {
1298         unsigned domainID = m_domainIDFromStringStatement.getColumnInt(0);
1299
1300         int resetResult = m_domainIDFromStringStatement.reset();
1301         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1302         return { AddedRecord::No, domainID };
1303     }
1304
1305     int resetResult = m_domainIDFromStringStatement.reset();
1306     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1307
1308     ResourceLoadStatistics newObservation(domain);
1309     insertObservedDomain(newObservation);
1310
1311     return { AddedRecord::Yes, domainID(domain).value() };
1312 }
1313
1314 void ResourceLoadStatisticsDatabaseStore::clearDatabaseContents()
1315 {
1316     m_database.clearAllTables();
1317
1318     if (!createSchema()) {
1319         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::clearDatabaseContents failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
1320         ASSERT_NOT_REACHED();
1321         return;
1322     }
1323 }
1324
1325 void ResourceLoadStatisticsDatabaseStore::clear(CompletionHandler<void()>&& completionHandler)
1326 {
1327     ASSERT(!RunLoop::isMain());
1328
1329     clearDatabaseContents();
1330     clearOperatingDates();
1331
1332     auto callbackAggregator = CallbackAggregator::create(WTFMove(completionHandler));
1333
1334     removeAllStorageAccess([callbackAggregator = callbackAggregator.copyRef()] { });
1335
1336     auto registrableDomainsToBlockAndDeleteCookiesFor = ensurePrevalentResourcesForDebugMode();
1337     RegistrableDomainsToBlockCookiesFor domainsToBlock { registrableDomainsToBlockAndDeleteCookiesFor, { } };
1338     updateCookieBlockingForDomains(domainsToBlock, [callbackAggregator = callbackAggregator.copyRef()] { });
1339 }
1340
1341 ResourceLoadStatisticsDatabaseStore::CookieTreatmentResult ResourceLoadStatisticsDatabaseStore::cookieTreatmentForOrigin(const RegistrableDomain& domain) const
1342 {
1343     ASSERT(!RunLoop::isMain());
1344
1345     SQLiteStatement statement(m_database, "SELECT isPrevalent, hadUserInteraction FROM ObservedDomains WHERE registrableDomain = ?");
1346     if (statement.prepare() != SQLITE_OK
1347         || statement.bindText(1, domain.string()) != SQLITE_OK) {
1348         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::cookieTreatmentForOrigin failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1349         ASSERT_NOT_REACHED();
1350     }
1351     
1352     if (statement.step() != SQLITE_ROW)
1353         return CookieTreatmentResult::Allow;
1354     
1355     bool isPrevalent = !!statement.getColumnInt(0);
1356     if (!isPrevalent)
1357         return CookieTreatmentResult::Allow;
1358
1359     bool hadUserInteraction = statement.getColumnInt(1) ? true : false;
1360     return hadUserInteraction ? CookieTreatmentResult::BlockAndKeep : CookieTreatmentResult::BlockAndPurge;
1361 }
1362     
1363 StorageAccessPromptWasShown ResourceLoadStatisticsDatabaseStore::hasUserGrantedStorageAccessThroughPrompt(unsigned requestingDomainID, const RegistrableDomain& firstPartyDomain) const
1364 {
1365     ASSERT(!RunLoop::isMain());
1366
1367     auto firstPartyPrimaryDomainID = domainID(firstPartyDomain).value();
1368
1369     SQLiteStatement statement(m_database, "SELECT COUNT(*) FROM StorageAccessUnderTopFrameDomains WHERE domainID = ? AND topLevelDomainID = ?");
1370     if (statement.prepare() != SQLITE_OK
1371         || statement.bindInt(1, requestingDomainID) != SQLITE_OK
1372         || statement.bindInt(2, firstPartyPrimaryDomainID) != SQLITE_OK
1373         || statement.step() != SQLITE_ROW)
1374         return StorageAccessPromptWasShown::No;
1375
1376     return !statement.getColumnInt(0) ? StorageAccessPromptWasShown::Yes : StorageAccessPromptWasShown::No;
1377 }
1378
1379 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::domainsToBlockAndDeleteCookiesFor() const
1380 {
1381     ASSERT(!RunLoop::isMain());
1382     
1383     Vector<RegistrableDomain> results;
1384     SQLiteStatement statement(m_database, "SELECT registrableDomain FROM ObservedDomains WHERE isPrevalent = 1 AND hadUserInteraction = 0"_s);
1385     if (statement.prepare() != SQLITE_OK)
1386         return results;
1387     
1388     while (statement.step() == SQLITE_ROW)
1389         results.append(RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(0)));
1390     
1391     return results;
1392 }
1393
1394 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::domainsToBlockButKeepCookiesFor() const
1395 {
1396     ASSERT(!RunLoop::isMain());
1397
1398     Vector<RegistrableDomain> results;
1399     SQLiteStatement statement(m_database, "SELECT registrableDomain FROM ObservedDomains WHERE isPrevalent = 1 AND hadUserInteraction = 1"_s);
1400     if (statement.prepare() != SQLITE_OK)
1401         return results;
1402     
1403     while (statement.step() == SQLITE_ROW)
1404         results.append(RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(0)));
1405
1406     return results;
1407 }
1408
1409 void ResourceLoadStatisticsDatabaseStore::updateCookieBlocking(CompletionHandler<void()>&& completionHandler)
1410 {
1411     ASSERT(!RunLoop::isMain());
1412
1413     auto domainsToBlockAndDeleteCookiesFor = this->domainsToBlockAndDeleteCookiesFor();
1414     auto domainsToBlockButKeepCookiesFor = this->domainsToBlockButKeepCookiesFor();
1415
1416     if (domainsToBlockAndDeleteCookiesFor.isEmpty() && domainsToBlockButKeepCookiesFor.isEmpty()) {
1417         completionHandler();
1418         return;
1419     }
1420
1421     RegistrableDomainsToBlockCookiesFor domainsToBlock { domainsToBlockAndDeleteCookiesFor, domainsToBlockButKeepCookiesFor };
1422
1423     if (debugLoggingEnabled() && !domainsToBlockAndDeleteCookiesFor.isEmpty() && !domainsToBlockButKeepCookiesFor.isEmpty())
1424         debugLogDomainsInBatches("block", domainsToBlock);
1425
1426     RunLoop::main().dispatch([weakThis = makeWeakPtr(*this), store = makeRef(store()), domainsToBlock = crossThreadCopy(domainsToBlock), completionHandler = WTFMove(completionHandler)] () mutable {
1427         store->callUpdatePrevalentDomainsToBlockCookiesForHandler(domainsToBlock, [weakThis = WTFMove(weakThis), store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1428             store->statisticsQueue().dispatch([weakThis = WTFMove(weakThis), completionHandler = WTFMove(completionHandler)]() mutable {
1429                 completionHandler();
1430                 if (!weakThis)
1431                     return;
1432                 RELEASE_LOG_INFO_IF(weakThis->debugLoggingEnabled(), ITPDebug, "Done updating cookie blocking.");
1433             });
1434         });
1435     });
1436 }
1437
1438 Vector<ResourceLoadStatisticsDatabaseStore::PrevalentDomainData> ResourceLoadStatisticsDatabaseStore::prevalentDomains() const
1439 {
1440     ASSERT(!RunLoop::isMain());
1441     
1442     Vector<PrevalentDomainData> results;
1443     SQLiteStatement statement(m_database, "SELECT domainID, registrableDomain, mostRecentUserInteractionTime, hadUserInteraction, grandfathered FROM ObservedDomains WHERE isPrevalent = 1"_s);
1444     if (statement.prepare() != SQLITE_OK)
1445         return results;
1446     
1447     while (statement.step() == SQLITE_ROW) {
1448         results.append({ static_cast<unsigned>(statement.getColumnInt(0))
1449             , RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(1))
1450             , WallTime::fromRawSeconds(statement.getColumnDouble(2))
1451             , statement.getColumnInt(3) ? true : false
1452             , statement.getColumnInt(4) ? true : false
1453         });
1454     }
1455     
1456     return results;
1457 }
1458
1459 Vector<unsigned> ResourceLoadStatisticsDatabaseStore::findExpiredUserInteractions() const
1460 {
1461     ASSERT(!RunLoop::isMain());
1462
1463     Vector<unsigned> results;
1464     Optional<Seconds> expirationDateTime = statisticsEpirationTime();
1465     if (!expirationDateTime)
1466         return results;
1467
1468     if (m_findExpiredUserInteractionStatement.bindDouble(1, expirationDateTime.value().value()) != SQLITE_OK)
1469         return results;
1470
1471     while (m_findExpiredUserInteractionStatement.step() == SQLITE_ROW)
1472         results.append(m_findExpiredUserInteractionStatement.getColumnInt(0));
1473
1474     int resetResult = m_findExpiredUserInteractionStatement.reset();
1475     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1476
1477     return results;
1478 }
1479
1480 void ResourceLoadStatisticsDatabaseStore::clearExpiredUserInteractions()
1481 {
1482     ASSERT(!RunLoop::isMain());
1483
1484     auto expiredRecords = findExpiredUserInteractions();
1485     if (expiredRecords.isEmpty())
1486         return;
1487
1488     auto expiredRecordIDs = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(expiredRecords.begin(), expiredRecords.end()));
1489
1490     SQLiteStatement clearExpiredInteraction(m_database, makeString("UPDATE ObservedDomains SET mostRecentUserInteractionTime = 0, hadUserInteraction = 1 WHERE domainID IN (", expiredRecordIDs, ")"));
1491     if (clearExpiredInteraction.prepare() != SQLITE_OK)
1492         return;
1493
1494     SQLiteStatement removeStorageAccess(m_database, makeString("DELETE FROM StorageAccessUnderTopFrameDomains ", expiredRecordIDs, ")"));
1495     if (removeStorageAccess.prepare() != SQLITE_OK)
1496         return;
1497
1498     if (clearExpiredInteraction.step() != SQLITE_DONE
1499         || removeStorageAccess.step() != SQLITE_DONE) {
1500         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::clearExpiredUserInteractions statement(s) failed to step, error message: %{public}s", this, m_database.lastErrorMsg());
1501         ASSERT_NOT_REACHED();
1502     }
1503 }
1504
1505 void ResourceLoadStatisticsDatabaseStore::clearGrandfathering(Vector<unsigned>&& domainIDsToClear)
1506 {
1507     ASSERT(!RunLoop::isMain());
1508
1509     if (domainIDsToClear.isEmpty())
1510         return;
1511
1512     auto listToClear = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(domainIDsToClear.begin(), domainIDsToClear.end()));
1513
1514     SQLiteStatement clearGrandfatheringStatement(m_database, makeString("UPDATE ObservedDomains SET grandfathered = 0 WHERE domainID IN (", listToClear, ")"));
1515     if (clearGrandfatheringStatement.prepare() != SQLITE_OK)
1516         return;
1517     
1518     if (clearGrandfatheringStatement.step() != SQLITE_DONE) {
1519         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::clearGrandfathering failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1520         ASSERT_NOT_REACHED();
1521     }
1522 }
1523
1524 bool ResourceLoadStatisticsDatabaseStore::hasHadUnexpiredRecentUserInteraction(const PrevalentDomainData& resourceStatistic, OperatingDatesWindow operatingDatesWindow)
1525 {
1526     if (resourceStatistic.hadUserInteraction && hasStatisticsExpired(resourceStatistic.mostRecentUserInteractionTime, operatingDatesWindow)) {
1527         clearUserInteraction(resourceStatistic.registrableDomain);
1528         return false;
1529     }
1530
1531     return resourceStatistic.hadUserInteraction;
1532 }
1533
1534 bool ResourceLoadStatisticsDatabaseStore::shouldRemoveAllWebsiteDataFor(const PrevalentDomainData& resourceStatistic, bool shouldCheckForGrandfathering)
1535 {
1536     return !hasHadUnexpiredRecentUserInteraction(resourceStatistic, OperatingDatesWindow::Long) && (!shouldCheckForGrandfathering || !resourceStatistic.grandfathered);
1537 }
1538
1539 bool ResourceLoadStatisticsDatabaseStore::shouldRemoveAllButCookiesFor(const PrevalentDomainData& resourceStatistic, bool shouldCheckForGrandfathering) const
1540 {
1541     UNUSED_PARAM(resourceStatistic);
1542     UNUSED_PARAM(shouldCheckForGrandfathering);
1543     return false;
1544 }
1545
1546 Vector<std::pair<RegistrableDomain, WebsiteDataToRemove>> ResourceLoadStatisticsDatabaseStore::registrableDomainsToRemoveWebsiteDataFor()
1547 {
1548     ASSERT(!RunLoop::isMain());
1549
1550     bool shouldCheckForGrandfathering = endOfGrandfatheringTimestamp() > WallTime::now();
1551     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && endOfGrandfatheringTimestamp();
1552
1553     if (shouldClearGrandfathering)
1554         clearEndOfGrandfatheringTimeStamp();
1555
1556     clearExpiredUserInteractions();
1557     
1558     Vector<std::pair<RegistrableDomain, WebsiteDataToRemove>> domainsToRemoveWebsiteDataFor;
1559
1560     Vector<PrevalentDomainData> prevalentDomains = this->prevalentDomains();
1561     Vector<unsigned> domainIDsToClearGrandfathering;
1562     for (auto& statistic : prevalentDomains) {
1563         if (shouldRemoveAllWebsiteDataFor(statistic, shouldCheckForGrandfathering))
1564             domainsToRemoveWebsiteDataFor.append(std::make_pair(statistic.registrableDomain, WebsiteDataToRemove::All));
1565         else if (shouldRemoveAllButCookiesFor(statistic, shouldCheckForGrandfathering))
1566             domainsToRemoveWebsiteDataFor.append(std::make_pair(statistic.registrableDomain, WebsiteDataToRemove::AllButCookies));
1567
1568         if (shouldClearGrandfathering && statistic.grandfathered)
1569             domainIDsToClearGrandfathering.append(statistic.domainID);
1570     }
1571
1572     clearGrandfathering(WTFMove(domainIDsToClearGrandfathering));
1573     
1574     return domainsToRemoveWebsiteDataFor;
1575 }
1576
1577 void ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded()
1578 {
1579     ASSERT(!RunLoop::isMain());
1580
1581     unsigned count = 0;
1582     if (m_observedDomainCount.step() == SQLITE_ROW)
1583         count = m_observedDomainCount.getColumnInt(0);
1584
1585     int resetResult = m_observedDomainCount.reset();
1586     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1587
1588     if (count <= parameters().maxStatisticsEntries)
1589         return;
1590
1591     ASSERT(parameters().pruneEntriesDownTo <= parameters().maxStatisticsEntries);
1592
1593     size_t countLeftToPrune = count - parameters().pruneEntriesDownTo;
1594     ASSERT(countLeftToPrune);
1595
1596     SQLiteStatement recordsToPrune(m_database, "SELECT domainID FROM ObservedDomains ORDER BY hadUserInteraction, isPrevalent, lastSeen LIMIT ?");
1597     if (recordsToPrune.prepare() != SQLITE_OK
1598         || recordsToPrune.bindInt(1, countLeftToPrune) != SQLITE_OK) {
1599         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded failed, error message: %{public}s", this, m_database.lastErrorMsg());
1600         ASSERT_NOT_REACHED();
1601         return;
1602     }
1603
1604     Vector<unsigned> entriesToPrune;
1605     while (recordsToPrune.step() == SQLITE_ROW)
1606         entriesToPrune.append(recordsToPrune.getColumnInt(0));
1607
1608     auto listToPrune = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(entriesToPrune.begin(), entriesToPrune.end()));
1609
1610     SQLiteStatement pruneCommand(m_database, makeString("DELETE from ObservedDomains WHERE domainID IN (", listToPrune, ")"));
1611     if (pruneCommand.prepare() != SQLITE_OK
1612         || pruneCommand.step() != SQLITE_DONE) {
1613         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded failed, error message: %{public}s", this, m_database.lastErrorMsg());
1614         ASSERT_NOT_REACHED();
1615         return;
1616     }
1617 }
1618
1619 void ResourceLoadStatisticsDatabaseStore::updateLastSeen(const RegistrableDomain& domain, WallTime lastSeen)
1620 {
1621     ASSERT(!RunLoop::isMain());
1622
1623     if (m_updateLastSeenStatement.bindDouble(1, lastSeen.secondsSinceEpoch().value()) != SQLITE_OK
1624         || m_updateLastSeenStatement.bindText(2, domain.string()) != SQLITE_OK
1625         || m_updateLastSeenStatement.step() != SQLITE_DONE) {
1626         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::updateLastSeen failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1627         ASSERT_NOT_REACHED();
1628         return;
1629     }
1630     
1631     int resetResult = m_updateLastSeenStatement.reset();
1632     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1633 }
1634
1635 void ResourceLoadStatisticsDatabaseStore::setLastSeen(const RegistrableDomain& domain, Seconds seconds)
1636 {
1637     ASSERT(!RunLoop::isMain());
1638
1639     ensureResourceStatisticsForRegistrableDomain(domain);
1640     updateLastSeen(domain, WallTime::fromRawSeconds(seconds.seconds()));
1641 }
1642
1643 void ResourceLoadStatisticsDatabaseStore::setPrevalentResource(const RegistrableDomain& domain)
1644 {
1645     ASSERT(!RunLoop::isMain());
1646
1647     if (shouldSkip(domain))
1648         return;
1649
1650     ensureResourceStatisticsForRegistrableDomain(domain);
1651     setPrevalentResource(domain, ResourceLoadPrevalence::High);
1652 }
1653
1654 void ResourceLoadStatisticsDatabaseStore::setVeryPrevalentResource(const RegistrableDomain& domain)
1655 {
1656     ASSERT(!RunLoop::isMain());
1657
1658     if (shouldSkip(domain))
1659         return;
1660     
1661     ensureResourceStatisticsForRegistrableDomain(domain);
1662     setPrevalentResource(domain, ResourceLoadPrevalence::VeryHigh);
1663 }
1664
1665 } // namespace WebKit
1666
1667 #endif