f76e9d307f6dfe04fb0a6ed4109c1c3d521a72e7
[WebKit-https.git] / Source / WebKit / NetworkProcess / Classifier / ResourceLoadStatisticsDatabaseStore.cpp
1 /*
2  * Copyright (C) 2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ResourceLoadStatisticsDatabaseStore.h"
28
29 #if ENABLE(RESOURCE_LOAD_STATISTICS)
30
31 #include "Logging.h"
32 #include "NetworkSession.h"
33 #include "PluginProcessManager.h"
34 #include "PluginProcessProxy.h"
35 #include "ResourceLoadStatisticsMemoryStore.h"
36 #include "StorageAccessStatus.h"
37 #include "WebProcessProxy.h"
38 #include "WebResourceLoadStatisticsTelemetry.h"
39 #include "WebsiteDataStore.h"
40 #include <WebCore/DocumentStorageAccess.h>
41 #include <WebCore/KeyedCoding.h>
42 #include <WebCore/NetworkStorageSession.h>
43 #include <WebCore/ResourceLoadStatistics.h>
44 #include <WebCore/SQLiteDatabase.h>
45 #include <WebCore/SQLiteStatement.h>
46 #include <WebCore/UserGestureIndicator.h>
47 #include <wtf/CallbackAggregator.h>
48 #include <wtf/DateMath.h>
49 #include <wtf/HashMap.h>
50 #include <wtf/MathExtras.h>
51 #include <wtf/StdSet.h>
52 #include <wtf/text/StringBuilder.h>
53
54 namespace WebKit {
55 using namespace WebCore;
56
57 #if PLATFORM(COCOA)
58 #define RELEASE_LOG_IF_ALLOWED(sessionID, fmt, ...) RELEASE_LOG_IF(sessionID.isAlwaysOnLoggingAllowed(), Network, "%p - ResourceLoadStatisticsDatabaseStore::" fmt, this, ##__VA_ARGS__)
59 #define RELEASE_LOG_ERROR_IF_ALLOWED(sessionID, fmt, ...) RELEASE_LOG_ERROR_IF(sessionID.isAlwaysOnLoggingAllowed(), Network, "%p - ResourceLoadStatisticsDatabaseStore::" fmt, this, ##__VA_ARGS__)
60 #else
61 #define RELEASE_LOG_IF_ALLOWED(sessionID, fmt, ...)  ((void)0)
62 #define RELEASE_LOG_ERROR_IF_ALLOWED(sessionID, fmt, ...)  ((void)0)
63 #endif
64
65 constexpr auto observedDomainCountQuery = "SELECT COUNT(*) FROM ObservedDomains"_s;
66 constexpr auto insertObservedDomainQuery = "INSERT INTO ObservedDomains (registrableDomain, lastSeen, hadUserInteraction,"
67     "mostRecentUserInteractionTime, grandfathered, isPrevalent, isVeryPrevalent, dataRecordsRemoved, timesAccessedAsFirstPartyDueToUserInteraction,"
68     "timesAccessedAsFirstPartyDueToStorageAccessAPI) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"_s;
69 constexpr auto insertTopLevelDomainQuery = "INSERT INTO TopLevelDomains VALUES (?)"_s;
70 constexpr auto domainIDFromStringQuery = "SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
71 constexpr auto storageAccessUnderTopFrameDomainsQuery = "INSERT INTO StorageAccessUnderTopFrameDomains (domainID, topLevelDomainID) "
72     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
73 constexpr auto topFrameUniqueRedirectsToQuery = "INSERT INTO TopFrameUniqueRedirectsTo (sourceDomainID, toDomainID) SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
74 constexpr auto topFrameUniqueRedirectsToExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameUniqueRedirectsTo WHERE sourceDomainID = ? "
75     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
76 constexpr auto topFrameUniqueRedirectsFromQuery = "INSERT INTO TopFrameUniqueRedirectsFrom (targetDomainID, fromDomainID) "
77     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
78 constexpr auto topFrameUniqueRedirectsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameUniqueRedirectsFrom WHERE targetDomainID = ? "
79     "AND fromDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
80 constexpr auto topFrameLinkDecorationsFromQuery = "INSERT INTO TopFrameLinkDecorationsFrom (fromDomainID, toDomainID) "
81     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
82 constexpr auto topFrameLinkDecorationsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameLinkDecorationsFrom WHERE fromDomainID = ? "
83     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
84 constexpr auto subframeUnderTopFrameDomainsQuery = "INSERT INTO SubframeUnderTopFrameDomains (subFrameDomainID, topFrameDomainID) "
85     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
86 constexpr auto subframeUnderTopFrameDomainExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubframeUnderTopFrameDomains WHERE subFrameDomainID = ? "
87     "AND topFrameDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
88 constexpr auto subresourceUnderTopFrameDomainsQuery = "INSERT INTO SubresourceUnderTopFrameDomains (subresourceDomainID, topFrameDomainID) "
89     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
90 constexpr auto subresourceUnderTopFrameDomainExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUnderTopFrameDomains "
91     "WHERE subresourceDomainID = ? AND topFrameDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
92 constexpr auto subresourceUniqueRedirectsToQuery = "INSERT INTO SubresourceUniqueRedirectsTo (subresourceDomainID, toDomainID) "
93     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
94 constexpr auto subresourceUniqueRedirectsToExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUniqueRedirectsTo WHERE subresourceDomainID = ? "
95     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
96 constexpr auto subresourceUniqueRedirectsFromQuery = "INSERT INTO SubresourceUniqueRedirectsFrom (subresourceDomainID, fromDomainID) "
97     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
98 constexpr auto subresourceUniqueRedirectsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUniqueRedirectsFrom WHERE subresourceDomainID = ? "
99     "AND fromDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
100 constexpr auto mostRecentUserInteractionQuery = "UPDATE ObservedDomains SET hadUserInteraction = ?, mostRecentUserInteractionTime = ? "
101     "WHERE registrableDomain = ?"_s;
102 constexpr auto updateLastSeenQuery = "UPDATE ObservedDomains SET lastSeen = ? WHERE registrableDomain = ?"_s;
103 constexpr auto updatePrevalentResourceQuery = "UPDATE ObservedDomains SET isPrevalent = ? WHERE registrableDomain = ?"_s;
104 constexpr auto isPrevalentResourceQuery = "SELECT isPrevalent FROM ObservedDomains WHERE registrableDomain = ?"_s;
105 constexpr auto updateVeryPrevalentResourceQuery = "UPDATE ObservedDomains SET isVeryPrevalent = ? WHERE registrableDomain = ?"_s;
106 constexpr auto isVeryPrevalentResourceQuery = "SELECT isVeryPrevalent FROM ObservedDomains WHERE registrableDomain = ?"_s;
107 constexpr auto clearPrevalentResourceQuery = "UPDATE ObservedDomains SET isPrevalent = 0, isVeryPrevalent = 0 WHERE registrableDomain = ?"_s;
108 constexpr auto hadUserInteractionQuery = "SELECT hadUserInteraction, mostRecentUserInteractionTime FROM ObservedDomains WHERE registrableDomain = ?"_s;
109 constexpr auto updateGrandfatheredQuery = "UPDATE ObservedDomains SET grandfathered = ? WHERE registrableDomain = ?"_s;
110 constexpr auto isGrandfatheredQuery = "SELECT grandfathered FROM ObservedDomains WHERE registrableDomain = ?"_s;
111 constexpr auto findExpiredUserInteractionQuery = "SELECT domainID FROM ObservedDomains WHERE hadUserInteraction = 1 AND mostRecentUserInteractionTime < ?"_s;
112
113 constexpr auto createObservedDomain = "CREATE TABLE ObservedDomains ("
114     "domainID INTEGER PRIMARY KEY, registrableDomain TEXT NOT NULL UNIQUE ON CONFLICT FAIL, lastSeen REAL NOT NULL, "
115     "hadUserInteraction INTEGER NOT NULL, mostRecentUserInteractionTime REAL NOT NULL, grandfathered INTEGER NOT NULL, "
116     "isPrevalent INTEGER NOT NULL, isVeryPrevalent INTEGER NOT NULL, dataRecordsRemoved INTEGER NOT NULL,"
117     "timesAccessedAsFirstPartyDueToUserInteraction INTEGER NOT NULL, timesAccessedAsFirstPartyDueToStorageAccessAPI INTEGER NOT NULL);"_s;
118     
119 constexpr auto createTopLevelDomains = "CREATE TABLE TopLevelDomains ("
120     "topLevelDomainID INTEGER PRIMARY KEY, CONSTRAINT fkDomainID FOREIGN KEY(topLevelDomainID) "
121     "REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
122     
123 constexpr auto createStorageAccessUnderTopFrameDomains = "CREATE TABLE StorageAccessUnderTopFrameDomains ("
124     "domainID INTEGER NOT NULL, topLevelDomainID INTEGER NOT NULL ON CONFLICT FAIL, "
125     "CONSTRAINT fkDomainID FOREIGN KEY(domainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
126     "FOREIGN KEY(topLevelDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
127     
128 constexpr auto createTopFrameUniqueRedirectsTo = "CREATE TABLE TopFrameUniqueRedirectsTo ("
129     "sourceDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
130     "FOREIGN KEY(sourceDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
131     "FOREIGN KEY(toDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
132     
133 constexpr auto createTopFrameUniqueRedirectsFrom = "CREATE TABLE TopFrameUniqueRedirectsFrom ("
134     "targetDomainID INTEGER NOT NULL, fromDomainID INTEGER NOT NULL, "
135     "FOREIGN KEY(targetDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
136     "FOREIGN KEY(fromDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
137
138 constexpr auto createTopFrameLinkDecorationsFrom = "CREATE TABLE TopFrameLinkDecorationsFrom ("
139     "fromDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
140     "FOREIGN KEY(fromDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
141     "FOREIGN KEY(toDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
142
143 constexpr auto createSubframeUnderTopFrameDomains = "CREATE TABLE SubframeUnderTopFrameDomains ("
144     "subFrameDomainID INTEGER NOT NULL, topFrameDomainID INTEGER NOT NULL, "
145     "FOREIGN KEY(subFrameDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
146     "FOREIGN KEY(topFrameDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
147     
148 constexpr auto createSubresourceUnderTopFrameDomains = "CREATE TABLE SubresourceUnderTopFrameDomains ("
149     "subresourceDomainID INTEGER NOT NULL, topFrameDomainID INTEGER NOT NULL, "
150     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
151     "FOREIGN KEY(topFrameDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
152     
153 constexpr auto createSubresourceUniqueRedirectsTo = "CREATE TABLE SubresourceUniqueRedirectsTo ("
154     "subresourceDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
155     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
156     "FOREIGN KEY(toDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
157     
158 constexpr auto createSubresourceUniqueRedirectsFrom = "CREATE TABLE SubresourceUniqueRedirectsFrom ("
159     "subresourceDomainID INTEGER NOT NULL, fromDomainID INTEGER NOT NULL, "
160     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
161     "FOREIGN KEY(fromDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
162     
163 ResourceLoadStatisticsDatabaseStore::ResourceLoadStatisticsDatabaseStore(WebResourceLoadStatisticsStore& store, WorkQueue& workQueue, ShouldIncludeLocalhost shouldIncludeLocalhost, const String& storageDirectoryPath, PAL::SessionID sessionID)
164     : ResourceLoadStatisticsStore(store, workQueue, shouldIncludeLocalhost)
165     , m_storageDirectoryPath(storageDirectoryPath + "/observations.db")
166     , m_observedDomainCount(m_database, observedDomainCountQuery)
167     , m_insertObservedDomainStatement(m_database, insertObservedDomainQuery)
168     , m_insertTopLevelDomainStatement(m_database, insertTopLevelDomainQuery)
169     , m_domainIDFromStringStatement(m_database, domainIDFromStringQuery)
170     , m_storageAccessUnderTopFrameDomainsStatement(m_database, storageAccessUnderTopFrameDomainsQuery)
171     , m_topFrameUniqueRedirectsTo(m_database, topFrameUniqueRedirectsToQuery)
172     , m_topFrameUniqueRedirectsToExists(m_database, topFrameUniqueRedirectsToExistsQuery)
173     , m_topFrameUniqueRedirectsFrom(m_database, topFrameUniqueRedirectsFromQuery)
174     , m_topFrameUniqueRedirectsFromExists(m_database, topFrameUniqueRedirectsFromExistsQuery)
175     , m_topFrameLinkDecorationsFrom(m_database, topFrameLinkDecorationsFromQuery)
176     , m_topFrameLinkDecorationsFromExists(m_database, topFrameLinkDecorationsFromExistsQuery)
177     , m_subframeUnderTopFrameDomains(m_database, subframeUnderTopFrameDomainsQuery)
178     , m_subframeUnderTopFrameDomainExists(m_database, subframeUnderTopFrameDomainExistsQuery)
179     , m_subresourceUnderTopFrameDomains(m_database, subresourceUnderTopFrameDomainsQuery)
180     , m_subresourceUnderTopFrameDomainExists(m_database, subresourceUnderTopFrameDomainExistsQuery)
181     , m_subresourceUniqueRedirectsTo(m_database, subresourceUniqueRedirectsToQuery)
182     , m_subresourceUniqueRedirectsToExists(m_database, subresourceUniqueRedirectsToExistsQuery)
183     , m_subresourceUniqueRedirectsFrom(m_database, subresourceUniqueRedirectsFromQuery)
184     , m_subresourceUniqueRedirectsFromExists(m_database, subresourceUniqueRedirectsFromExistsQuery)
185     , m_mostRecentUserInteractionStatement(m_database, mostRecentUserInteractionQuery)
186     , m_updateLastSeenStatement(m_database, updateLastSeenQuery)
187     , m_updatePrevalentResourceStatement(m_database, updatePrevalentResourceQuery)
188     , m_isPrevalentResourceStatement(m_database, isPrevalentResourceQuery)
189     , m_updateVeryPrevalentResourceStatement(m_database, updateVeryPrevalentResourceQuery)
190     , m_isVeryPrevalentResourceStatement(m_database, isVeryPrevalentResourceQuery)
191     , m_clearPrevalentResourceStatement(m_database, clearPrevalentResourceQuery)
192     , m_hadUserInteractionStatement(m_database, hadUserInteractionQuery)
193     , m_updateGrandfatheredStatement(m_database, updateGrandfatheredQuery)
194     , m_isGrandfatheredStatement(m_database, isGrandfatheredQuery)
195     , m_findExpiredUserInteractionStatement(m_database, findExpiredUserInteractionQuery)
196     , m_sessionID(sessionID)
197 {
198     ASSERT(!RunLoop::isMain());
199
200     if (!m_database.open(m_storageDirectoryPath)) {
201         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::open failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
202         ASSERT_NOT_REACHED();
203         return;
204     }
205     
206     // Since we are using a workerQueue, the sequential dispatch blocks may be called by different threads.
207     m_database.disableThreadingChecks();
208
209     if (!m_database.tableExists("ObservedDomains"_s)) {
210         if (!createSchema()) {
211             RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::createSchema failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
212             ASSERT_NOT_REACHED();
213             return;
214         }
215     }
216     
217     if (!m_database.turnOnIncrementalAutoVacuum())
218         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::turnOnIncrementalAutoVacuum failed, error message: %{public}s", this, m_database.lastErrorMsg());
219
220     if (!prepareStatements()) {
221         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::prepareStatements failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
222         ASSERT_NOT_REACHED();
223         return;
224     }
225
226     workQueue.dispatchAfter(5_s, [weakThis = makeWeakPtr(*this)] {
227         if (weakThis)
228             weakThis->calculateAndSubmitTelemetry();
229     });
230 }
231
232 bool ResourceLoadStatisticsDatabaseStore::isEmpty() const
233 {
234     ASSERT(!RunLoop::isMain());
235     
236     bool result = false;
237     if (m_observedDomainCount.step() == SQLITE_ROW)
238         result = !m_observedDomainCount.getColumnInt(0);
239     
240     int resetResult = m_observedDomainCount.reset();
241     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
242     
243     return result;
244 }
245
246 bool ResourceLoadStatisticsDatabaseStore::createSchema()
247 {
248     ASSERT(!RunLoop::isMain());
249
250     if (!m_database.executeCommand(createObservedDomain)) {
251         LOG_ERROR("Could not create ObservedDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
252         return false;
253     }
254
255     if (!m_database.executeCommand(createTopLevelDomains)) {
256         LOG_ERROR("Could not create TopLevelDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
257         return false;
258     }
259
260     if (!m_database.executeCommand(createStorageAccessUnderTopFrameDomains)) {
261         LOG_ERROR("Could not create StorageAccessUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
262         return false;
263     }
264
265     if (!m_database.executeCommand(createTopFrameUniqueRedirectsTo)) {
266         LOG_ERROR("Could not create TopFrameUniqueRedirectsTo table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
267         return false;
268     }
269
270     if (!m_database.executeCommand(createTopFrameUniqueRedirectsFrom)) {
271         LOG_ERROR("Could not create TopFrameUniqueRedirectsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
272         return false;
273     }
274
275     if (!m_database.executeCommand(createTopFrameLinkDecorationsFrom)) {
276         LOG_ERROR("Could not create TopFrameLinkDecorationsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
277         return false;
278     }
279     
280     if (!m_database.executeCommand(createSubframeUnderTopFrameDomains)) {
281         LOG_ERROR("Could not create SubframeUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
282         return false;
283     }
284
285     if (!m_database.executeCommand(createSubresourceUnderTopFrameDomains)) {
286         LOG_ERROR("Could not create SubresourceUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
287         return false;
288     }
289
290     if (!m_database.executeCommand(createSubresourceUniqueRedirectsTo)) {
291         LOG_ERROR("Could not create SubresourceUniqueRedirectsTo table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
292         return false;
293     }
294
295     if (!m_database.executeCommand(createSubresourceUniqueRedirectsFrom)) {
296         LOG_ERROR("Could not create SubresourceUniqueRedirectsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
297         return false;
298     }
299
300     return true;
301 }
302
303 bool ResourceLoadStatisticsDatabaseStore::prepareStatements()
304 {
305     ASSERT(!RunLoop::isMain());
306         
307     if (m_observedDomainCount.prepare() != SQLITE_OK
308         || m_insertObservedDomainStatement.prepare() != SQLITE_OK
309         || m_insertTopLevelDomainStatement.prepare() != SQLITE_OK
310         || m_domainIDFromStringStatement.prepare() != SQLITE_OK
311         || m_storageAccessUnderTopFrameDomainsStatement.prepare() != SQLITE_OK
312         || m_topFrameUniqueRedirectsTo.prepare() != SQLITE_OK
313         || m_topFrameUniqueRedirectsToExists.prepare() != SQLITE_OK
314         || m_topFrameUniqueRedirectsFrom.prepare() != SQLITE_OK
315         || m_topFrameUniqueRedirectsFromExists.prepare() != SQLITE_OK
316         || m_subframeUnderTopFrameDomains.prepare() != SQLITE_OK
317         || m_subframeUnderTopFrameDomainExists.prepare() != SQLITE_OK
318         || m_subresourceUnderTopFrameDomains.prepare() != SQLITE_OK
319         || m_subresourceUnderTopFrameDomainExists.prepare() != SQLITE_OK
320         || m_subresourceUniqueRedirectsTo.prepare() != SQLITE_OK
321         || m_subresourceUniqueRedirectsToExists.prepare() != SQLITE_OK
322         || m_subresourceUniqueRedirectsFrom.prepare() != SQLITE_OK
323         || m_subresourceUniqueRedirectsFromExists.prepare() != SQLITE_OK
324         || m_updateLastSeenStatement.prepare() != SQLITE_OK
325         || m_mostRecentUserInteractionStatement.prepare() != SQLITE_OK
326         || m_updatePrevalentResourceStatement.prepare() != SQLITE_OK
327         || m_isPrevalentResourceStatement.prepare() != SQLITE_OK
328         || m_updateVeryPrevalentResourceStatement.prepare() != SQLITE_OK
329         || m_isVeryPrevalentResourceStatement.prepare() != SQLITE_OK
330         || m_clearPrevalentResourceStatement.prepare() != SQLITE_OK
331         || m_hadUserInteractionStatement.prepare() != SQLITE_OK
332         || m_updateGrandfatheredStatement.prepare() != SQLITE_OK
333         || m_isGrandfatheredStatement.prepare() != SQLITE_OK
334         || m_findExpiredUserInteractionStatement.prepare() != SQLITE_OK
335         || m_topFrameLinkDecorationsFrom.prepare() != SQLITE_OK
336         || m_topFrameLinkDecorationsFromExists.prepare() != SQLITE_OK) {
337         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::prepareStatements failed to prepare, error message: %{public}s", this, m_database.lastErrorMsg());
338         ASSERT_NOT_REACHED();
339         return false;
340     }
341
342     return true;
343 }
344
345 bool ResourceLoadStatisticsDatabaseStore::insertObservedDomain(const ResourceLoadStatistics& loadStatistics)
346 {
347     ASSERT(!RunLoop::isMain());
348
349 #ifndef NDEBUG
350     ASSERT(confirmDomainDoesNotExist(loadStatistics.registrableDomain));
351 #endif
352
353     if (m_insertObservedDomainStatement.bindText(1, loadStatistics.registrableDomain.string()) != SQLITE_OK
354         || m_insertObservedDomainStatement.bindDouble(2, loadStatistics.lastSeen.secondsSinceEpoch().value()) != SQLITE_OK
355         || m_insertObservedDomainStatement.bindInt(3, loadStatistics.hadUserInteraction) != SQLITE_OK
356         || m_insertObservedDomainStatement.bindDouble(4, loadStatistics.mostRecentUserInteractionTime.secondsSinceEpoch().value()) != SQLITE_OK
357         || m_insertObservedDomainStatement.bindInt(5, loadStatistics.grandfathered) != SQLITE_OK
358         || m_insertObservedDomainStatement.bindInt(6, loadStatistics.isPrevalentResource) != SQLITE_OK
359         || m_insertObservedDomainStatement.bindInt(7, loadStatistics.isVeryPrevalentResource) != SQLITE_OK
360         || m_insertObservedDomainStatement.bindInt(8, loadStatistics.dataRecordsRemoved) != SQLITE_OK
361         || m_insertObservedDomainStatement.bindInt(9, loadStatistics.timesAccessedAsFirstPartyDueToUserInteraction) != SQLITE_OK
362         || m_insertObservedDomainStatement.bindInt(10, loadStatistics.timesAccessedAsFirstPartyDueToStorageAccessAPI) != SQLITE_OK) {
363         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::insertObservedDomain failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
364         ASSERT_NOT_REACHED();
365         return false;
366     }
367
368     if (m_insertObservedDomainStatement.step() != SQLITE_DONE) {
369         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::insertObservedDomain failed to commit, error message: %{public}s", this, m_database.lastErrorMsg());
370         ASSERT_NOT_REACHED();
371         return false;
372     }
373
374     int resetResult = m_insertObservedDomainStatement.reset();
375     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
376
377     return true;
378 }
379
380 bool ResourceLoadStatisticsDatabaseStore::relationshipExists(WebCore::SQLiteStatement& statement, Optional<unsigned> firstDomainID, const RegistrableDomain& secondDomain) const
381 {
382     if (!firstDomainID)
383         return false;
384     
385     ASSERT(!RunLoop::isMain());
386
387     if (statement.bindInt(1, *firstDomainID) != SQLITE_OK
388         || statement.bindText(2, secondDomain.string()) != SQLITE_OK
389         || statement.step() != SQLITE_ROW) {
390         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_insertDomainRelationshipStatement failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
391         ASSERT_NOT_REACHED();
392         return false;
393     }
394     bool relationshipExists = !!statement.getColumnInt(0);
395
396     int resetResult = statement.reset();
397     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
398     
399     return relationshipExists;
400 }
401
402 bool ResourceLoadStatisticsDatabaseStore::insertDomainRelationship(WebCore::SQLiteStatement& statement, unsigned domainID, const RegistrableDomain& topFrame)
403 {
404     ASSERT(!RunLoop::isMain());
405
406     if (statement.bindInt(1, domainID) != SQLITE_OK
407         || statement.bindText(2, topFrame.string()) != SQLITE_OK
408         || statement.step() != SQLITE_DONE) {
409         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_insertDomainRelationshipStatement failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
410         ASSERT_NOT_REACHED();
411         return false;
412     }
413
414     int resetResult = statement.reset();
415     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
416     
417     return true;
418 }
419
420 #ifndef NDEBUG
421 bool ResourceLoadStatisticsDatabaseStore::confirmDomainDoesNotExist(const RegistrableDomain& domain) const
422 {
423     ASSERT(!RunLoop::isMain());
424
425     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK) {
426         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::confirmDomainDoesNotExist failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
427         ASSERT_NOT_REACHED();
428         return false;
429     }
430         
431     if (m_domainIDFromStringStatement.step() == SQLITE_ROW) {
432         int resetResult = m_domainIDFromStringStatement.reset();
433         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
434         return false;
435     }
436
437     int resetResult = m_domainIDFromStringStatement.reset();
438     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
439     
440     return true;
441 }
442 #endif
443
444 Optional<unsigned> ResourceLoadStatisticsDatabaseStore::domainID(const RegistrableDomain& domain) const
445 {
446     ASSERT(!RunLoop::isMain());
447
448     unsigned domainID = 0;
449
450     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK) {
451         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::domainIDFromString failed, error message: %{public}s", this, m_database.lastErrorMsg());
452         return WTF::nullopt;
453     }
454     
455     if (m_domainIDFromStringStatement.step() != SQLITE_ROW) {
456         int resetResult = m_domainIDFromStringStatement.reset();
457         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
458         return WTF::nullopt;
459     }
460
461     domainID = m_domainIDFromStringStatement.getColumnInt(0);
462
463     int resetResult = m_domainIDFromStringStatement.reset();
464     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
465     
466     return domainID;
467 }
468
469 void ResourceLoadStatisticsDatabaseStore::insertDomainRelationships(const ResourceLoadStatistics& loadStatistics)
470 {
471     ASSERT(!RunLoop::isMain());
472     auto registrableDomainID = domainID(loadStatistics.registrableDomain);
473     
474     if (!registrableDomainID)
475         return;
476     
477     for (auto& topFrameDomain : loadStatistics.storageAccessUnderTopFrameDomains)
478         insertDomainRelationship(m_storageAccessUnderTopFrameDomainsStatement, registrableDomainID.value(), topFrameDomain);
479
480     for (auto& toDomain : loadStatistics.topFrameUniqueRedirectsTo)
481         insertDomainRelationship(m_topFrameUniqueRedirectsTo, registrableDomainID.value(), toDomain);
482     
483     for (auto& fromDomain : loadStatistics.topFrameUniqueRedirectsFrom)
484         insertDomainRelationship(m_topFrameUniqueRedirectsFrom, registrableDomainID.value(), fromDomain);
485     
486     for (auto& topFrameDomain : loadStatistics.subframeUnderTopFrameDomains)
487         insertDomainRelationship(m_subframeUnderTopFrameDomains, registrableDomainID.value(), topFrameDomain);
488     
489     for (auto& topFrameDomain : loadStatistics.subresourceUnderTopFrameDomains)
490         insertDomainRelationship(m_subresourceUnderTopFrameDomains, registrableDomainID.value(), topFrameDomain);
491     
492     for (auto& toDomain : loadStatistics.subresourceUniqueRedirectsTo)
493         insertDomainRelationship(m_subresourceUniqueRedirectsTo, registrableDomainID.value(), toDomain);
494     
495     for (auto& fromDomain : loadStatistics.subresourceUniqueRedirectsFrom)
496         insertDomainRelationship(m_subresourceUniqueRedirectsFrom, registrableDomainID.value(), fromDomain);
497 }
498
499 void ResourceLoadStatisticsDatabaseStore::populateFromMemoryStore(const ResourceLoadStatisticsMemoryStore& memoryStore)
500 {
501     ASSERT(!RunLoop::isMain());
502
503     if (!isEmpty())
504         return;
505
506     auto& statisticsMap = memoryStore.data();
507     for (const auto& statistic : statisticsMap)
508         insertObservedDomain(statistic.value);
509
510     // Make a separate pass for inter-domain relationships so we
511     // can refer to the ObservedDomain table entries
512     for (auto& statistic : statisticsMap)
513         insertDomainRelationships(statistic.value);
514 }
515
516 void ResourceLoadStatisticsDatabaseStore::calculateAndSubmitTelemetry() const
517 {
518     ASSERT(!RunLoop::isMain());
519
520     // FIXME(195088): Implement for Database version.
521 }
522
523 static String domainsToString(const HashSet<RegistrableDomain>& domains)
524 {
525     StringBuilder builder;
526     for (const auto& domainName : domains) {
527         if (!builder.isEmpty())
528             builder.appendLiteral(", ");
529         builder.append('"');
530         builder.append(domainName.string());
531         builder.append('"');
532     }
533
534     return builder.toString();
535 }
536
537 void ResourceLoadStatisticsDatabaseStore::incrementRecordsDeletedCountForDomains(HashSet<RegistrableDomain>&& domains)
538 {
539     ASSERT(!RunLoop::isMain());
540
541     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET dataRecordsRemoved = dataRecordsRemoved + 1 WHERE registrableDomain IN (", domainsToString(domains), ")"));
542     if (domainsToUpdateStatement.prepare() != SQLITE_OK
543         || domainsToUpdateStatement.step() != SQLITE_DONE) {
544         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::incrementStatisticsForDomains failed, error message: %{public}s", this, m_database.lastErrorMsg());
545         ASSERT_NOT_REACHED();
546     }
547 }
548
549 unsigned ResourceLoadStatisticsDatabaseStore::recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(unsigned primaryDomainID, StdSet<unsigned>& nonPrevalentRedirectionSources, unsigned numberOfRecursiveCalls)
550 {
551     ASSERT(!RunLoop::isMain());
552
553     if (numberOfRecursiveCalls >= maxNumberOfRecursiveCallsInRedirectTraceBack) {
554         RELEASE_LOG(ResourceLoadStatistics, "Hit %u recursive calls in redirect backtrace. Returning early.", maxNumberOfRecursiveCallsInRedirectTraceBack);
555         return numberOfRecursiveCalls;
556     }
557     
558     ++numberOfRecursiveCalls;
559
560     StdSet<unsigned> newlyIdentifiedDomains;
561     SQLiteStatement findSubresources(m_database, "SELECT SubresourceUniqueRedirectsFrom.fromDomainID from SubresourceUniqueRedirectsFrom INNER JOIN ObservedDomains ON ObservedDomains.domainID = SubresourceUniqueRedirectsFrom.fromDomainID WHERE subresourceDomainID = ? AND ObservedDomains.isPrevalent = 0"_s);
562     if (findSubresources.prepare() != SQLITE_OK
563         || findSubresources.bindInt(1, primaryDomainID) != SQLITE_OK) {
564         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain failed, error message: %{public}s", this, m_database.lastErrorMsg());
565         ASSERT_NOT_REACHED();
566         return 0;
567     }
568     
569     while (findSubresources.step() == SQLITE_ROW) {
570         int newDomainID = findSubresources.getColumnInt(0);
571         auto insertResult = nonPrevalentRedirectionSources.insert(newDomainID);
572         if (insertResult.second)
573             newlyIdentifiedDomains.insert(newDomainID);
574     }
575
576     SQLiteStatement findTopFrames(m_database, "SELECT TopFrameUniqueRedirectsFrom.fromDomainID from TopFrameUniqueRedirectsFrom INNER JOIN ObservedDomains ON ObservedDomains.domainID = TopFrameUniqueRedirectsFrom.fromDomainID WHERE targetDomainID = ? AND ObservedDomains.isPrevalent = 0"_s);
577     if (findTopFrames.prepare() != SQLITE_OK
578         || findTopFrames.bindInt(1, primaryDomainID) != SQLITE_OK) {
579         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain failed, error message: %{public}s", this, m_database.lastErrorMsg());
580         ASSERT_NOT_REACHED();
581         return 0;
582     }
583     
584     while (findTopFrames.step() == SQLITE_ROW) {
585         int newDomainID = findTopFrames.getColumnInt(0);
586         auto insertResult = nonPrevalentRedirectionSources.insert(newDomainID);
587         if (insertResult.second)
588             newlyIdentifiedDomains.insert(newDomainID);
589     }
590
591     if (newlyIdentifiedDomains.empty())
592         return numberOfRecursiveCalls;
593
594     for (auto domainID : newlyIdentifiedDomains)
595         numberOfRecursiveCalls = recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(domainID, nonPrevalentRedirectionSources, numberOfRecursiveCalls);
596
597     return numberOfRecursiveCalls;
598 }
599
600 template <typename IteratorType>
601 static String buildList(const WTF::IteratorRange<IteratorType>& values)
602 {
603     StringBuilder builder;
604     for (auto domainID : values) {
605         if (!builder.isEmpty())
606             builder.appendLiteral(", ");
607         builder.appendNumber(domainID);
608     }
609     return builder.toString();
610 }
611
612 void ResourceLoadStatisticsDatabaseStore::markAsPrevalentIfHasRedirectedToPrevalent()
613 {
614     ASSERT(!RunLoop::isMain());
615
616     StdSet<unsigned> prevalentDueToRedirect;
617     SQLiteStatement subresourceRedirectStatement(m_database, "SELECT DISTINCT SubresourceUniqueRedirectsTo.subresourceDomainID FROM SubresourceUniqueRedirectsTo JOIN ObservedDomains ON ObservedDomains.domainID = SubresourceUniqueRedirectsTo.toDomainID AND ObservedDomains.isPrevalent = 1"_s);
618     if (subresourceRedirectStatement.prepare() == SQLITE_OK) {
619         while (subresourceRedirectStatement.step() == SQLITE_ROW)
620             prevalentDueToRedirect.insert(subresourceRedirectStatement.getColumnInt(0));
621     }
622
623     SQLiteStatement topFrameRedirectStatement(m_database, "SELECT DISTINCT TopFrameUniqueRedirectsTo.sourceDomainID FROM TopFrameUniqueRedirectsTo JOIN ObservedDomains ON ObservedDomains.domainID = TopFrameUniqueRedirectsTo.toDomainID AND ObservedDomains.isPrevalent = 1"_s);
624     if (topFrameRedirectStatement.prepare() == SQLITE_OK) {
625         while (topFrameRedirectStatement.step() == SQLITE_ROW)
626             prevalentDueToRedirect.insert(topFrameRedirectStatement.getColumnInt(0));
627     }
628
629     SQLiteStatement markPrevalentStatement(m_database, makeString("UPDATE ObservedDomains SET isPrevalent = 1 WHERE domainID IN (", buildList(WTF::IteratorRange<StdSet<unsigned>::iterator>(prevalentDueToRedirect.begin(), prevalentDueToRedirect.end())), ")"));
630     if (markPrevalentStatement.prepare() != SQLITE_OK
631         || markPrevalentStatement.step() != SQLITE_DONE) {
632         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::markAsPrevalentIfHasRedirectedToPrevalent failed to execute, error message: %{public}s", this, m_database.lastErrorMsg());
633         ASSERT_NOT_REACHED();
634     }
635 }
636
637 HashMap<unsigned, ResourceLoadStatisticsDatabaseStore::NotVeryPrevalentResources> ResourceLoadStatisticsDatabaseStore::findNotVeryPrevalentResources()
638 {
639     ASSERT(!RunLoop::isMain());
640
641     HashMap<unsigned, NotVeryPrevalentResources> results;
642     SQLiteStatement notVeryPrevalentResourcesStatement(m_database, "SELECT domainID, registrableDomain, isPrevalent FROM ObservedDomains WHERE isVeryPrevalent = 0"_s);
643     if (notVeryPrevalentResourcesStatement.prepare() == SQLITE_OK) {
644         while (notVeryPrevalentResourcesStatement.step() == SQLITE_ROW) {
645             unsigned key = static_cast<unsigned>(notVeryPrevalentResourcesStatement.getColumnInt(0));
646             NotVeryPrevalentResources value({ RegistrableDomain::uncheckedCreateFromRegistrableDomainString(notVeryPrevalentResourcesStatement.getColumnText(1))
647                 , notVeryPrevalentResourcesStatement.getColumnInt(2) ? ResourceLoadPrevalence::High : ResourceLoadPrevalence::Low
648                 , 0, 0, 0, 0 });
649             results.add(key, value);
650         }
651     }
652
653     StringBuilder builder;
654     for (auto value : results.keys()) {
655         if (!builder.isEmpty())
656             builder.appendLiteral(", ");
657         builder.appendNumber(value);
658     }
659
660     auto domainIDsOfInterest = builder.toString();
661
662     SQLiteStatement subresourceUnderTopFrameDomainsStatement(m_database, makeString("SELECT subresourceDomainID, COUNT(topFrameDomainID) FROM SubresourceUnderTopFrameDomains WHERE subresourceDomainID IN (", domainIDsOfInterest, ") GROUP BY subresourceDomainID"));
663     if (subresourceUnderTopFrameDomainsStatement.prepare() == SQLITE_OK) {
664         while (subresourceUnderTopFrameDomainsStatement.step() == SQLITE_ROW) {
665             unsigned domainID = static_cast<unsigned>(subresourceUnderTopFrameDomainsStatement.getColumnInt(0));
666             auto result = results.find(domainID);
667             if (result != results.end())
668                 result->value.subresourceUnderTopFrameDomainsCount = static_cast<unsigned>(subresourceUnderTopFrameDomainsStatement.getColumnInt(1));
669         }
670     }
671
672     SQLiteStatement subresourceUniqueRedirectsToCountStatement(m_database, makeString("SELECT subresourceDomainID, COUNT(toDomainID) FROM SubresourceUniqueRedirectsTo WHERE subresourceDomainID IN (", domainIDsOfInterest, ") GROUP BY subresourceDomainID"));
673     if (subresourceUniqueRedirectsToCountStatement.prepare() == SQLITE_OK) {
674         while (subresourceUniqueRedirectsToCountStatement.step() == SQLITE_ROW) {
675             unsigned domainID = static_cast<unsigned>(subresourceUniqueRedirectsToCountStatement.getColumnInt(0));
676             auto result = results.find(domainID);
677             if (result != results.end())
678                 result->value.subresourceUniqueRedirectsToCount = static_cast<unsigned>(subresourceUniqueRedirectsToCountStatement.getColumnInt(1));
679         }
680     }
681
682     SQLiteStatement subframeUnderTopFrameDomainsCountStatement(m_database, makeString("SELECT subframeDomainID, COUNT(topFrameDomainID) FROM SubframeUnderTopFrameDomains WHERE subframeDomainID IN (", domainIDsOfInterest, ") GROUP BY subframeDomainID"));
683     if (subframeUnderTopFrameDomainsCountStatement.prepare() == SQLITE_OK) {
684         while (subframeUnderTopFrameDomainsCountStatement.step() == SQLITE_ROW) {
685             unsigned domainID = static_cast<unsigned>(subframeUnderTopFrameDomainsCountStatement.getColumnInt(0));
686             auto result = results.find(domainID);
687             if (result != results.end())
688                 result->value.subframeUnderTopFrameDomainsCount = static_cast<unsigned>(subframeUnderTopFrameDomainsCountStatement.getColumnInt(1));
689         }
690     }
691
692     SQLiteStatement topFrameUniqueRedirectsToCountStatement(m_database, makeString("SELECT sourceDomainID, COUNT(toDomainID) FROM TopFrameUniqueRedirectsTo WHERE sourceDomainID IN (", domainIDsOfInterest, ") GROUP BY sourceDomainID"));
693     if (topFrameUniqueRedirectsToCountStatement.prepare() == SQLITE_OK) {
694         while (topFrameUniqueRedirectsToCountStatement.step() == SQLITE_ROW) {
695             unsigned domainID = static_cast<unsigned>(topFrameUniqueRedirectsToCountStatement.getColumnInt(0));
696             auto result = results.find(domainID);
697             if (result != results.end())
698                 result->value.topFrameUniqueRedirectsToCount = static_cast<unsigned>(topFrameUniqueRedirectsToCountStatement.getColumnInt(1));
699         }
700     }
701
702     return results;
703 }
704
705 void ResourceLoadStatisticsDatabaseStore::reclassifyResources()
706 {
707     ASSERT(!RunLoop::isMain());
708
709     auto notVeryPrevalentResources = findNotVeryPrevalentResources();
710
711     for (auto& resourceStatistic : notVeryPrevalentResources.values()) {
712         if (shouldSkip(resourceStatistic.registrableDomain))
713             continue;
714
715         auto newPrevalence = classifier().calculateResourcePrevalence(resourceStatistic.subresourceUnderTopFrameDomainsCount, resourceStatistic.subresourceUniqueRedirectsToCount, resourceStatistic.subframeUnderTopFrameDomainsCount, resourceStatistic.topFrameUniqueRedirectsToCount, resourceStatistic.prevalence);
716         if (newPrevalence != resourceStatistic.prevalence)
717             setPrevalentResource(resourceStatistic.registrableDomain, newPrevalence);
718     }
719 }
720
721 void ResourceLoadStatisticsDatabaseStore::classifyPrevalentResources()
722 {
723     ASSERT(!RunLoop::isMain());
724     ensurePrevalentResourcesForDebugMode();
725     markAsPrevalentIfHasRedirectedToPrevalent();
726     reclassifyResources();
727 }
728
729 void ResourceLoadStatisticsDatabaseStore::syncStorageIfNeeded()
730 {
731     ASSERT(!RunLoop::isMain());
732     m_database.runVacuumCommand();
733 }
734
735 void ResourceLoadStatisticsDatabaseStore::syncStorageImmediately()
736 {
737     ASSERT(!RunLoop::isMain());
738     m_database.runVacuumCommand();
739 }
740
741 void ResourceLoadStatisticsDatabaseStore::hasStorageAccess(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain, Optional<FrameIdentifier> frameID, PageIdentifier pageID, CompletionHandler<void(bool)>&& completionHandler)
742 {
743     ASSERT(!RunLoop::isMain());
744
745     ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
746
747     switch (cookieTreatmentForOrigin(subFrameDomain)) {
748     case CookieTreatmentResult::BlockAndPurge:
749         completionHandler(false);
750         return;
751     case CookieTreatmentResult::BlockAndKeep:
752         completionHandler(true);
753         return;
754     case CookieTreatmentResult::Allow:
755         // Do nothing
756         break;
757     };
758
759     RunLoop::main().dispatch([store = makeRef(store()), subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, completionHandler = WTFMove(completionHandler)]() mutable {
760         store->callHasStorageAccessForFrameHandler(subFrameDomain, topFrameDomain, frameID.value(), pageID, [store = store.copyRef(), completionHandler = WTFMove(completionHandler)](bool result) mutable {
761             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler), result] () mutable {
762                 completionHandler(result);
763             });
764         });
765     });
766 }
767
768 void ResourceLoadStatisticsDatabaseStore::requestStorageAccess(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, FrameIdentifier frameID, PageIdentifier pageID, CompletionHandler<void(StorageAccessStatus)>&& completionHandler)
769 {
770     ASSERT(!RunLoop::isMain());
771
772     auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
773
774     switch (cookieTreatmentForOrigin(subFrameDomain)) {
775     case CookieTreatmentResult::BlockAndPurge: {
776         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "Cannot grant storage access to %{public}s since its cookies are blocked in third-party contexts and it has not received user interaction as first-party.", subFrameDomain.string().utf8().data());
777         completionHandler(StorageAccessStatus::CannotRequestAccess);
778         }
779         return;
780     case CookieTreatmentResult::BlockAndKeep: {
781         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "No need to grant storage access to %{public}s since its cookies are not blocked in third-party contexts.", subFrameDomain.string().utf8().data());
782         completionHandler(StorageAccessStatus::HasAccess);
783         }
784         return;
785     case CookieTreatmentResult::Allow:
786         // Do nothing
787         break;
788     };
789
790     auto userWasPromptedEarlier = hasUserGrantedStorageAccessThroughPrompt(subFrameStatus.second, topFrameDomain);
791     if (userWasPromptedEarlier == StorageAccessPromptWasShown::No) {
792         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "About to ask the user whether they want to grant storage access to %{public}s under %{public}s or not.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
793         completionHandler(StorageAccessStatus::RequiresUserPrompt);
794         return;
795     }
796
797     if (userWasPromptedEarlier == StorageAccessPromptWasShown::Yes)
798         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "Storage access was granted to %{public}s under %{public}s.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
799
800     SQLiteStatement incrementStorageAccess(m_database, "UPDATE ObservedDomains SET timesAccessedAsFirstPartyDueToStorageAccessAPI = timesAccessedAsFirstPartyDueToStorageAccessAPI + 1 WHERE domainID = ?");
801     if (incrementStorageAccess.prepare() != SQLITE_OK
802         || incrementStorageAccess.bindInt(1, subFrameStatus.second) != SQLITE_OK
803         || incrementStorageAccess.step() != SQLITE_DONE) {
804         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::requestStorageAccess failed, error message: %{public}s", this, m_database.lastErrorMsg());
805         ASSERT_NOT_REACHED();
806         return;
807     }
808     
809     grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, userWasPromptedEarlier, [completionHandler = WTFMove(completionHandler)] (StorageAccessWasGranted wasGranted) mutable {
810         completionHandler(wasGranted == StorageAccessWasGranted::Yes ? StorageAccessStatus::HasAccess : StorageAccessStatus::CannotRequestAccess);
811     });
812 }
813
814 void ResourceLoadStatisticsDatabaseStore::requestStorageAccessUnderOpener(DomainInNeedOfStorageAccess&& domainInNeedOfStorageAccess, PageIdentifier openerPageID, OpenerDomain&& openerDomain)
815 {
816     ASSERT(domainInNeedOfStorageAccess != openerDomain);
817     ASSERT(!RunLoop::isMain());
818
819     if (domainInNeedOfStorageAccess == openerDomain)
820         return;
821
822     ensureResourceStatisticsForRegistrableDomain(domainInNeedOfStorageAccess);
823
824     if (cookieTreatmentForOrigin(domainInNeedOfStorageAccess) != CookieTreatmentResult::Allow)
825         return;
826
827     RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "[Temporary combatibility fix] Storage access was granted for %{public}s under opener page from %{public}s, with user interaction in the opened window.", domainInNeedOfStorageAccess.string().utf8().data(), openerDomain.string().utf8().data());
828     grantStorageAccessInternal(WTFMove(domainInNeedOfStorageAccess), WTFMove(openerDomain), WTF::nullopt, openerPageID, StorageAccessPromptWasShown::No, [](StorageAccessWasGranted) { });
829 }
830
831 void ResourceLoadStatisticsDatabaseStore::grantStorageAccess(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, FrameIdentifier frameID, PageIdentifier pageID, StorageAccessPromptWasShown promptWasShown, CompletionHandler<void(StorageAccessWasGranted)>&& completionHandler)
832 {
833     ASSERT(!RunLoop::isMain());
834
835     if (promptWasShown == StorageAccessPromptWasShown::Yes) {
836         auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
837         ASSERT(subFrameStatus.first == AddedRecord::No);
838         ASSERT(hasHadUserInteraction(subFrameDomain, OperatingDatesWindow::Long));
839         insertDomainRelationship(m_storageAccessUnderTopFrameDomainsStatement, subFrameStatus.second, topFrameDomain);
840     }
841
842     grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, promptWasShown, WTFMove(completionHandler));
843 }
844
845 void ResourceLoadStatisticsDatabaseStore::grantStorageAccessInternal(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, Optional<FrameIdentifier> frameID, PageIdentifier pageID, StorageAccessPromptWasShown promptWasShownNowOrEarlier, CompletionHandler<void(StorageAccessWasGranted)>&& completionHandler)
846 {
847     ASSERT(!RunLoop::isMain());
848
849     if (subFrameDomain == topFrameDomain) {
850         completionHandler(StorageAccessWasGranted::Yes);
851         return;
852     }
853
854     if (promptWasShownNowOrEarlier == StorageAccessPromptWasShown::Yes) {
855 #ifndef NDEBUG
856         auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
857         ASSERT(subFrameStatus.first == AddedRecord::No);
858         ASSERT(hasHadUserInteraction(subFrameDomain, OperatingDatesWindow::Long));
859         ASSERT(hasUserGrantedStorageAccessThroughPrompt(subFrameStatus.second, topFrameDomain) == StorageAccessPromptWasShown::Yes);
860 #endif
861         setUserInteraction(subFrameDomain, true, WallTime::now());
862     }
863
864     RunLoop::main().dispatch([subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, store = makeRef(store()), completionHandler = WTFMove(completionHandler)]() mutable {
865         store->callGrantStorageAccessHandler(subFrameDomain, topFrameDomain, frameID, pageID, [completionHandler = WTFMove(completionHandler), store = store.copyRef()](StorageAccessWasGranted wasGranted) mutable {
866             store->statisticsQueue().dispatch([wasGranted, completionHandler = WTFMove(completionHandler)] () mutable {
867                 completionHandler(wasGranted);
868             });
869         });
870     });
871
872 }
873
874 void ResourceLoadStatisticsDatabaseStore::grandfatherDataForDomains(const HashSet<RegistrableDomain>& domains)
875 {
876     ASSERT(!RunLoop::isMain());
877
878     for (auto& registrableDomain : domains)
879         ensureResourceStatisticsForRegistrableDomain(registrableDomain);
880
881     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET grandfathered = 1 WHERE registrableDomain IN (", domainsToString(domains), ")"));
882     if (domainsToUpdateStatement.prepare() != SQLITE_OK
883         || domainsToUpdateStatement.step() != SQLITE_DONE) {
884         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::grandfatherDataForDomains failed, error message: %{public}s", this, m_database.lastErrorMsg());
885         ASSERT_NOT_REACHED();
886     }
887 }
888
889 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::ensurePrevalentResourcesForDebugMode()
890 {
891     ASSERT(!RunLoop::isMain());
892
893     if (!debugModeEnabled())
894         return { };
895
896     Vector<RegistrableDomain> primaryDomainsToBlock;
897     primaryDomainsToBlock.reserveInitialCapacity(2);
898
899     ensureResourceStatisticsForRegistrableDomain(debugStaticPrevalentResource());
900     setPrevalentResource(debugStaticPrevalentResource(), ResourceLoadPrevalence::High);
901     primaryDomainsToBlock.uncheckedAppend(debugStaticPrevalentResource());
902
903     if (!debugManualPrevalentResource().isEmpty()) {
904         ensureResourceStatisticsForRegistrableDomain(debugManualPrevalentResource());
905         setPrevalentResource(debugManualPrevalentResource(), ResourceLoadPrevalence::High);
906         primaryDomainsToBlock.uncheckedAppend(debugManualPrevalentResource());
907         RELEASE_LOG_INFO(ITPDebug, "Did set %{public}s as prevalent resource for the purposes of ITP Debug Mode.", debugManualPrevalentResource().string().utf8().data());
908     }
909
910     return primaryDomainsToBlock;
911 }
912
913 void ResourceLoadStatisticsDatabaseStore::logFrameNavigation(const RegistrableDomain& targetDomain, const RegistrableDomain& topFrameDomain, const RegistrableDomain& sourceDomain, bool isRedirect, bool isMainFrame)
914 {
915     ASSERT(!RunLoop::isMain());
916
917     bool areTargetAndTopFrameDomainsSameSite = targetDomain == topFrameDomain;
918     bool areTargetAndSourceDomainsSameSite = targetDomain == sourceDomain;
919
920     bool statisticsWereUpdated = false;
921     if (!isMainFrame && !(areTargetAndTopFrameDomainsSameSite || areTargetAndSourceDomainsSameSite)) {
922         auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
923         updateLastSeen(targetDomain, ResourceLoadStatistics::reduceTimeResolution(WallTime::now()));
924         ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
925         if (!relationshipExists(m_subframeUnderTopFrameDomainExists, targetResult.second, topFrameDomain)) {
926             insertDomainRelationship(m_subframeUnderTopFrameDomains, targetResult.second, topFrameDomain);
927             statisticsWereUpdated = true;
928         }
929     }
930
931     if (isRedirect && !areTargetAndSourceDomainsSameSite) {
932         if (isMainFrame) {
933             auto redirectingDomainResult = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
934             auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
935
936             if (!relationshipExists(m_topFrameUniqueRedirectsToExists, redirectingDomainResult.second, targetDomain)) {
937                 insertDomainRelationship(m_topFrameUniqueRedirectsTo, redirectingDomainResult.second, targetDomain);
938                 statisticsWereUpdated = true;
939             }
940
941             if (!relationshipExists(m_topFrameUniqueRedirectsFromExists, targetResult.second, sourceDomain)) {
942                 insertDomainRelationship(m_topFrameUniqueRedirectsFrom, targetResult.second, sourceDomain);
943                 statisticsWereUpdated = true;
944             }
945         } else {
946             auto redirectingDomainResult = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
947             auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
948             if (!relationshipExists(m_subresourceUniqueRedirectsToExists, redirectingDomainResult.second, targetDomain)) {
949                 insertDomainRelationship(m_subresourceUniqueRedirectsTo, redirectingDomainResult.second, targetDomain);
950                 statisticsWereUpdated = true;
951             }
952
953             if (!relationshipExists(m_subresourceUniqueRedirectsFromExists, targetResult.second, sourceDomain)) {
954                 insertDomainRelationship(m_subresourceUniqueRedirectsFrom, targetResult.second, sourceDomain);
955                 statisticsWereUpdated = true;
956             }
957         }
958     }
959
960     if (statisticsWereUpdated)
961         scheduleStatisticsProcessingRequestIfNecessary();
962 }
963
964 void ResourceLoadStatisticsDatabaseStore::logCrossSiteLoadWithLinkDecoration(const NavigatedFromDomain& fromDomain, const NavigatedToDomain& toDomain)
965 {
966     ASSERT(!RunLoop::isMain());
967     ASSERT(fromDomain != toDomain);
968
969     auto fromDomainResult = ensureResourceStatisticsForRegistrableDomain(fromDomain);
970
971     if (!relationshipExists(m_topFrameLinkDecorationsFromExists, fromDomainResult.second, toDomain)) {
972         insertDomainRelationship(m_topFrameLinkDecorationsFrom, fromDomainResult.second, toDomain);
973         scheduleStatisticsProcessingRequestIfNecessary();
974     }
975 }
976
977 void ResourceLoadStatisticsDatabaseStore::setUserInteraction(const RegistrableDomain& domain, bool hadUserInteraction, WallTime mostRecentInteraction)
978 {
979     ASSERT(!RunLoop::isMain());
980
981     if (m_mostRecentUserInteractionStatement.bindInt(1, hadUserInteraction) != SQLITE_OK
982         || m_mostRecentUserInteractionStatement.bindDouble(2, mostRecentInteraction.secondsSinceEpoch().value()) != SQLITE_OK
983         || m_mostRecentUserInteractionStatement.bindText(3, domain.string()) != SQLITE_OK
984         || m_mostRecentUserInteractionStatement.step() != SQLITE_DONE) {
985         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::setUserInteraction, error message: %{public}s", this, m_database.lastErrorMsg());
986         ASSERT_NOT_REACHED();
987         return;
988     }
989
990     int resetResult = m_mostRecentUserInteractionStatement.reset();
991     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
992 }
993
994 void ResourceLoadStatisticsDatabaseStore::logUserInteraction(const TopFrameDomain& domain)
995 {
996     ASSERT(!RunLoop::isMain());
997
998     ensureResourceStatisticsForRegistrableDomain(domain);
999     setUserInteraction(domain, true, WallTime::now());
1000 }
1001
1002 void ResourceLoadStatisticsDatabaseStore::clearUserInteraction(const RegistrableDomain& domain)
1003 {
1004     ASSERT(!RunLoop::isMain());
1005
1006     auto targetResult = ensureResourceStatisticsForRegistrableDomain(domain);
1007     setUserInteraction(domain, false, { });
1008
1009     SQLiteStatement removeStorageAccess(m_database, "DELETE FROM StorageAccessUnderTopFrameDomains WHERE domainID = ?");
1010     if (removeStorageAccess.prepare() != SQLITE_OK
1011         || removeStorageAccess.bindInt(1, targetResult.second) != SQLITE_OK
1012         || removeStorageAccess.step() != SQLITE_DONE) {
1013         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::logUserInteraction failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1014         ASSERT_NOT_REACHED();
1015     }
1016 }
1017
1018 bool ResourceLoadStatisticsDatabaseStore::hasHadUserInteraction(const RegistrableDomain& domain, OperatingDatesWindow operatingDatesWindow)
1019 {
1020     ASSERT(!RunLoop::isMain());
1021
1022     if (m_hadUserInteractionStatement.bindText(1, domain.string()) != SQLITE_OK
1023         || m_hadUserInteractionStatement.step() != SQLITE_ROW) {
1024         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_hadUserInteractionStatement failed, error message: %{public}s", this, m_database.lastErrorMsg());
1025
1026         int resetResult = m_hadUserInteractionStatement.reset();
1027         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1028         return false;
1029     }
1030
1031     bool hadUserInteraction = !!m_hadUserInteractionStatement.getColumnInt(0);
1032     if (!hadUserInteraction) {
1033         int resetResult = m_hadUserInteractionStatement.reset();
1034         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1035         return false;
1036     }
1037     
1038     WallTime mostRecentUserInteractionTime = WallTime::fromRawSeconds(m_hadUserInteractionStatement.getColumnDouble(1));
1039
1040     if (hasStatisticsExpired(mostRecentUserInteractionTime, operatingDatesWindow)) {
1041         // Drop privacy sensitive data because we no longer need it.
1042         // Set timestamp to 0 so that statistics merge will know
1043         // it has been reset as opposed to its default -1.
1044         clearUserInteraction(domain);
1045         hadUserInteraction = false;
1046     }
1047     
1048     int resetResult = m_hadUserInteractionStatement.reset();
1049     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1050
1051     return hadUserInteraction;
1052 }
1053
1054 void ResourceLoadStatisticsDatabaseStore::setPrevalentResource(const RegistrableDomain& domain, ResourceLoadPrevalence newPrevalence)
1055 {
1056     ASSERT(!RunLoop::isMain());
1057     if (shouldSkip(domain))
1058         return;
1059
1060     if (m_updatePrevalentResourceStatement.bindInt(1, 1) != SQLITE_OK
1061         || m_updatePrevalentResourceStatement.bindText(2, domain.string()) != SQLITE_OK
1062         || m_updatePrevalentResourceStatement.step() != SQLITE_DONE) {
1063         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_updatePrevalentResourceStatement failed, error message: %{public}s", this, m_database.lastErrorMsg());
1064         ASSERT_NOT_REACHED();
1065         return;
1066     }
1067     
1068     int resetResult = m_updatePrevalentResourceStatement.reset();
1069     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1070
1071     if (newPrevalence == ResourceLoadPrevalence::VeryHigh) {
1072         if (m_updateVeryPrevalentResourceStatement.bindInt(1, 1) != SQLITE_OK
1073             || m_updateVeryPrevalentResourceStatement.bindText(2, domain.string()) != SQLITE_OK
1074             || m_updateVeryPrevalentResourceStatement.step() != SQLITE_DONE) {
1075             RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_updateVeryPrevalentResourceStatement failed, error message: %{public}s", this, m_database.lastErrorMsg());
1076             ASSERT_NOT_REACHED();
1077             return;
1078         }
1079
1080         int resetResult = m_updateVeryPrevalentResourceStatement.reset();
1081         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1082     }
1083
1084     StdSet<unsigned> nonPrevalentRedirectionSources;
1085     recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(domainID(domain).value(), nonPrevalentRedirectionSources, 0);
1086     setDomainsAsPrevalent(WTFMove(nonPrevalentRedirectionSources));
1087 }
1088
1089 void ResourceLoadStatisticsDatabaseStore::setDomainsAsPrevalent(StdSet<unsigned>&& domains)
1090 {
1091     ASSERT(!RunLoop::isMain());
1092
1093     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET isPrevalent = 1 WHERE domainID IN (", buildList(WTF::IteratorRange<StdSet<unsigned>::iterator>(domains.begin(), domains.end())), ")"));
1094     if (domainsToUpdateStatement.prepare() != SQLITE_OK
1095         || domainsToUpdateStatement.step() != SQLITE_DONE) {
1096         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::setDomainsAsPrevalent failed, error message: %{public}s", this, m_database.lastErrorMsg());
1097         ASSERT_NOT_REACHED();
1098         return;
1099     }
1100 }
1101
1102 void ResourceLoadStatisticsDatabaseStore::dumpResourceLoadStatistics(CompletionHandler<void(const String&)>&& completionHandler)
1103 {
1104     ASSERT(!RunLoop::isMain());
1105
1106     // FIXME(195088): Implement SQLite-based dumping routines.
1107     ASSERT_NOT_REACHED();
1108     completionHandler({ });
1109 }
1110
1111 bool ResourceLoadStatisticsDatabaseStore::predicateValueForDomain(WebCore::SQLiteStatement& predicateStatement, const RegistrableDomain& domain) const
1112 {
1113     ASSERT(!RunLoop::isMain());
1114     
1115     if (predicateStatement.bindText(1, domain.string()) != SQLITE_OK
1116         || predicateStatement.step() != SQLITE_ROW) {
1117
1118         int resetResult = predicateStatement.reset();
1119         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1120
1121         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::predicateValueForDomain failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1122         return false;
1123     }
1124
1125     bool result = !!predicateStatement.getColumnInt(0);
1126     int resetResult = predicateStatement.reset();
1127     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1128     return result;
1129 }
1130
1131 bool ResourceLoadStatisticsDatabaseStore::isPrevalentResource(const RegistrableDomain& domain) const
1132 {
1133     ASSERT(!RunLoop::isMain());
1134
1135     if (shouldSkip(domain))
1136         return false;
1137
1138     return predicateValueForDomain(m_isPrevalentResourceStatement, domain);
1139 }
1140
1141 bool ResourceLoadStatisticsDatabaseStore::isVeryPrevalentResource(const RegistrableDomain& domain) const
1142 {
1143     ASSERT(!RunLoop::isMain());
1144
1145     if (shouldSkip(domain))
1146         return false;
1147
1148     return predicateValueForDomain(m_isVeryPrevalentResourceStatement, domain);
1149 }
1150
1151 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsSubresourceUnder(const SubResourceDomain& subresourceDomain, const TopFrameDomain& topFrameDomain) const
1152 {
1153     ASSERT(!RunLoop::isMain());
1154
1155     return relationshipExists(m_subresourceUnderTopFrameDomainExists, domainID(subresourceDomain), topFrameDomain);
1156 }
1157
1158 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsSubFrameUnder(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain) const
1159 {
1160     ASSERT(!RunLoop::isMain());
1161
1162     return relationshipExists(m_subframeUnderTopFrameDomainExists, domainID(subFrameDomain), topFrameDomain);
1163 }
1164
1165 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsRedirectingTo(const RedirectedFromDomain& redirectedFromDomain, const RedirectedToDomain& redirectedToDomain) const
1166 {
1167     ASSERT(!RunLoop::isMain());
1168
1169     return relationshipExists(m_subresourceUniqueRedirectsToExists, domainID(redirectedFromDomain), redirectedToDomain);
1170 }
1171
1172 void ResourceLoadStatisticsDatabaseStore::clearPrevalentResource(const RegistrableDomain& domain)
1173 {
1174     ASSERT(!RunLoop::isMain());
1175
1176     ensureResourceStatisticsForRegistrableDomain(domain);
1177     
1178     if (m_clearPrevalentResourceStatement.bindText(1, domain.string()) != SQLITE_OK
1179         || m_clearPrevalentResourceStatement.step() != SQLITE_DONE) {
1180         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::clearPrevalentResource, error message: %{public}s", this, m_database.lastErrorMsg());
1181         ASSERT_NOT_REACHED();
1182         return;
1183     }
1184     
1185     int resetResult = m_clearPrevalentResourceStatement.reset();
1186     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1187 }
1188
1189 void ResourceLoadStatisticsDatabaseStore::setGrandfathered(const RegistrableDomain& domain, bool value)
1190 {
1191     ASSERT(!RunLoop::isMain());
1192
1193     ensureResourceStatisticsForRegistrableDomain(domain);
1194     
1195     if (m_updateGrandfatheredStatement.bindInt(1, value) != SQLITE_OK
1196         || m_updateGrandfatheredStatement.bindText(1, domain.string()) != SQLITE_OK
1197         || m_updateGrandfatheredStatement.step() != SQLITE_DONE) {
1198         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::setGrandfathered failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1199         ASSERT_NOT_REACHED();
1200         return;
1201     }
1202     
1203     int resetResult = m_updateGrandfatheredStatement.reset();
1204     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1205 }
1206
1207 bool ResourceLoadStatisticsDatabaseStore::isGrandfathered(const RegistrableDomain& domain) const
1208 {
1209     ASSERT(!RunLoop::isMain());
1210
1211     return predicateValueForDomain(m_isGrandfatheredStatement, domain);
1212 }
1213
1214 void ResourceLoadStatisticsDatabaseStore::setSubframeUnderTopFrameDomain(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain)
1215 {
1216     ASSERT(!RunLoop::isMain());
1217
1218     auto result = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
1219
1220     // For consistency, make sure we also have a statistics entry for the top frame domain.
1221     ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1222
1223     insertDomainRelationship(m_subframeUnderTopFrameDomains, result.second, topFrameDomain);
1224 }
1225
1226 void ResourceLoadStatisticsDatabaseStore::setSubresourceUnderTopFrameDomain(const SubResourceDomain& subresourceDomain, const TopFrameDomain& topFrameDomain)
1227 {
1228     ASSERT(!RunLoop::isMain());
1229
1230     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1231
1232     // For consistency, make sure we also have a statistics entry for the top frame domain.
1233     ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1234
1235     insertDomainRelationship(m_subresourceUnderTopFrameDomains, result.second, topFrameDomain);
1236 }
1237
1238 void ResourceLoadStatisticsDatabaseStore::setSubresourceUniqueRedirectTo(const SubResourceDomain& subresourceDomain, const RedirectDomain& redirectDomain)
1239 {
1240     ASSERT(!RunLoop::isMain());
1241
1242     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1243
1244     // For consistency, make sure we also have a statistics entry for the redirect domain.
1245     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1246
1247     insertDomainRelationship(m_subresourceUniqueRedirectsTo, result.second, redirectDomain);
1248 }
1249
1250 void ResourceLoadStatisticsDatabaseStore::setSubresourceUniqueRedirectFrom(const SubResourceDomain& subresourceDomain, const RedirectDomain& redirectDomain)
1251 {
1252     ASSERT(!RunLoop::isMain());
1253
1254     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1255
1256     // For consistency, make sure we also have a statistics entry for the redirect domain.
1257     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1258
1259     insertDomainRelationship(m_subresourceUniqueRedirectsFrom, result.second, redirectDomain);
1260 }
1261
1262 void ResourceLoadStatisticsDatabaseStore::setTopFrameUniqueRedirectTo(const TopFrameDomain& topFrameDomain, const RedirectDomain& redirectDomain)
1263 {
1264     ASSERT(!RunLoop::isMain());
1265
1266     auto result = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1267
1268     // For consistency, make sure we also have a statistics entry for the redirect domain.
1269     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1270
1271     insertDomainRelationship(m_topFrameUniqueRedirectsTo, result.second, redirectDomain);
1272 }
1273
1274 void ResourceLoadStatisticsDatabaseStore::setTopFrameUniqueRedirectFrom(const TopFrameDomain& topFrameDomain, const RedirectDomain& redirectDomain)
1275 {
1276     ASSERT(!RunLoop::isMain());
1277
1278     auto result = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1279
1280     // For consistency, make sure we also have a statistics entry for the redirect domain.
1281     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1282
1283     insertDomainRelationship(m_topFrameUniqueRedirectsFrom, result.second, redirectDomain);
1284 }
1285
1286 std::pair<ResourceLoadStatisticsDatabaseStore::AddedRecord, unsigned> ResourceLoadStatisticsDatabaseStore::ensureResourceStatisticsForRegistrableDomain(const RegistrableDomain& domain)
1287 {
1288     ASSERT(!RunLoop::isMain());
1289
1290     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK) {
1291         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::ensureResourceStatisticsForRegistrableDomain failed, error message: %{public}s", this, m_database.lastErrorMsg());
1292         ASSERT_NOT_REACHED();
1293         return { AddedRecord::No, 0 };
1294     }
1295     
1296     if (m_domainIDFromStringStatement.step() == SQLITE_ROW) {
1297         unsigned domainID = m_domainIDFromStringStatement.getColumnInt(0);
1298
1299         int resetResult = m_domainIDFromStringStatement.reset();
1300         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1301         return { AddedRecord::No, domainID };
1302     }
1303
1304     int resetResult = m_domainIDFromStringStatement.reset();
1305     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1306
1307     ResourceLoadStatistics newObservation(domain);
1308     insertObservedDomain(newObservation);
1309
1310     return { AddedRecord::Yes, domainID(domain).value() };
1311 }
1312
1313 void ResourceLoadStatisticsDatabaseStore::clearDatabaseContents()
1314 {
1315     m_database.clearAllTables();
1316
1317     if (!createSchema()) {
1318         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::clearDatabaseContents failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
1319         ASSERT_NOT_REACHED();
1320         return;
1321     }
1322 }
1323
1324 void ResourceLoadStatisticsDatabaseStore::clear(CompletionHandler<void()>&& completionHandler)
1325 {
1326     ASSERT(!RunLoop::isMain());
1327
1328     clearDatabaseContents();
1329     clearOperatingDates();
1330
1331     auto callbackAggregator = CallbackAggregator::create(WTFMove(completionHandler));
1332
1333     removeAllStorageAccess([callbackAggregator = callbackAggregator.copyRef()] { });
1334
1335     auto registrableDomainsToBlockAndDeleteCookiesFor = ensurePrevalentResourcesForDebugMode();
1336     RegistrableDomainsToBlockCookiesFor domainsToBlock { registrableDomainsToBlockAndDeleteCookiesFor, { } };
1337     updateCookieBlockingForDomains(domainsToBlock, [callbackAggregator = callbackAggregator.copyRef()] { });
1338 }
1339
1340 ResourceLoadStatisticsDatabaseStore::CookieTreatmentResult ResourceLoadStatisticsDatabaseStore::cookieTreatmentForOrigin(const RegistrableDomain& domain) const
1341 {
1342     ASSERT(!RunLoop::isMain());
1343
1344     SQLiteStatement statement(m_database, "SELECT isPrevalent, hadUserInteraction FROM ObservedDomains WHERE registrableDomain = ?");
1345     if (statement.prepare() != SQLITE_OK
1346         || statement.bindText(1, domain.string()) != SQLITE_OK) {
1347         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::cookieTreatmentForOrigin failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1348         ASSERT_NOT_REACHED();
1349     }
1350     
1351     if (statement.step() != SQLITE_ROW)
1352         return CookieTreatmentResult::Allow;
1353     
1354     bool isPrevalent = !!statement.getColumnInt(0);
1355     if (!isPrevalent)
1356         return CookieTreatmentResult::Allow;
1357
1358     bool hadUserInteraction = statement.getColumnInt(1) ? true : false;
1359     return hadUserInteraction ? CookieTreatmentResult::BlockAndKeep : CookieTreatmentResult::BlockAndPurge;
1360 }
1361     
1362 StorageAccessPromptWasShown ResourceLoadStatisticsDatabaseStore::hasUserGrantedStorageAccessThroughPrompt(unsigned requestingDomainID, const RegistrableDomain& firstPartyDomain) const
1363 {
1364     ASSERT(!RunLoop::isMain());
1365
1366     auto firstPartyPrimaryDomainID = domainID(firstPartyDomain).value();
1367
1368     SQLiteStatement statement(m_database, "SELECT COUNT(*) FROM StorageAccessUnderTopFrameDomains WHERE domainID = ? AND topLevelDomainID = ?");
1369     if (statement.prepare() != SQLITE_OK
1370         || statement.bindInt(1, requestingDomainID) != SQLITE_OK
1371         || statement.bindInt(2, firstPartyPrimaryDomainID) != SQLITE_OK
1372         || statement.step() != SQLITE_ROW)
1373         return StorageAccessPromptWasShown::No;
1374
1375     return !statement.getColumnInt(0) ? StorageAccessPromptWasShown::Yes : StorageAccessPromptWasShown::No;
1376 }
1377
1378 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::domainsToBlockAndDeleteCookiesFor() const
1379 {
1380     ASSERT(!RunLoop::isMain());
1381     
1382     Vector<RegistrableDomain> results;
1383     SQLiteStatement statement(m_database, "SELECT registrableDomain FROM ObservedDomains WHERE isPrevalent = 1 AND hadUserInteraction = 0"_s);
1384     if (statement.prepare() != SQLITE_OK)
1385         return results;
1386     
1387     while (statement.step() == SQLITE_ROW)
1388         results.append(RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(0)));
1389     
1390     return results;
1391 }
1392
1393 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::domainsToBlockButKeepCookiesFor() const
1394 {
1395     ASSERT(!RunLoop::isMain());
1396
1397     Vector<RegistrableDomain> results;
1398     SQLiteStatement statement(m_database, "SELECT registrableDomain FROM ObservedDomains WHERE isPrevalent = 1 AND hadUserInteraction = 1"_s);
1399     if (statement.prepare() != SQLITE_OK)
1400         return results;
1401     
1402     while (statement.step() == SQLITE_ROW)
1403         results.append(RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(0)));
1404
1405     return results;
1406 }
1407
1408 void ResourceLoadStatisticsDatabaseStore::updateCookieBlocking(CompletionHandler<void()>&& completionHandler)
1409 {
1410     ASSERT(!RunLoop::isMain());
1411
1412     auto domainsToBlockAndDeleteCookiesFor = this->domainsToBlockAndDeleteCookiesFor();
1413     auto domainsToBlockButKeepCookiesFor = this->domainsToBlockButKeepCookiesFor();
1414
1415     if (domainsToBlockAndDeleteCookiesFor.isEmpty() && domainsToBlockButKeepCookiesFor.isEmpty()) {
1416         completionHandler();
1417         return;
1418     }
1419
1420     RegistrableDomainsToBlockCookiesFor domainsToBlock { domainsToBlockAndDeleteCookiesFor, domainsToBlockButKeepCookiesFor };
1421
1422     if (debugLoggingEnabled() && !domainsToBlockAndDeleteCookiesFor.isEmpty() && !domainsToBlockButKeepCookiesFor.isEmpty())
1423         debugLogDomainsInBatches("block", domainsToBlock);
1424
1425     RunLoop::main().dispatch([weakThis = makeWeakPtr(*this), store = makeRef(store()), domainsToBlock = crossThreadCopy(domainsToBlock), completionHandler = WTFMove(completionHandler)] () mutable {
1426         store->callUpdatePrevalentDomainsToBlockCookiesForHandler(domainsToBlock, [weakThis = WTFMove(weakThis), store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1427             store->statisticsQueue().dispatch([weakThis = WTFMove(weakThis), completionHandler = WTFMove(completionHandler)]() mutable {
1428                 completionHandler();
1429                 if (!weakThis)
1430                     return;
1431                 RELEASE_LOG_INFO_IF(weakThis->debugLoggingEnabled(), ITPDebug, "Done updating cookie blocking.");
1432             });
1433         });
1434     });
1435 }
1436
1437 Vector<ResourceLoadStatisticsDatabaseStore::PrevalentDomainData> ResourceLoadStatisticsDatabaseStore::prevalentDomains() const
1438 {
1439     ASSERT(!RunLoop::isMain());
1440     
1441     Vector<PrevalentDomainData> results;
1442     SQLiteStatement statement(m_database, "SELECT domainID, registrableDomain, mostRecentUserInteractionTime, hadUserInteraction, grandfathered FROM ObservedDomains WHERE isPrevalent = 1"_s);
1443     if (statement.prepare() != SQLITE_OK)
1444         return results;
1445     
1446     while (statement.step() == SQLITE_ROW) {
1447         results.append({ static_cast<unsigned>(statement.getColumnInt(0))
1448             , RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(1))
1449             , WallTime::fromRawSeconds(statement.getColumnDouble(2))
1450             , statement.getColumnInt(3) ? true : false
1451             , statement.getColumnInt(4) ? true : false
1452         });
1453     }
1454     
1455     return results;
1456 }
1457
1458 Vector<unsigned> ResourceLoadStatisticsDatabaseStore::findExpiredUserInteractions() const
1459 {
1460     ASSERT(!RunLoop::isMain());
1461
1462     Vector<unsigned> results;
1463     Optional<Seconds> expirationDateTime = statisticsEpirationTime();
1464     if (!expirationDateTime)
1465         return results;
1466
1467     if (m_findExpiredUserInteractionStatement.bindDouble(1, expirationDateTime.value().value()) != SQLITE_OK)
1468         return results;
1469
1470     while (m_findExpiredUserInteractionStatement.step() == SQLITE_ROW)
1471         results.append(m_findExpiredUserInteractionStatement.getColumnInt(0));
1472
1473     int resetResult = m_findExpiredUserInteractionStatement.reset();
1474     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1475
1476     return results;
1477 }
1478
1479 void ResourceLoadStatisticsDatabaseStore::clearExpiredUserInteractions()
1480 {
1481     ASSERT(!RunLoop::isMain());
1482
1483     auto expiredRecords = findExpiredUserInteractions();
1484     if (expiredRecords.isEmpty())
1485         return;
1486
1487     auto expiredRecordIDs = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(expiredRecords.begin(), expiredRecords.end()));
1488
1489     SQLiteStatement clearExpiredInteraction(m_database, makeString("UPDATE ObservedDomains SET mostRecentUserInteractionTime = 0, hadUserInteraction = 1 WHERE domainID IN (", expiredRecordIDs, ")"));
1490     if (clearExpiredInteraction.prepare() != SQLITE_OK)
1491         return;
1492
1493     SQLiteStatement removeStorageAccess(m_database, makeString("DELETE FROM StorageAccessUnderTopFrameDomains ", expiredRecordIDs, ")"));
1494     if (removeStorageAccess.prepare() != SQLITE_OK)
1495         return;
1496
1497     if (clearExpiredInteraction.step() != SQLITE_DONE
1498         || removeStorageAccess.step() != SQLITE_DONE) {
1499         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::clearExpiredUserInteractions statement(s) failed to step, error message: %{public}s", this, m_database.lastErrorMsg());
1500         ASSERT_NOT_REACHED();
1501     }
1502 }
1503
1504 void ResourceLoadStatisticsDatabaseStore::clearGrandfathering(Vector<unsigned>&& domainIDsToClear)
1505 {
1506     ASSERT(!RunLoop::isMain());
1507
1508     if (domainIDsToClear.isEmpty())
1509         return;
1510
1511     auto listToClear = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(domainIDsToClear.begin(), domainIDsToClear.end()));
1512
1513     SQLiteStatement clearGrandfatheringStatement(m_database, makeString("UPDATE ObservedDomains SET grandfathered = 0 WHERE domainID IN (", listToClear, ")"));
1514     if (clearGrandfatheringStatement.prepare() != SQLITE_OK)
1515         return;
1516     
1517     if (clearGrandfatheringStatement.step() != SQLITE_DONE) {
1518         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::clearGrandfathering failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1519         ASSERT_NOT_REACHED();
1520     }
1521 }
1522
1523 bool ResourceLoadStatisticsDatabaseStore::hasHadUnexpiredRecentUserInteraction(const PrevalentDomainData& resourceStatistic, OperatingDatesWindow operatingDatesWindow)
1524 {
1525     if (resourceStatistic.hadUserInteraction && hasStatisticsExpired(resourceStatistic.mostRecentUserInteractionTime, operatingDatesWindow)) {
1526         clearUserInteraction(resourceStatistic.registrableDomain);
1527         return false;
1528     }
1529
1530     return resourceStatistic.hadUserInteraction;
1531 }
1532
1533 bool ResourceLoadStatisticsDatabaseStore::shouldRemoveAllWebsiteDataFor(const PrevalentDomainData& resourceStatistic, bool shouldCheckForGrandfathering)
1534 {
1535     return !hasHadUnexpiredRecentUserInteraction(resourceStatistic, OperatingDatesWindow::Long) && (!shouldCheckForGrandfathering || !resourceStatistic.grandfathered);
1536 }
1537
1538 bool ResourceLoadStatisticsDatabaseStore::shouldRemoveAllButCookiesFor(const PrevalentDomainData& resourceStatistic, bool shouldCheckForGrandfathering) const
1539 {
1540     UNUSED_PARAM(resourceStatistic);
1541     UNUSED_PARAM(shouldCheckForGrandfathering);
1542     return false;
1543 }
1544
1545 Vector<std::pair<RegistrableDomain, WebsiteDataToRemove>> ResourceLoadStatisticsDatabaseStore::registrableDomainsToRemoveWebsiteDataFor()
1546 {
1547     ASSERT(!RunLoop::isMain());
1548
1549     bool shouldCheckForGrandfathering = endOfGrandfatheringTimestamp() > WallTime::now();
1550     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && endOfGrandfatheringTimestamp();
1551
1552     if (shouldClearGrandfathering)
1553         clearEndOfGrandfatheringTimeStamp();
1554
1555     clearExpiredUserInteractions();
1556     
1557     Vector<std::pair<RegistrableDomain, WebsiteDataToRemove>> domainsToRemoveWebsiteDataFor;
1558
1559     Vector<PrevalentDomainData> prevalentDomains = this->prevalentDomains();
1560     Vector<unsigned> domainIDsToClearGrandfathering;
1561     for (auto& statistic : prevalentDomains) {
1562         if (shouldRemoveAllWebsiteDataFor(statistic, shouldCheckForGrandfathering))
1563             domainsToRemoveWebsiteDataFor.append(std::make_pair(statistic.registrableDomain, WebsiteDataToRemove::All));
1564         else if (shouldRemoveAllButCookiesFor(statistic, shouldCheckForGrandfathering))
1565             domainsToRemoveWebsiteDataFor.append(std::make_pair(statistic.registrableDomain, WebsiteDataToRemove::AllButCookies));
1566
1567         if (shouldClearGrandfathering && statistic.grandfathered)
1568             domainIDsToClearGrandfathering.append(statistic.domainID);
1569     }
1570
1571     clearGrandfathering(WTFMove(domainIDsToClearGrandfathering));
1572     
1573     return domainsToRemoveWebsiteDataFor;
1574 }
1575
1576 void ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded()
1577 {
1578     ASSERT(!RunLoop::isMain());
1579
1580     unsigned count = 0;
1581     if (m_observedDomainCount.step() == SQLITE_ROW)
1582         count = m_observedDomainCount.getColumnInt(0);
1583
1584     int resetResult = m_observedDomainCount.reset();
1585     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1586
1587     if (count <= parameters().maxStatisticsEntries)
1588         return;
1589
1590     ASSERT(parameters().pruneEntriesDownTo <= parameters().maxStatisticsEntries);
1591
1592     size_t countLeftToPrune = count - parameters().pruneEntriesDownTo;
1593     ASSERT(countLeftToPrune);
1594
1595     SQLiteStatement recordsToPrune(m_database, "SELECT domainID FROM ObservedDomains ORDER BY hadUserInteraction, isPrevalent, lastSeen LIMIT ?");
1596     if (recordsToPrune.prepare() != SQLITE_OK
1597         || recordsToPrune.bindInt(1, countLeftToPrune) != SQLITE_OK) {
1598         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded failed, error message: %{public}s", this, m_database.lastErrorMsg());
1599         ASSERT_NOT_REACHED();
1600         return;
1601     }
1602
1603     Vector<unsigned> entriesToPrune;
1604     while (recordsToPrune.step() == SQLITE_ROW)
1605         entriesToPrune.append(recordsToPrune.getColumnInt(0));
1606
1607     auto listToPrune = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(entriesToPrune.begin(), entriesToPrune.end()));
1608
1609     SQLiteStatement pruneCommand(m_database, makeString("DELETE from ObservedDomains WHERE domainID IN (", listToPrune, ")"));
1610     if (pruneCommand.prepare() != SQLITE_OK
1611         || pruneCommand.step() != SQLITE_DONE) {
1612         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded failed, error message: %{public}s", this, m_database.lastErrorMsg());
1613         ASSERT_NOT_REACHED();
1614         return;
1615     }
1616 }
1617
1618 void ResourceLoadStatisticsDatabaseStore::updateLastSeen(const RegistrableDomain& domain, WallTime lastSeen)
1619 {
1620     ASSERT(!RunLoop::isMain());
1621
1622     if (m_updateLastSeenStatement.bindDouble(1, lastSeen.secondsSinceEpoch().value()) != SQLITE_OK
1623         || m_updateLastSeenStatement.bindText(2, domain.string()) != SQLITE_OK
1624         || m_updateLastSeenStatement.step() != SQLITE_DONE) {
1625         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::updateLastSeen failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1626         ASSERT_NOT_REACHED();
1627         return;
1628     }
1629     
1630     int resetResult = m_updateLastSeenStatement.reset();
1631     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1632 }
1633
1634 void ResourceLoadStatisticsDatabaseStore::setLastSeen(const RegistrableDomain& domain, Seconds seconds)
1635 {
1636     ASSERT(!RunLoop::isMain());
1637
1638     ensureResourceStatisticsForRegistrableDomain(domain);
1639     updateLastSeen(domain, WallTime::fromRawSeconds(seconds.seconds()));
1640 }
1641
1642 void ResourceLoadStatisticsDatabaseStore::setPrevalentResource(const RegistrableDomain& domain)
1643 {
1644     ASSERT(!RunLoop::isMain());
1645
1646     if (shouldSkip(domain))
1647         return;
1648
1649     ensureResourceStatisticsForRegistrableDomain(domain);
1650     setPrevalentResource(domain, ResourceLoadPrevalence::High);
1651 }
1652
1653 void ResourceLoadStatisticsDatabaseStore::setVeryPrevalentResource(const RegistrableDomain& domain)
1654 {
1655     ASSERT(!RunLoop::isMain());
1656
1657     if (shouldSkip(domain))
1658         return;
1659     
1660     ensureResourceStatisticsForRegistrableDomain(domain);
1661     setPrevalentResource(domain, ResourceLoadPrevalence::VeryHigh);
1662 }
1663
1664 } // namespace WebKit
1665
1666 #endif