Resource Load Statistics: Switch NSURLSession on top navigation to prevalent resource...
[WebKit.git] / Source / WebKit / NetworkProcess / Classifier / ResourceLoadStatisticsDatabaseStore.cpp
1 /*
2  * Copyright (C) 2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ResourceLoadStatisticsDatabaseStore.h"
28
29 #if ENABLE(RESOURCE_LOAD_STATISTICS)
30
31 #include "Logging.h"
32 #include "NetworkSession.h"
33 #include "PluginProcessManager.h"
34 #include "PluginProcessProxy.h"
35 #include "ResourceLoadStatisticsMemoryStore.h"
36 #include "StorageAccessStatus.h"
37 #include "WebProcessProxy.h"
38 #include "WebResourceLoadStatisticsTelemetry.h"
39 #include "WebsiteDataStore.h"
40 #include <WebCore/DocumentStorageAccess.h>
41 #include <WebCore/KeyedCoding.h>
42 #include <WebCore/NetworkStorageSession.h>
43 #include <WebCore/ResourceLoadStatistics.h>
44 #include <WebCore/SQLiteDatabase.h>
45 #include <WebCore/SQLiteStatement.h>
46 #include <WebCore/UserGestureIndicator.h>
47 #include <wtf/CallbackAggregator.h>
48 #include <wtf/DateMath.h>
49 #include <wtf/HashMap.h>
50 #include <wtf/MathExtras.h>
51 #include <wtf/StdSet.h>
52 #include <wtf/text/StringBuilder.h>
53
54 namespace WebKit {
55 using namespace WebCore;
56
57 #if PLATFORM(COCOA)
58 #define RELEASE_LOG_IF_ALLOWED(sessionID, fmt, ...) RELEASE_LOG_IF(sessionID.isAlwaysOnLoggingAllowed(), Network, "%p - ResourceLoadStatisticsDatabaseStore::" fmt, this, ##__VA_ARGS__)
59 #define RELEASE_LOG_ERROR_IF_ALLOWED(sessionID, fmt, ...) RELEASE_LOG_ERROR_IF(sessionID.isAlwaysOnLoggingAllowed(), Network, "%p - ResourceLoadStatisticsDatabaseStore::" fmt, this, ##__VA_ARGS__)
60 #else
61 #define RELEASE_LOG_IF_ALLOWED(sessionID, fmt, ...)  ((void)0)
62 #define RELEASE_LOG_ERROR_IF_ALLOWED(sessionID, fmt, ...)  ((void)0)
63 #endif
64
65 constexpr auto observedDomainCountQuery = "SELECT COUNT(*) FROM ObservedDomains"_s;
66 constexpr auto insertObservedDomainQuery = "INSERT INTO ObservedDomains (registrableDomain, lastSeen, hadUserInteraction,"
67     "mostRecentUserInteractionTime, grandfathered, isPrevalent, isVeryPrevalent, dataRecordsRemoved, timesAccessedAsFirstPartyDueToUserInteraction,"
68     "timesAccessedAsFirstPartyDueToStorageAccessAPI) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"_s;
69 constexpr auto insertTopLevelDomainQuery = "INSERT INTO TopLevelDomains VALUES (?)"_s;
70 constexpr auto domainIDFromStringQuery = "SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
71 constexpr auto storageAccessUnderTopFrameDomainsQuery = "INSERT INTO StorageAccessUnderTopFrameDomains (domainID, topLevelDomainID) "
72     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
73 constexpr auto topFrameUniqueRedirectsToQuery = "INSERT INTO TopFrameUniqueRedirectsTo (sourceDomainID, toDomainID) SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
74 constexpr auto topFrameUniqueRedirectsToExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameUniqueRedirectsTo WHERE sourceDomainID = ? "
75     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
76 constexpr auto topFrameUniqueRedirectsFromQuery = "INSERT INTO TopFrameUniqueRedirectsFrom (targetDomainID, fromDomainID) "
77     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
78 constexpr auto topFrameUniqueRedirectsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameUniqueRedirectsFrom WHERE targetDomainID = ? "
79     "AND fromDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
80 constexpr auto topFrameLinkDecorationsFromQuery = "INSERT INTO TopFrameLinkDecorationsFrom (fromDomainID, toDomainID) "
81     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
82 constexpr auto topFrameLinkDecorationsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameLinkDecorationsFrom WHERE fromDomainID = ? "
83     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
84 constexpr auto subframeUnderTopFrameDomainsQuery = "INSERT INTO SubframeUnderTopFrameDomains (subFrameDomainID, topFrameDomainID) "
85     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
86 constexpr auto subframeUnderTopFrameDomainExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubframeUnderTopFrameDomains WHERE subFrameDomainID = ? "
87     "AND topFrameDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
88 constexpr auto subresourceUnderTopFrameDomainsQuery = "INSERT INTO SubresourceUnderTopFrameDomains (subresourceDomainID, topFrameDomainID) "
89     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
90 constexpr auto subresourceUnderTopFrameDomainExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUnderTopFrameDomains "
91     "WHERE subresourceDomainID = ? AND topFrameDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
92 constexpr auto subresourceUniqueRedirectsToQuery = "INSERT INTO SubresourceUniqueRedirectsTo (subresourceDomainID, toDomainID) "
93     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
94 constexpr auto subresourceUniqueRedirectsToExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUniqueRedirectsTo WHERE subresourceDomainID = ? "
95     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
96 constexpr auto subresourceUniqueRedirectsFromQuery = "INSERT INTO SubresourceUniqueRedirectsFrom (subresourceDomainID, fromDomainID) "
97     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
98 constexpr auto subresourceUniqueRedirectsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUniqueRedirectsFrom WHERE subresourceDomainID = ? "
99     "AND fromDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
100 constexpr auto mostRecentUserInteractionQuery = "UPDATE ObservedDomains SET hadUserInteraction = ?, mostRecentUserInteractionTime = ? "
101     "WHERE registrableDomain = ?"_s;
102 constexpr auto updateLastSeenQuery = "UPDATE ObservedDomains SET lastSeen = ? WHERE registrableDomain = ?"_s;
103 constexpr auto updatePrevalentResourceQuery = "UPDATE ObservedDomains SET isPrevalent = ? WHERE registrableDomain = ?"_s;
104 constexpr auto isPrevalentResourceQuery = "SELECT isPrevalent FROM ObservedDomains WHERE registrableDomain = ?"_s;
105 constexpr auto updateVeryPrevalentResourceQuery = "UPDATE ObservedDomains SET isVeryPrevalent = ? WHERE registrableDomain = ?"_s;
106 constexpr auto isVeryPrevalentResourceQuery = "SELECT isVeryPrevalent FROM ObservedDomains WHERE registrableDomain = ?"_s;
107 constexpr auto clearPrevalentResourceQuery = "UPDATE ObservedDomains SET isPrevalent = 0, isVeryPrevalent = 0 WHERE registrableDomain = ?"_s;
108 constexpr auto hadUserInteractionQuery = "SELECT hadUserInteraction, mostRecentUserInteractionTime FROM ObservedDomains WHERE registrableDomain = ?"_s;
109 constexpr auto updateGrandfatheredQuery = "UPDATE ObservedDomains SET grandfathered = ? WHERE registrableDomain = ?"_s;
110 constexpr auto isGrandfatheredQuery = "SELECT grandfathered FROM ObservedDomains WHERE registrableDomain = ?"_s;
111 constexpr auto findExpiredUserInteractionQuery = "SELECT domainID FROM ObservedDomains WHERE hadUserInteraction = 1 AND mostRecentUserInteractionTime < ?"_s;
112
113 constexpr auto createObservedDomain = "CREATE TABLE ObservedDomains ("
114     "domainID INTEGER PRIMARY KEY, registrableDomain TEXT NOT NULL UNIQUE ON CONFLICT FAIL, lastSeen REAL NOT NULL, "
115     "hadUserInteraction INTEGER NOT NULL, mostRecentUserInteractionTime REAL NOT NULL, grandfathered INTEGER NOT NULL, "
116     "isPrevalent INTEGER NOT NULL, isVeryPrevalent INTEGER NOT NULL, dataRecordsRemoved INTEGER NOT NULL,"
117     "timesAccessedAsFirstPartyDueToUserInteraction INTEGER NOT NULL, timesAccessedAsFirstPartyDueToStorageAccessAPI INTEGER NOT NULL);"_s;
118     
119 constexpr auto createTopLevelDomains = "CREATE TABLE TopLevelDomains ("
120     "topLevelDomainID INTEGER PRIMARY KEY, CONSTRAINT fkDomainID FOREIGN KEY(topLevelDomainID) "
121     "REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
122     
123 constexpr auto createStorageAccessUnderTopFrameDomains = "CREATE TABLE StorageAccessUnderTopFrameDomains ("
124     "domainID INTEGER NOT NULL, topLevelDomainID INTEGER NOT NULL ON CONFLICT FAIL, "
125     "CONSTRAINT fkDomainID FOREIGN KEY(domainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
126     "FOREIGN KEY(topLevelDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
127     
128 constexpr auto createTopFrameUniqueRedirectsTo = "CREATE TABLE TopFrameUniqueRedirectsTo ("
129     "sourceDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
130     "FOREIGN KEY(sourceDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
131     "FOREIGN KEY(toDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
132     
133 constexpr auto createTopFrameUniqueRedirectsFrom = "CREATE TABLE TopFrameUniqueRedirectsFrom ("
134     "targetDomainID INTEGER NOT NULL, fromDomainID INTEGER NOT NULL, "
135     "FOREIGN KEY(targetDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
136     "FOREIGN KEY(fromDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
137
138 constexpr auto createTopFrameLinkDecorationsFrom = "CREATE TABLE TopFrameLinkDecorationsFrom ("
139     "fromDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
140     "FOREIGN KEY(fromDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
141     "FOREIGN KEY(toDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
142
143 constexpr auto createSubframeUnderTopFrameDomains = "CREATE TABLE SubframeUnderTopFrameDomains ("
144     "subFrameDomainID INTEGER NOT NULL, topFrameDomainID INTEGER NOT NULL, "
145     "FOREIGN KEY(subFrameDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
146     "FOREIGN KEY(topFrameDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
147     
148 constexpr auto createSubresourceUnderTopFrameDomains = "CREATE TABLE SubresourceUnderTopFrameDomains ("
149     "subresourceDomainID INTEGER NOT NULL, topFrameDomainID INTEGER NOT NULL, "
150     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
151     "FOREIGN KEY(topFrameDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
152     
153 constexpr auto createSubresourceUniqueRedirectsTo = "CREATE TABLE SubresourceUniqueRedirectsTo ("
154     "subresourceDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
155     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
156     "FOREIGN KEY(toDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
157     
158 constexpr auto createSubresourceUniqueRedirectsFrom = "CREATE TABLE SubresourceUniqueRedirectsFrom ("
159     "subresourceDomainID INTEGER NOT NULL, fromDomainID INTEGER NOT NULL, "
160     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
161     "FOREIGN KEY(fromDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
162     
163 ResourceLoadStatisticsDatabaseStore::ResourceLoadStatisticsDatabaseStore(WebResourceLoadStatisticsStore& store, WorkQueue& workQueue, ShouldIncludeLocalhost shouldIncludeLocalhost, const String& storageDirectoryPath)
164     : ResourceLoadStatisticsStore(store, workQueue, shouldIncludeLocalhost)
165     , m_storageDirectoryPath(storageDirectoryPath + "/observations.db")
166     , m_observedDomainCount(m_database, observedDomainCountQuery)
167     , m_insertObservedDomainStatement(m_database, insertObservedDomainQuery)
168     , m_insertTopLevelDomainStatement(m_database, insertTopLevelDomainQuery)
169     , m_domainIDFromStringStatement(m_database, domainIDFromStringQuery)
170     , m_storageAccessUnderTopFrameDomainsStatement(m_database, storageAccessUnderTopFrameDomainsQuery)
171     , m_topFrameUniqueRedirectsTo(m_database, topFrameUniqueRedirectsToQuery)
172     , m_topFrameUniqueRedirectsToExists(m_database, topFrameUniqueRedirectsToExistsQuery)
173     , m_topFrameUniqueRedirectsFrom(m_database, topFrameUniqueRedirectsFromQuery)
174     , m_topFrameUniqueRedirectsFromExists(m_database, topFrameUniqueRedirectsFromExistsQuery)
175     , m_topFrameLinkDecorationsFrom(m_database, topFrameLinkDecorationsFromQuery)
176     , m_topFrameLinkDecorationsFromExists(m_database, topFrameLinkDecorationsFromExistsQuery)
177     , m_subframeUnderTopFrameDomains(m_database, subframeUnderTopFrameDomainsQuery)
178     , m_subframeUnderTopFrameDomainExists(m_database, subframeUnderTopFrameDomainExistsQuery)
179     , m_subresourceUnderTopFrameDomains(m_database, subresourceUnderTopFrameDomainsQuery)
180     , m_subresourceUnderTopFrameDomainExists(m_database, subresourceUnderTopFrameDomainExistsQuery)
181     , m_subresourceUniqueRedirectsTo(m_database, subresourceUniqueRedirectsToQuery)
182     , m_subresourceUniqueRedirectsToExists(m_database, subresourceUniqueRedirectsToExistsQuery)
183     , m_subresourceUniqueRedirectsFrom(m_database, subresourceUniqueRedirectsFromQuery)
184     , m_subresourceUniqueRedirectsFromExists(m_database, subresourceUniqueRedirectsFromExistsQuery)
185     , m_mostRecentUserInteractionStatement(m_database, mostRecentUserInteractionQuery)
186     , m_updateLastSeenStatement(m_database, updateLastSeenQuery)
187     , m_updatePrevalentResourceStatement(m_database, updatePrevalentResourceQuery)
188     , m_isPrevalentResourceStatement(m_database, isPrevalentResourceQuery)
189     , m_updateVeryPrevalentResourceStatement(m_database, updateVeryPrevalentResourceQuery)
190     , m_isVeryPrevalentResourceStatement(m_database, isVeryPrevalentResourceQuery)
191     , m_clearPrevalentResourceStatement(m_database, clearPrevalentResourceQuery)
192     , m_hadUserInteractionStatement(m_database, hadUserInteractionQuery)
193     , m_updateGrandfatheredStatement(m_database, updateGrandfatheredQuery)
194     , m_isGrandfatheredStatement(m_database, isGrandfatheredQuery)
195     , m_findExpiredUserInteractionStatement(m_database, findExpiredUserInteractionQuery)
196 {
197     ASSERT(!RunLoop::isMain());
198
199     if (!m_database.open(m_storageDirectoryPath)) {
200         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::open failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
201         ASSERT_NOT_REACHED();
202         return;
203     }
204     
205     // Since we are using a workerQueue, the sequential dispatch blocks may be called by different threads.
206     m_database.disableThreadingChecks();
207
208     if (!m_database.tableExists("ObservedDomains"_s)) {
209         if (!createSchema()) {
210             RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::createSchema failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
211             ASSERT_NOT_REACHED();
212             return;
213         }
214     }
215
216     if (!prepareStatements()) {
217         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::prepareStatements failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
218         ASSERT_NOT_REACHED();
219         return;
220     }
221
222     workQueue.dispatchAfter(5_s, [weakThis = makeWeakPtr(*this)] {
223         if (weakThis)
224             weakThis->calculateAndSubmitTelemetry();
225     });
226 }
227
228 bool ResourceLoadStatisticsDatabaseStore::isEmpty() const
229 {
230     ASSERT(!RunLoop::isMain());
231     
232     bool result = false;
233     if (m_observedDomainCount.step() == SQLITE_ROW)
234         result = !m_observedDomainCount.getColumnInt(0);
235     
236     int resetResult = m_observedDomainCount.reset();
237     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
238     
239     return result;
240 }
241
242 bool ResourceLoadStatisticsDatabaseStore::createSchema()
243 {
244     ASSERT(!RunLoop::isMain());
245
246     if (!m_database.executeCommand(createObservedDomain)) {
247         LOG_ERROR("Could not create ObservedDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
248         return false;
249     }
250
251     if (!m_database.executeCommand(createTopLevelDomains)) {
252         LOG_ERROR("Could not create TopLevelDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
253         return false;
254     }
255
256     if (!m_database.executeCommand(createStorageAccessUnderTopFrameDomains)) {
257         LOG_ERROR("Could not create StorageAccessUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
258         return false;
259     }
260
261     if (!m_database.executeCommand(createTopFrameUniqueRedirectsTo)) {
262         LOG_ERROR("Could not create TopFrameUniqueRedirectsTo table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
263         return false;
264     }
265
266     if (!m_database.executeCommand(createTopFrameUniqueRedirectsFrom)) {
267         LOG_ERROR("Could not create TopFrameUniqueRedirectsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
268         return false;
269     }
270
271     if (!m_database.executeCommand(createTopFrameLinkDecorationsFrom)) {
272         LOG_ERROR("Could not create TopFrameLinkDecorationsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
273         return false;
274     }
275     
276     if (!m_database.executeCommand(createSubframeUnderTopFrameDomains)) {
277         LOG_ERROR("Could not create SubframeUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
278         return false;
279     }
280
281     if (!m_database.executeCommand(createSubresourceUnderTopFrameDomains)) {
282         LOG_ERROR("Could not create SubresourceUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
283         return false;
284     }
285
286     if (!m_database.executeCommand(createSubresourceUniqueRedirectsTo)) {
287         LOG_ERROR("Could not create SubresourceUniqueRedirectsTo table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
288         return false;
289     }
290
291     if (!m_database.executeCommand(createSubresourceUniqueRedirectsFrom)) {
292         LOG_ERROR("Could not create SubresourceUniqueRedirectsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
293         return false;
294     }
295
296     return true;
297 }
298
299 bool ResourceLoadStatisticsDatabaseStore::prepareStatements()
300 {
301     ASSERT(!RunLoop::isMain());
302         
303     if (m_observedDomainCount.prepare() != SQLITE_OK
304         || m_insertObservedDomainStatement.prepare() != SQLITE_OK
305         || m_insertTopLevelDomainStatement.prepare() != SQLITE_OK
306         || m_domainIDFromStringStatement.prepare() != SQLITE_OK
307         || m_storageAccessUnderTopFrameDomainsStatement.prepare() != SQLITE_OK
308         || m_topFrameUniqueRedirectsTo.prepare() != SQLITE_OK
309         || m_topFrameUniqueRedirectsToExists.prepare() != SQLITE_OK
310         || m_topFrameUniqueRedirectsFrom.prepare() != SQLITE_OK
311         || m_topFrameUniqueRedirectsFromExists.prepare() != SQLITE_OK
312         || m_subframeUnderTopFrameDomains.prepare() != SQLITE_OK
313         || m_subframeUnderTopFrameDomainExists.prepare() != SQLITE_OK
314         || m_subresourceUnderTopFrameDomains.prepare() != SQLITE_OK
315         || m_subresourceUnderTopFrameDomainExists.prepare() != SQLITE_OK
316         || m_subresourceUniqueRedirectsTo.prepare() != SQLITE_OK
317         || m_subresourceUniqueRedirectsToExists.prepare() != SQLITE_OK
318         || m_subresourceUniqueRedirectsFrom.prepare() != SQLITE_OK
319         || m_subresourceUniqueRedirectsFromExists.prepare() != SQLITE_OK
320         || m_updateLastSeenStatement.prepare() != SQLITE_OK
321         || m_mostRecentUserInteractionStatement.prepare() != SQLITE_OK
322         || m_updatePrevalentResourceStatement.prepare() != SQLITE_OK
323         || m_isPrevalentResourceStatement.prepare() != SQLITE_OK
324         || m_updateVeryPrevalentResourceStatement.prepare() != SQLITE_OK
325         || m_isVeryPrevalentResourceStatement.prepare() != SQLITE_OK
326         || m_clearPrevalentResourceStatement.prepare() != SQLITE_OK
327         || m_hadUserInteractionStatement.prepare() != SQLITE_OK
328         || m_updateGrandfatheredStatement.prepare() != SQLITE_OK
329         || m_isGrandfatheredStatement.prepare() != SQLITE_OK
330         || m_findExpiredUserInteractionStatement.prepare() != SQLITE_OK
331         ) {
332         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::prepareStatements failed to prepare, error message: %{public}s", this, m_database.lastErrorMsg());
333         ASSERT_NOT_REACHED();
334         return false;
335     }
336
337     return true;
338 }
339
340 bool ResourceLoadStatisticsDatabaseStore::insertObservedDomain(const ResourceLoadStatistics& loadStatistics)
341 {
342     ASSERT(!RunLoop::isMain());
343
344 #ifndef NDEBUG
345     ASSERT(confirmDomainDoesNotExist(loadStatistics.registrableDomain));
346 #endif
347
348     if (m_insertObservedDomainStatement.bindText(1, loadStatistics.registrableDomain.string()) != SQLITE_OK
349         || m_insertObservedDomainStatement.bindDouble(2, loadStatistics.lastSeen.secondsSinceEpoch().value()) != SQLITE_OK
350         || m_insertObservedDomainStatement.bindInt(3, loadStatistics.hadUserInteraction) != SQLITE_OK
351         || m_insertObservedDomainStatement.bindDouble(4, loadStatistics.mostRecentUserInteractionTime.secondsSinceEpoch().value()) != SQLITE_OK
352         || m_insertObservedDomainStatement.bindInt(5, loadStatistics.grandfathered) != SQLITE_OK
353         || m_insertObservedDomainStatement.bindInt(6, loadStatistics.isPrevalentResource) != SQLITE_OK
354         || m_insertObservedDomainStatement.bindInt(7, loadStatistics.isVeryPrevalentResource) != SQLITE_OK
355         || m_insertObservedDomainStatement.bindInt(8, loadStatistics.dataRecordsRemoved) != SQLITE_OK
356         || m_insertObservedDomainStatement.bindInt(9, loadStatistics.timesAccessedAsFirstPartyDueToUserInteraction) != SQLITE_OK
357         || m_insertObservedDomainStatement.bindInt(10, loadStatistics.timesAccessedAsFirstPartyDueToStorageAccessAPI) != SQLITE_OK) {
358         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::insertObservedDomain failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
359         ASSERT_NOT_REACHED();
360         return false;
361     }
362
363     if (m_insertObservedDomainStatement.step() != SQLITE_DONE) {
364         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::insertObservedDomain failed to commit, error message: %{public}s", this, m_database.lastErrorMsg());
365         ASSERT_NOT_REACHED();
366         return false;
367     }
368
369     int resetResult = m_insertObservedDomainStatement.reset();
370     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
371
372     return true;
373 }
374
375 bool ResourceLoadStatisticsDatabaseStore::relationshipExists(WebCore::SQLiteStatement& statement, unsigned firstDomainID, const RegistrableDomain& secondDomain) const
376 {
377     ASSERT(!RunLoop::isMain());
378
379     if (statement.bindInt(1, firstDomainID) != SQLITE_OK
380         || statement.bindText(2, secondDomain.string()) != SQLITE_OK
381         || statement.step() != SQLITE_ROW) {
382         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::m_insertDomainRelationshipStatement failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
383         ASSERT_NOT_REACHED();
384         return false;
385     }
386     
387     bool relationShipExists = !!statement.getColumnInt(0);
388
389     int resetResult = statement.reset();
390     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
391     
392     return relationShipExists;
393 }
394
395 bool ResourceLoadStatisticsDatabaseStore::insertDomainRelationship(WebCore::SQLiteStatement& statement, unsigned domainID, const RegistrableDomain& topFrame)
396 {
397     ASSERT(!RunLoop::isMain());
398
399     if (statement.bindInt(1, domainID) != SQLITE_OK
400         || statement.bindText(2, topFrame.string()) != SQLITE_OK
401         || statement.step() != SQLITE_DONE) {
402         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::m_insertDomainRelationshipStatement failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
403         ASSERT_NOT_REACHED();
404         return false;
405     }
406
407     int resetResult = statement.reset();
408     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
409     
410     return true;
411 }
412
413 #ifndef NDEBUG
414 bool ResourceLoadStatisticsDatabaseStore::confirmDomainDoesNotExist(const RegistrableDomain& domain) const
415 {
416     ASSERT(!RunLoop::isMain());
417
418     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK) {
419         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::confirmDomainDoesNotExist failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
420         ASSERT_NOT_REACHED();
421         return false;
422     }
423         
424     if (m_domainIDFromStringStatement.step() == SQLITE_ROW) {
425         int resetResult = m_domainIDFromStringStatement.reset();
426         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
427         return false;
428     }
429
430     int resetResult = m_domainIDFromStringStatement.reset();
431     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
432     
433     return true;
434 }
435 #endif
436
437 unsigned ResourceLoadStatisticsDatabaseStore::domainID(const RegistrableDomain& domain) const
438 {
439     ASSERT(!RunLoop::isMain());
440
441     unsigned domainID = 0;
442
443     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK
444         || m_domainIDFromStringStatement.step() != SQLITE_ROW) {
445         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::domainIDFromString failed, error message: %{public}s", this, m_database.lastErrorMsg());
446         ASSERT_NOT_REACHED();
447         return domainID;
448     }
449
450     domainID = m_domainIDFromStringStatement.getColumnInt(0);
451
452     int resetResult = m_domainIDFromStringStatement.reset();
453     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
454     
455     return domainID;
456 }
457
458 void ResourceLoadStatisticsDatabaseStore::insertDomainRelationships(const ResourceLoadStatistics& loadStatistics)
459 {
460     ASSERT(!RunLoop::isMain());
461
462     auto registrableDomainID = domainID(loadStatistics.registrableDomain);
463     for (auto& topFrameDomain : loadStatistics.storageAccessUnderTopFrameDomains)
464         insertDomainRelationship(m_storageAccessUnderTopFrameDomainsStatement, registrableDomainID, topFrameDomain);
465
466     for (auto& toDomain : loadStatistics.topFrameUniqueRedirectsTo)
467         insertDomainRelationship(m_topFrameUniqueRedirectsTo, registrableDomainID, toDomain);
468     
469     for (auto& fromDomain : loadStatistics.topFrameUniqueRedirectsFrom)
470         insertDomainRelationship(m_topFrameUniqueRedirectsFrom, registrableDomainID, fromDomain);
471     
472     for (auto& topFrameDomain : loadStatistics.subframeUnderTopFrameDomains)
473         insertDomainRelationship(m_subframeUnderTopFrameDomains, registrableDomainID, topFrameDomain);
474     
475     for (auto& topFrameDomain : loadStatistics.subresourceUnderTopFrameDomains)
476         insertDomainRelationship(m_subresourceUnderTopFrameDomains, registrableDomainID, topFrameDomain);
477     
478     for (auto& toDomain : loadStatistics.subresourceUniqueRedirectsTo)
479         insertDomainRelationship(m_subresourceUniqueRedirectsTo, registrableDomainID, toDomain);
480     
481     for (auto& fromDomain : loadStatistics.subresourceUniqueRedirectsFrom)
482         insertDomainRelationship(m_subresourceUniqueRedirectsFrom, registrableDomainID, fromDomain);
483 }
484
485 void ResourceLoadStatisticsDatabaseStore::populateFromMemoryStore(const ResourceLoadStatisticsMemoryStore& memoryStore)
486 {
487     ASSERT(!RunLoop::isMain());
488
489     if (!isEmpty())
490         return;
491
492     auto& statisticsMap = memoryStore.data();
493     for (const auto& statistic : statisticsMap)
494         insertObservedDomain(statistic.value);
495
496     // Make a separate pass for inter-domain relationships so we
497     // can refer to the ObservedDomain table entries
498     for (auto& statistic : statisticsMap)
499         insertDomainRelationships(statistic.value);
500
501     m_database.runVacuumCommand();
502 }
503
504 void ResourceLoadStatisticsDatabaseStore::calculateAndSubmitTelemetry() const
505 {
506     ASSERT(!RunLoop::isMain());
507
508     // FIXME(195088): Implement for Database version.
509 }
510
511 static String domainsToString(const HashSet<RegistrableDomain>& domains)
512 {
513     StringBuilder builder;
514     for (auto domainName : domains) {
515         if (!builder.isEmpty())
516             builder.appendLiteral(", ");
517         builder.append('"');
518         builder.append(domainName.string());
519         builder.append('"');
520     }
521
522     return builder.toString();
523 }
524
525 void ResourceLoadStatisticsDatabaseStore::incrementRecordsDeletedCountForDomains(HashSet<RegistrableDomain>&& domains)
526 {
527     ASSERT(!RunLoop::isMain());
528
529     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET dataRecordsRemoved = dataRecordsRemoved + 1 WHERE registrableDomain IN (", domainsToString(domains), ")"));
530     if (domainsToUpdateStatement.prepare() != SQLITE_OK
531         || domainsToUpdateStatement.step() != SQLITE_DONE) {
532         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::incrementStatisticsForDomains failed, error message: %{public}s", this, m_database.lastErrorMsg());
533         ASSERT_NOT_REACHED();
534     }
535 }
536
537 unsigned ResourceLoadStatisticsDatabaseStore::recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(unsigned primaryDomainID, StdSet<unsigned>& nonPrevalentRedirectionSources, unsigned numberOfRecursiveCalls)
538 {
539     ASSERT(!RunLoop::isMain());
540
541     if (numberOfRecursiveCalls >= maxNumberOfRecursiveCallsInRedirectTraceBack) {
542         RELEASE_LOG(ResourceLoadStatistics, "Hit %u recursive calls in redirect backtrace. Returning early.", maxNumberOfRecursiveCallsInRedirectTraceBack);
543         return numberOfRecursiveCalls;
544     }
545     
546     ++numberOfRecursiveCalls;
547
548     StdSet<unsigned> newlyIdentifiedDomains;
549     SQLiteStatement findSubresources(m_database, makeString("SELECT SubresourceUniqueRedirectsFrom.fromDomainID from SubresourceUniqueRedirectsFrom INNER JOIN ObservedDomains ON ObservedDomains.domainID = SubresourceUniqueRedirectsFrom.fromDomainID WHERE subresourceDomainID = ", String::number(primaryDomainID), "AND ObservedDomains.isPrevalent = 0"));
550     if (findSubresources.prepare() == SQLITE_OK) {
551         while (findSubresources.step() == SQLITE_ROW) {
552             auto newDomainID = findSubresources.getColumnInt(0);
553             auto insertResult = nonPrevalentRedirectionSources.insert(newDomainID);
554             if (insertResult.second)
555                 newlyIdentifiedDomains.insert(newDomainID);
556         }
557     }
558
559     SQLiteStatement findTopFrames(m_database, makeString("SELECT TopFrameUniqueRedirectsFrom.fromDomainID from TopFrameUniqueRedirectsFrom INNER JOIN ObservedDomains ON ObservedDomains.domainID = TopFrameUniqueRedirectsFrom.fromDomainID WHERE targetDomainID = ", String::number(primaryDomainID), "AND ObservedDomains.isPrevalent = 0"));
560     if (findTopFrames.prepare() == SQLITE_OK) {
561         while (findTopFrames.step() == SQLITE_ROW) {
562             auto newDomainID = findTopFrames.getColumnInt(0);
563             auto insertResult = nonPrevalentRedirectionSources.insert(newDomainID);
564             if (insertResult.second)
565                 newlyIdentifiedDomains.insert(newDomainID);
566         }
567     }
568
569     if (newlyIdentifiedDomains.empty())
570         return numberOfRecursiveCalls;
571
572     for (auto domainID : newlyIdentifiedDomains)
573         numberOfRecursiveCalls = recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(domainID, nonPrevalentRedirectionSources, numberOfRecursiveCalls);
574
575     return numberOfRecursiveCalls;
576 }
577
578 template <typename IteratorType>
579 static String buildList(const WTF::IteratorRange<IteratorType>& values)
580 {
581     StringBuilder builder;
582     for (auto domainID : values) {
583         if (!builder.isEmpty())
584             builder.appendLiteral(", ");
585         builder.appendNumber(domainID);
586     }
587     return builder.toString();
588 }
589
590 void ResourceLoadStatisticsDatabaseStore::markAsPrevalentIfHasRedirectedToPrevalent()
591 {
592     ASSERT(!RunLoop::isMain());
593
594     StdSet<unsigned> prevalentDueToRedirect;
595     SQLiteStatement subresourceRedirectStatement(m_database, "SELECT DISTINCT SubresourceUniqueRedirectsTo.subresourceDomainID FROM SubresourceUniqueRedirectsTo JOIN ObservedDomains ON ObservedDomains.domainID = SubresourceUniqueRedirectsTo.toDomainID AND ObservedDomains.isPrevalent = 1"_s);
596     if (subresourceRedirectStatement.prepare() == SQLITE_OK) {
597         while (subresourceRedirectStatement.step() == SQLITE_ROW)
598             prevalentDueToRedirect.insert(subresourceRedirectStatement.getColumnInt(0));
599     }
600
601     SQLiteStatement topFrameRedirectStatement(m_database, "SELECT DISTINCT TopFrameUniqueRedirectsTo.sourceDomainID FROM TopFrameUniqueRedirectsTo JOIN ObservedDomains ON ObservedDomains.domainID = TopFrameUniqueRedirectsTo.toDomainID AND ObservedDomains.isPrevalent = 1"_s);
602     if (topFrameRedirectStatement.prepare() == SQLITE_OK) {
603         while (topFrameRedirectStatement.step() == SQLITE_ROW)
604             prevalentDueToRedirect.insert(topFrameRedirectStatement.getColumnInt(0));
605     }
606
607     SQLiteStatement markPrevalentStatement(m_database, makeString("UPDATE ObservedDomains SET isPrevalent = 1 WHERE domainID IN (", buildList(WTF::IteratorRange<StdSet<unsigned>::iterator>(prevalentDueToRedirect.begin(), prevalentDueToRedirect.end())), ")"));
608     if (markPrevalentStatement.prepare() != SQLITE_OK
609         || markPrevalentStatement.step() != SQLITE_DONE) {
610         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::markAsPrevalentIfHasRedirectedToPrevalent failed to execute, error message: %{public}s", this, m_database.lastErrorMsg());
611         ASSERT_NOT_REACHED();
612     }
613 }
614
615 HashMap<unsigned, ResourceLoadStatisticsDatabaseStore::NotVeryPrevalentResources> ResourceLoadStatisticsDatabaseStore::findNotVeryPrevalentResources()
616 {
617     ASSERT(!RunLoop::isMain());
618
619     HashMap<unsigned, NotVeryPrevalentResources> results;
620     SQLiteStatement notVeryPrevalentResourcesStatement(m_database, "SELECT domainID, registrableDomain, isPrevalent FROM ObservedDomains WHERE isVeryPrevalent = 0"_s);
621     if (notVeryPrevalentResourcesStatement.prepare() == SQLITE_OK) {
622         while (notVeryPrevalentResourcesStatement.step() == SQLITE_ROW) {
623             unsigned key = static_cast<unsigned>(notVeryPrevalentResourcesStatement.getColumnInt(0));
624             NotVeryPrevalentResources value({ RegistrableDomain::uncheckedCreateFromRegistrableDomainString(notVeryPrevalentResourcesStatement.getColumnText(1))
625                 , notVeryPrevalentResourcesStatement.getColumnInt(2) ? ResourceLoadPrevalence::High : ResourceLoadPrevalence::Low
626                 , 0, 0, 0, 0 });
627             results.add(key, value);
628         }
629     }
630
631     StringBuilder builder;
632     for (auto value : results.keys()) {
633         if (!builder.isEmpty())
634             builder.appendLiteral(", ");
635         builder.appendNumber(value);
636     }
637
638     auto domainIDsOfInterest = builder.toString();
639
640     SQLiteStatement subresourceUnderTopFrameDomainsStatement(m_database, makeString("SELECT subresourceDomainID, COUNT(topFrameDomainID) FROM SubresourceUnderTopFrameDomains WHERE subresourceDomainID IN (", domainIDsOfInterest, ") GROUP BY subresourceDomainID"));
641     if (subresourceUnderTopFrameDomainsStatement.prepare() == SQLITE_OK) {
642         while (subresourceUnderTopFrameDomainsStatement.step() == SQLITE_ROW) {
643             unsigned domainID = static_cast<unsigned>(subresourceUnderTopFrameDomainsStatement.getColumnInt(0));
644             auto result = results.find(domainID);
645             if (result != results.end())
646                 result->value.subresourceUnderTopFrameDomainsCount = static_cast<unsigned>(subresourceUnderTopFrameDomainsStatement.getColumnInt(1));
647         }
648     }
649
650     SQLiteStatement subresourceUniqueRedirectsToCountStatement(m_database, makeString("SELECT subresourceDomainID, COUNT(toDomainID) FROM SubresourceUniqueRedirectsTo WHERE subresourceDomainID IN (", domainIDsOfInterest, ") GROUP BY subresourceDomainID"));
651     if (subresourceUniqueRedirectsToCountStatement.prepare() == SQLITE_OK) {
652         while (subresourceUniqueRedirectsToCountStatement.step() == SQLITE_ROW) {
653             unsigned domainID = static_cast<unsigned>(subresourceUniqueRedirectsToCountStatement.getColumnInt(0));
654             auto result = results.find(domainID);
655             if (result != results.end())
656                 result->value.subresourceUniqueRedirectsToCount = static_cast<unsigned>(subresourceUniqueRedirectsToCountStatement.getColumnInt(1));
657         }
658     }
659
660     SQLiteStatement subframeUnderTopFrameDomainsCountStatement(m_database, makeString("SELECT subresourceDomainID, COUNT(topFrameDomainID) FROM SubresourceUnderTopFrameDomains WHERE subresourceDomainID IN (", domainIDsOfInterest, ") GROUP BY subresourceDomainID"));
661     if (subframeUnderTopFrameDomainsCountStatement.prepare() == SQLITE_OK) {
662         while (subframeUnderTopFrameDomainsCountStatement.step() == SQLITE_ROW) {
663             unsigned domainID = static_cast<unsigned>(subframeUnderTopFrameDomainsCountStatement.getColumnInt(0));
664             auto result = results.find(domainID);
665             if (result != results.end())
666                 result->value.subframeUnderTopFrameDomainsCount = static_cast<unsigned>(subframeUnderTopFrameDomainsCountStatement.getColumnInt(1));
667         }
668     }
669
670     SQLiteStatement topFrameUniqueRedirectsToCountStatement(m_database, makeString("SELECT sourceDomainID, COUNT(toDomainID) FROM TopFrameUniqueRedirectsTo WHERE sourceDomainID IN (", domainIDsOfInterest, ") GROUP BY sourceDomainID"));
671     if (topFrameUniqueRedirectsToCountStatement.prepare() == SQLITE_OK) {
672         while (topFrameUniqueRedirectsToCountStatement.step() == SQLITE_ROW) {
673             unsigned domainID = static_cast<unsigned>(topFrameUniqueRedirectsToCountStatement.getColumnInt(0));
674             auto result = results.find(domainID);
675             if (result != results.end())
676                 result->value.topFrameUniqueRedirectsToCount = static_cast<unsigned>(topFrameUniqueRedirectsToCountStatement.getColumnInt(1));
677         }
678     }
679
680     return results;
681 }
682
683 void ResourceLoadStatisticsDatabaseStore::reclassifyResources()
684 {
685     ASSERT(!RunLoop::isMain());
686
687     auto notVeryPrevalentResources = findNotVeryPrevalentResources();
688
689     for (auto& resourceStatistic : notVeryPrevalentResources.values()) {
690         if (shouldSkip(resourceStatistic.registerableDomain))
691             continue;
692
693         auto newPrevalence = classifier().calculateResourcePrevalence(resourceStatistic.subresourceUnderTopFrameDomainsCount, resourceStatistic.subresourceUniqueRedirectsToCount, resourceStatistic.subframeUnderTopFrameDomainsCount, resourceStatistic.topFrameUniqueRedirectsToCount, resourceStatistic.prevalence);
694         if (newPrevalence != resourceStatistic.prevalence)
695             setPrevalentResource(resourceStatistic.registerableDomain, newPrevalence);
696     }
697 }
698
699 void ResourceLoadStatisticsDatabaseStore::classifyPrevalentResources()
700 {
701     ASSERT(!RunLoop::isMain());
702     ensurePrevalentResourcesForDebugMode();
703     markAsPrevalentIfHasRedirectedToPrevalent();
704     reclassifyResources();
705 }
706
707 void ResourceLoadStatisticsDatabaseStore::syncStorageIfNeeded()
708 {
709     ASSERT(!RunLoop::isMain());
710     m_database.runVacuumCommand();
711 }
712
713 void ResourceLoadStatisticsDatabaseStore::syncStorageImmediately()
714 {
715     ASSERT(!RunLoop::isMain());
716     m_database.runVacuumCommand();
717 }
718
719 void ResourceLoadStatisticsDatabaseStore::hasStorageAccess(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain, Optional<FrameID> frameID, PageIdentifier pageID, CompletionHandler<void(bool)>&& completionHandler)
720 {
721     ASSERT(!RunLoop::isMain());
722
723     ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
724
725     switch (cookieTreatmentForOrigin(subFrameDomain)) {
726     case CookieTreatmentResult::BlockAndPurge:
727         completionHandler(false);
728         return;
729     case CookieTreatmentResult::BlockAndKeep:
730         completionHandler(true);
731         return;
732     case CookieTreatmentResult::Allow:
733         // Do nothing
734         break;
735     };
736
737     RunLoop::main().dispatch([store = makeRef(store()), subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, completionHandler = WTFMove(completionHandler)]() mutable {
738         store->callHasStorageAccessForFrameHandler(subFrameDomain, topFrameDomain, frameID.value(), pageID, [store = store.copyRef(), completionHandler = WTFMove(completionHandler)](bool result) mutable {
739             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler), result] () mutable {
740                 completionHandler(result);
741             });
742         });
743     });
744 }
745
746 void ResourceLoadStatisticsDatabaseStore::requestStorageAccess(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, FrameID frameID, PageIdentifier pageID, CompletionHandler<void(StorageAccessStatus)>&& completionHandler)
747 {
748     ASSERT(!RunLoop::isMain());
749
750     auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
751
752     switch (cookieTreatmentForOrigin(subFrameDomain)) {
753     case CookieTreatmentResult::BlockAndPurge: {
754         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "Cannot grant storage access to %{public}s since its cookies are blocked in third-party contexts and it has not received user interaction as first-party.", subFrameDomain.string().utf8().data());
755         completionHandler(StorageAccessStatus::CannotRequestAccess);
756         }
757         return;
758     case CookieTreatmentResult::BlockAndKeep: {
759         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "No need to grant storage access to %{public}s since its cookies are not blocked in third-party contexts.", subFrameDomain.string().utf8().data());
760         completionHandler(StorageAccessStatus::HasAccess);
761         }
762         return;
763     case CookieTreatmentResult::Allow:
764         // Do nothing
765         break;
766     };
767
768     auto userWasPromptedEarlier = hasUserGrantedStorageAccessThroughPrompt(subFrameStatus.second, topFrameDomain);
769     if (userWasPromptedEarlier == StorageAccessPromptWasShown::No) {
770         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "About to ask the user whether they want to grant storage access to %{public}s under %{public}s or not.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
771         completionHandler(StorageAccessStatus::RequiresUserPrompt);
772         return;
773     }
774
775     if (userWasPromptedEarlier == StorageAccessPromptWasShown::Yes)
776         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "Storage access was granted to %{public}s under %{public}s.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
777
778     SQLiteStatement incrementStorageAccess(m_database, makeString("UPDATE ObservedDomains SET timesAccessedAsFirstPartyDueToStorageAccessAPI = timesAccessedAsFirstPartyDueToStorageAccessAPI + 1 WHERE domainID = ", String::number(subFrameStatus.second)));
779     if (incrementStorageAccess.prepare() != SQLITE_OK
780         || incrementStorageAccess.step() != SQLITE_DONE) {
781         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::requestStorageAccess failed, error message: %{public}s", this, m_database.lastErrorMsg());
782         ASSERT_NOT_REACHED();
783     }
784     
785     grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, userWasPromptedEarlier, [completionHandler = WTFMove(completionHandler)] (StorageAccessWasGranted wasGranted) mutable {
786         completionHandler(wasGranted == StorageAccessWasGranted::Yes ? StorageAccessStatus::HasAccess : StorageAccessStatus::CannotRequestAccess);
787     });
788 }
789
790 void ResourceLoadStatisticsDatabaseStore::requestStorageAccessUnderOpener(DomainInNeedOfStorageAccess&& domainInNeedOfStorageAccess, PageIdentifier openerPageID, OpenerDomain&& openerDomain)
791 {
792     ASSERT(domainInNeedOfStorageAccess != openerDomain);
793     ASSERT(!RunLoop::isMain());
794
795     if (domainInNeedOfStorageAccess == openerDomain)
796         return;
797
798     ensureResourceStatisticsForRegistrableDomain(domainInNeedOfStorageAccess);
799
800     if (cookieTreatmentForOrigin(domainInNeedOfStorageAccess) != CookieTreatmentResult::Allow)
801         return;
802
803     RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "[Temporary combatibility fix] Storage access was granted for %{public}s under opener page from %{public}s, with user interaction in the opened window.", domainInNeedOfStorageAccess.string().utf8().data(), openerDomain.string().utf8().data());
804     grantStorageAccessInternal(WTFMove(domainInNeedOfStorageAccess), WTFMove(openerDomain), WTF::nullopt, openerPageID, StorageAccessPromptWasShown::No, [](StorageAccessWasGranted) { });
805 }
806
807 void ResourceLoadStatisticsDatabaseStore::grantStorageAccess(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, uint64_t frameID, PageIdentifier pageID, StorageAccessPromptWasShown promptWasShown, CompletionHandler<void(StorageAccessWasGranted)>&& completionHandler)
808 {
809     ASSERT(!RunLoop::isMain());
810
811     if (promptWasShown == StorageAccessPromptWasShown::Yes) {
812         auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
813         ASSERT(subFrameStatus.first == AddedRecord::No);
814         ASSERT(hasHadUserInteraction(subFrameDomain, OperatingDatesWindow::Long));
815         insertDomainRelationship(m_storageAccessUnderTopFrameDomainsStatement, subFrameStatus.second, topFrameDomain);
816     }
817
818     grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, promptWasShown, WTFMove(completionHandler));
819 }
820
821 void ResourceLoadStatisticsDatabaseStore::grantStorageAccessInternal(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, Optional<FrameID> frameID, PageIdentifier pageID, StorageAccessPromptWasShown promptWasShownNowOrEarlier, CompletionHandler<void(StorageAccessWasGranted)>&& completionHandler)
822 {
823     ASSERT(!RunLoop::isMain());
824
825     if (subFrameDomain == topFrameDomain) {
826         completionHandler(StorageAccessWasGranted::Yes);
827         return;
828     }
829
830     if (promptWasShownNowOrEarlier == StorageAccessPromptWasShown::Yes) {
831 #ifndef NDEBUG
832         auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
833         ASSERT(subFrameStatus.first == AddedRecord::No);
834         ASSERT(hasHadUserInteraction(subFrameDomain, OperatingDatesWindow::Long));
835         ASSERT(hasUserGrantedStorageAccessThroughPrompt(subFrameStatus.second, topFrameDomain) == StorageAccessPromptWasShown::Yes);
836 #endif
837         setUserInteraction(subFrameDomain, true, WallTime::now());
838     }
839
840     RunLoop::main().dispatch([subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, store = makeRef(store()), completionHandler = WTFMove(completionHandler)]() mutable {
841         store->callGrantStorageAccessHandler(subFrameDomain, topFrameDomain, frameID, pageID, [completionHandler = WTFMove(completionHandler), store = store.copyRef()](StorageAccessWasGranted wasGranted) mutable {
842             store->statisticsQueue().dispatch([wasGranted, completionHandler = WTFMove(completionHandler)] () mutable {
843                 completionHandler(wasGranted);
844             });
845         });
846     });
847
848 }
849
850 void ResourceLoadStatisticsDatabaseStore::grandfatherDataForDomains(const HashSet<RegistrableDomain>& domains)
851 {
852     ASSERT(!RunLoop::isMain());
853
854     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET grandfathered = 1 WHERE registrableDomain IN (", domainsToString(domains), ")"));
855     if (domainsToUpdateStatement.prepare() != SQLITE_OK
856         || domainsToUpdateStatement.step() != SQLITE_DONE) {
857         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::grandfatherDataForDomains failed, error message: %{public}s", this, m_database.lastErrorMsg());
858         ASSERT_NOT_REACHED();
859     }
860 }
861
862 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::ensurePrevalentResourcesForDebugMode()
863 {
864     ASSERT(!RunLoop::isMain());
865
866     if (!debugModeEnabled())
867         return { };
868
869     Vector<RegistrableDomain> primaryDomainsToBlock;
870     primaryDomainsToBlock.reserveInitialCapacity(2);
871
872     ensureResourceStatisticsForRegistrableDomain(debugStaticPrevalentResource());
873     setPrevalentResource(debugStaticPrevalentResource(), ResourceLoadPrevalence::High);
874     primaryDomainsToBlock.uncheckedAppend(debugStaticPrevalentResource());
875
876     if (!debugManualPrevalentResource().isEmpty()) {
877         ensureResourceStatisticsForRegistrableDomain(debugManualPrevalentResource());
878         setPrevalentResource(debugManualPrevalentResource(), ResourceLoadPrevalence::High);
879         primaryDomainsToBlock.uncheckedAppend(debugManualPrevalentResource());
880         RELEASE_LOG_INFO(ITPDebug, "Did set %{public}s as prevalent resource for the purposes of ITP Debug Mode.", debugManualPrevalentResource().string().utf8().data());
881     }
882
883     return primaryDomainsToBlock;
884 }
885
886 void ResourceLoadStatisticsDatabaseStore::logFrameNavigation(const RegistrableDomain& targetDomain, const RegistrableDomain& topFrameDomain, const RegistrableDomain& sourceDomain, bool isRedirect, bool isMainFrame)
887 {
888     ASSERT(!RunLoop::isMain());
889
890     bool areTargetAndTopFrameDomainsSameSite = targetDomain == topFrameDomain;
891     bool areTargetAndSourceDomainsSameSite = targetDomain == sourceDomain;
892
893     bool statisticsWereUpdated = false;
894     if (!isMainFrame && !(areTargetAndTopFrameDomainsSameSite || areTargetAndSourceDomainsSameSite)) {
895         auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
896         updateLastSeen(targetDomain, ResourceLoadStatistics::reduceTimeResolution(WallTime::now()));
897
898         if (!relationshipExists(m_subframeUnderTopFrameDomainExists, targetResult.second, topFrameDomain)) {
899             insertDomainRelationship(m_subframeUnderTopFrameDomains, targetResult.second, topFrameDomain);
900             statisticsWereUpdated = true;
901         }
902     }
903
904     if (isRedirect && !areTargetAndSourceDomainsSameSite) {
905         if (isMainFrame) {
906             auto redirectingDomainResult = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
907             if (!relationshipExists(m_topFrameUniqueRedirectsToExists, redirectingDomainResult.second, targetDomain)) {
908                 insertDomainRelationship(m_topFrameUniqueRedirectsTo, redirectingDomainResult.second, targetDomain);
909                 statisticsWereUpdated = true;
910             }
911
912             auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
913             if (!relationshipExists(m_topFrameUniqueRedirectsFromExists, targetResult.second, sourceDomain)) {
914                 insertDomainRelationship(m_topFrameUniqueRedirectsFrom, targetResult.second, sourceDomain);
915                 statisticsWereUpdated = true;
916             }
917         } else {
918             auto redirectingDomainResult = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
919             if (!relationshipExists(m_subresourceUniqueRedirectsToExists, redirectingDomainResult.second, targetDomain)) {
920                 insertDomainRelationship(m_subresourceUniqueRedirectsTo, redirectingDomainResult.second, targetDomain);
921                 statisticsWereUpdated = true;
922             }
923
924             auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
925             if (!relationshipExists(m_subresourceUniqueRedirectsFromExists, targetResult.second, sourceDomain)) {
926                 insertDomainRelationship(m_subresourceUniqueRedirectsFrom, targetResult.second, sourceDomain);
927                 statisticsWereUpdated = true;
928             }
929         }
930     }
931
932     if (statisticsWereUpdated)
933         scheduleStatisticsProcessingRequestIfNecessary();
934 }
935
936 void ResourceLoadStatisticsDatabaseStore::logSubresourceLoading(const SubResourceDomain& targetDomain, const TopFrameDomain& topFrameDomain, WallTime lastSeen)
937 {
938     ASSERT(!RunLoop::isMain());
939
940     auto result = ensureResourceStatisticsForRegistrableDomain(targetDomain);
941     updateLastSeen(targetDomain, lastSeen);
942
943     auto targetDomainID = result.second;
944     if (!relationshipExists(m_subresourceUnderTopFrameDomainExists, targetDomainID, topFrameDomain)) {
945         insertDomainRelationship(m_subresourceUnderTopFrameDomains, targetDomainID, topFrameDomain);
946         scheduleStatisticsProcessingRequestIfNecessary();
947     }
948 }
949
950 void ResourceLoadStatisticsDatabaseStore::logSubresourceRedirect(const RedirectedFromDomain& sourceDomain, const RedirectedToDomain& targetDomain)
951 {
952     ASSERT(!RunLoop::isMain());
953
954     auto sourceDomainResult = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
955     auto targetDomainResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
956
957     bool isNewRedirectToEntry = false;
958     if (!relationshipExists(m_subresourceUniqueRedirectsToExists, sourceDomainResult.second, targetDomain)) {
959         insertDomainRelationship(m_subresourceUniqueRedirectsTo, sourceDomainResult.second, targetDomain);
960         isNewRedirectToEntry = true;
961     }
962
963     bool isNewRedirectFromEntry = false;
964     if (!relationshipExists(m_subresourceUniqueRedirectsFromExists, targetDomainResult.second, sourceDomain)) {
965         insertDomainRelationship(m_subresourceUniqueRedirectsFrom, targetDomainResult.second, sourceDomain);
966         isNewRedirectFromEntry = true;
967     }
968
969     if (isNewRedirectToEntry || isNewRedirectFromEntry)
970         scheduleStatisticsProcessingRequestIfNecessary();
971 }
972
973 void ResourceLoadStatisticsDatabaseStore::logCrossSiteLoadWithLinkDecoration(const NavigatedFromDomain& fromDomain, const NavigatedToDomain& toDomain)
974 {
975     ASSERT(!RunLoop::isMain());
976     ASSERT(fromDomain != toDomain);
977
978     auto fromDomainResult = ensureResourceStatisticsForRegistrableDomain(fromDomain);
979
980     if (!relationshipExists(m_topFrameLinkDecorationsFromExists, fromDomainResult.second, toDomain)) {
981         insertDomainRelationship(m_topFrameLinkDecorationsFrom, fromDomainResult.second, toDomain);
982         scheduleStatisticsProcessingRequestIfNecessary();
983     }
984 }
985
986 void ResourceLoadStatisticsDatabaseStore::setUserInteraction(const RegistrableDomain& domain, bool hadUserInteraction, WallTime mostRecentInteraction)
987 {
988     ASSERT(!RunLoop::isMain());
989
990     if (m_mostRecentUserInteractionStatement.bindInt(1, hadUserInteraction)
991         || m_mostRecentUserInteractionStatement.bindDouble(2, mostRecentInteraction.secondsSinceEpoch().value()) != SQLITE_OK
992         || m_mostRecentUserInteractionStatement.bindText(3, domain.string()) != SQLITE_OK
993         || m_mostRecentUserInteractionStatement.step() != SQLITE_DONE) {
994         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::setUserInteraction, error message: %{public}s", this, m_database.lastErrorMsg());
995         ASSERT_NOT_REACHED();
996         return;
997     }
998
999     int resetResult = m_mostRecentUserInteractionStatement.reset();
1000     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1001 }
1002
1003 void ResourceLoadStatisticsDatabaseStore::logUserInteraction(const TopFrameDomain& domain)
1004 {
1005     ASSERT(!RunLoop::isMain());
1006
1007     ensureResourceStatisticsForRegistrableDomain(domain);
1008     setUserInteraction(domain, true, WallTime::now());
1009 }
1010
1011 void ResourceLoadStatisticsDatabaseStore::clearUserInteraction(const RegistrableDomain& domain)
1012 {
1013     ASSERT(!RunLoop::isMain());
1014
1015     auto targetResult = ensureResourceStatisticsForRegistrableDomain(domain);
1016     setUserInteraction(domain, false, { });
1017
1018     SQLiteStatement removeStorageAccess(m_database, makeString("DELETE FROM StorageAccessUnderTopFrameDomains WHERE domainID = ", String::number(targetResult.second)));
1019     if (removeStorageAccess.prepare() != SQLITE_OK
1020         || removeStorageAccess.step() != SQLITE_DONE) {
1021         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::logUserInteraction failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1022         ASSERT_NOT_REACHED();
1023     }
1024 }
1025
1026 bool ResourceLoadStatisticsDatabaseStore::hasHadUserInteraction(const RegistrableDomain& domain, OperatingDatesWindow operatingDatesWindow)
1027 {
1028     ASSERT(!RunLoop::isMain());
1029
1030     if (m_hadUserInteractionStatement.bindText(1, domain.string()) != SQLITE_OK
1031         || m_hadUserInteractionStatement.step() != SQLITE_DONE) {
1032         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::m_updatePrevalentResourceStatement failed, error message: %{public}s", this, m_database.lastErrorMsg());
1033         ASSERT_NOT_REACHED();
1034         return false;
1035     }
1036
1037     bool hadUserInteraction = !!m_hadUserInteractionStatement.getColumnInt(0);
1038     if (!hadUserInteraction)
1039         return false;
1040     
1041     WallTime mostRecentUserInteractionTime = WallTime::fromRawSeconds(m_hadUserInteractionStatement.getColumnDouble(1));
1042
1043     if (hasStatisticsExpired(mostRecentUserInteractionTime, operatingDatesWindow)) {
1044         // Drop privacy sensitive data because we no longer need it.
1045         // Set timestamp to 0 so that statistics merge will know
1046         // it has been reset as opposed to its default -1.
1047         clearUserInteraction(domain);
1048         hadUserInteraction = false;
1049     }
1050     
1051     return hadUserInteraction;
1052 }
1053
1054 void ResourceLoadStatisticsDatabaseStore::setPrevalentResource(const RegistrableDomain& domain, ResourceLoadPrevalence newPrevalence)
1055 {
1056     ASSERT(!RunLoop::isMain());
1057     if (shouldSkip(domain))
1058         return;
1059
1060     if (m_updatePrevalentResourceStatement.bindInt(1, 1) != SQLITE_OK
1061         || m_updatePrevalentResourceStatement.bindText(2, domain.string()) != SQLITE_OK
1062         || m_updatePrevalentResourceStatement.step() != SQLITE_DONE) {
1063         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::m_updatePrevalentResourceStatement failed, error message: %{public}s", this, m_database.lastErrorMsg());
1064         ASSERT_NOT_REACHED();
1065         return;
1066     }
1067     
1068     int resetResult = m_updatePrevalentResourceStatement.reset();
1069     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1070
1071     if (newPrevalence == ResourceLoadPrevalence::VeryHigh) {
1072         if (m_updateVeryPrevalentResourceStatement.bindInt(1, 1) != SQLITE_OK
1073             || m_updateVeryPrevalentResourceStatement.bindText(2, domain.string()) != SQLITE_OK
1074             || m_updateVeryPrevalentResourceStatement.step() != SQLITE_DONE) {
1075             RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::m_updateVeryPrevalentResourceStatement failed, error message: %{public}s", this, m_database.lastErrorMsg());
1076             ASSERT_NOT_REACHED();
1077             return;
1078         }
1079
1080         int resetResult = m_updateVeryPrevalentResourceStatement.reset();
1081         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1082     }
1083
1084     StdSet<unsigned> nonPrevalentRedirectionSources;
1085     recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(domainID(domain), nonPrevalentRedirectionSources, 0);
1086     setDomainsAsPrevalent(WTFMove(nonPrevalentRedirectionSources));
1087 }
1088
1089 void ResourceLoadStatisticsDatabaseStore::setDomainsAsPrevalent(StdSet<unsigned>&& domains)
1090 {
1091     ASSERT(!RunLoop::isMain());
1092
1093     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET isPrevalent = 1 WHERE domainID IN (", buildList(WTF::IteratorRange<StdSet<unsigned>::iterator>(domains.begin(), domains.end())), ")"));
1094     if (domainsToUpdateStatement.prepare() != SQLITE_OK
1095         || domainsToUpdateStatement.step() != SQLITE_DONE) {
1096         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::setDomainsAsPrevalent failed, error message: %{public}s", this, m_database.lastErrorMsg());
1097         ASSERT_NOT_REACHED();
1098         return;
1099     }
1100 }
1101
1102 void ResourceLoadStatisticsDatabaseStore::dumpResourceLoadStatistics(CompletionHandler<void(const String&)>&& completionHandler)
1103 {
1104     ASSERT(!RunLoop::isMain());
1105
1106     // FIXME(195088): Implement SQLite-based dumping routines.
1107     ASSERT_NOT_REACHED();
1108     completionHandler({ });
1109 }
1110
1111 bool ResourceLoadStatisticsDatabaseStore::predicateValueForDomain(WebCore::SQLiteStatement& predicateStatement, const RegistrableDomain& domain) const
1112 {
1113     ASSERT(!RunLoop::isMain());
1114     
1115     if (predicateStatement.bindText(1, domain.string()) != SQLITE_OK
1116         || predicateStatement.step() != SQLITE_DONE) {
1117         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::predicateValueForDomain failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1118         ASSERT_NOT_REACHED();
1119         return false;
1120     }
1121
1122     bool result = !!predicateStatement.getColumnInt(0);
1123     int resetResult = predicateStatement.reset();
1124     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1125     return result;
1126 }
1127
1128 bool ResourceLoadStatisticsDatabaseStore::isPrevalentResource(const RegistrableDomain& domain) const
1129 {
1130     ASSERT(!RunLoop::isMain());
1131
1132     if (shouldSkip(domain))
1133         return false;
1134
1135     return predicateValueForDomain(m_isPrevalentResourceStatement, domain);
1136 }
1137
1138 bool ResourceLoadStatisticsDatabaseStore::isVeryPrevalentResource(const RegistrableDomain& domain) const
1139 {
1140     ASSERT(!RunLoop::isMain());
1141
1142     if (shouldSkip(domain))
1143         return false;
1144
1145     return predicateValueForDomain(m_isVeryPrevalentResourceStatement, domain);
1146 }
1147
1148 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsSubresourceUnder(const SubResourceDomain& subresourceDomain, const TopFrameDomain& topFrameDomain) const
1149 {
1150     ASSERT(!RunLoop::isMain());
1151
1152     return relationshipExists(m_subresourceUnderTopFrameDomainExists, domainID(subresourceDomain), topFrameDomain);
1153 }
1154
1155 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsSubFrameUnder(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain) const
1156 {
1157     ASSERT(!RunLoop::isMain());
1158
1159     return relationshipExists(m_subframeUnderTopFrameDomainExists, domainID(subFrameDomain), topFrameDomain);
1160 }
1161
1162 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsRedirectingTo(const RedirectedFromDomain& redirectedFromDomain, const RedirectedToDomain& redirectedToDomain) const
1163 {
1164     ASSERT(!RunLoop::isMain());
1165
1166     return relationshipExists(m_subresourceUniqueRedirectsToExists, domainID(redirectedFromDomain), redirectedToDomain);
1167 }
1168
1169 void ResourceLoadStatisticsDatabaseStore::clearPrevalentResource(const RegistrableDomain& domain)
1170 {
1171     ASSERT(!RunLoop::isMain());
1172
1173     ensureResourceStatisticsForRegistrableDomain(domain);
1174     
1175     if (m_clearPrevalentResourceStatement.bindText(1, domain.string()) != SQLITE_OK
1176         || m_clearPrevalentResourceStatement.step() != SQLITE_DONE) {
1177         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::clearPrevalentResource, error message: %{public}s", this, m_database.lastErrorMsg());
1178         ASSERT_NOT_REACHED();
1179         return;
1180     }
1181     
1182     int resetResult = m_clearPrevalentResourceStatement.reset();
1183     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1184 }
1185
1186 void ResourceLoadStatisticsDatabaseStore::setGrandfathered(const RegistrableDomain& domain, bool value)
1187 {
1188     ASSERT(!RunLoop::isMain());
1189
1190     ensureResourceStatisticsForRegistrableDomain(domain);
1191     
1192     if (m_updateGrandfatheredStatement.bindInt(1, value) != SQLITE_OK
1193         || m_updateGrandfatheredStatement.bindText(1, domain.string()) != SQLITE_OK
1194         || m_updateGrandfatheredStatement.step() != SQLITE_DONE) {
1195         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::setGrandfathered failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1196         ASSERT_NOT_REACHED();
1197         return;
1198     }
1199     
1200     int resetResult = m_updateGrandfatheredStatement.reset();
1201     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1202 }
1203
1204 bool ResourceLoadStatisticsDatabaseStore::isGrandfathered(const RegistrableDomain& domain) const
1205 {
1206     ASSERT(!RunLoop::isMain());
1207
1208     return predicateValueForDomain(m_isGrandfatheredStatement, domain);
1209 }
1210
1211 void ResourceLoadStatisticsDatabaseStore::setSubframeUnderTopFrameDomain(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain)
1212 {
1213     ASSERT(!RunLoop::isMain());
1214
1215     auto result = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
1216
1217     // For consistency, make sure we also have a statistics entry for the top frame domain.
1218     ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1219
1220     insertDomainRelationship(m_subframeUnderTopFrameDomains, result.second, topFrameDomain);
1221 }
1222
1223 void ResourceLoadStatisticsDatabaseStore::setSubresourceUnderTopFrameDomain(const SubResourceDomain& subresourceDomain, const TopFrameDomain& topFrameDomain)
1224 {
1225     ASSERT(!RunLoop::isMain());
1226
1227     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1228
1229     // For consistency, make sure we also have a statistics entry for the top frame domain.
1230     ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1231
1232     insertDomainRelationship(m_subresourceUnderTopFrameDomains, result.second, topFrameDomain);
1233 }
1234
1235 void ResourceLoadStatisticsDatabaseStore::setSubresourceUniqueRedirectTo(const SubResourceDomain& subresourceDomain, const RedirectDomain& redirectDomain)
1236 {
1237     ASSERT(!RunLoop::isMain());
1238
1239     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1240
1241     // For consistency, make sure we also have a statistics entry for the redirect domain.
1242     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1243
1244     insertDomainRelationship(m_subresourceUniqueRedirectsTo, result.second, redirectDomain);
1245 }
1246
1247 void ResourceLoadStatisticsDatabaseStore::setSubresourceUniqueRedirectFrom(const SubResourceDomain& subresourceDomain, const RedirectDomain& redirectDomain)
1248 {
1249     ASSERT(!RunLoop::isMain());
1250
1251     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1252
1253     // For consistency, make sure we also have a statistics entry for the redirect domain.
1254     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1255
1256     insertDomainRelationship(m_subresourceUniqueRedirectsFrom, result.second, redirectDomain);
1257 }
1258
1259 void ResourceLoadStatisticsDatabaseStore::setTopFrameUniqueRedirectTo(const TopFrameDomain& topFrameDomain, const RedirectDomain& redirectDomain)
1260 {
1261     ASSERT(!RunLoop::isMain());
1262
1263     auto result = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1264
1265     // For consistency, make sure we also have a statistics entry for the redirect domain.
1266     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1267
1268     insertDomainRelationship(m_topFrameUniqueRedirectsTo, result.second, redirectDomain);
1269 }
1270
1271 void ResourceLoadStatisticsDatabaseStore::setTopFrameUniqueRedirectFrom(const TopFrameDomain& topFrameDomain, const RedirectDomain& redirectDomain)
1272 {
1273     ASSERT(!RunLoop::isMain());
1274
1275     auto result = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1276
1277     // For consistency, make sure we also have a statistics entry for the redirect domain.
1278     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1279
1280     insertDomainRelationship(m_topFrameUniqueRedirectsFrom, result.second, redirectDomain);
1281 }
1282
1283 std::pair<ResourceLoadStatisticsDatabaseStore::AddedRecord, unsigned> ResourceLoadStatisticsDatabaseStore::ensureResourceStatisticsForRegistrableDomain(const RegistrableDomain& domain)
1284 {
1285     ASSERT(!RunLoop::isMain());
1286
1287     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK) {
1288         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::createSchema failed, error message: %{public}s", this, m_database.lastErrorMsg());
1289         ASSERT_NOT_REACHED();
1290         return { AddedRecord::No, 0 };
1291     }
1292     
1293     if (m_domainIDFromStringStatement.step() == SQLITE_ROW) {
1294         unsigned domainID = m_domainIDFromStringStatement.getColumnInt(0);
1295
1296         int resetResult = m_domainIDFromStringStatement.reset();
1297         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1298         return { AddedRecord::No, domainID };
1299     }
1300
1301     int resetResult = m_domainIDFromStringStatement.reset();
1302     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1303
1304     ResourceLoadStatistics newObservation(domain);
1305     insertObservedDomain(newObservation);
1306
1307     return { AddedRecord::Yes, domainID(domain) };
1308 }
1309
1310 void ResourceLoadStatisticsDatabaseStore::clear(CompletionHandler<void()>&& completionHandler)
1311 {
1312     ASSERT(!RunLoop::isMain());
1313
1314     clearOperatingDates();
1315
1316     auto callbackAggregator = CallbackAggregator::create(WTFMove(completionHandler));
1317
1318     removeAllStorageAccess([callbackAggregator = callbackAggregator.copyRef()] { });
1319
1320     auto registrableDomainsToBlockAndDeleteCookiesFor = ensurePrevalentResourcesForDebugMode();
1321     RegistrableDomainsToBlockCookiesFor domainsToBlock { registrableDomainsToBlockAndDeleteCookiesFor, { } };
1322     updateCookieBlockingForDomains(domainsToBlock, [callbackAggregator = callbackAggregator.copyRef()] { });
1323 }
1324
1325 ResourceLoadStatisticsDatabaseStore::CookieTreatmentResult ResourceLoadStatisticsDatabaseStore::cookieTreatmentForOrigin(const RegistrableDomain& domain) const
1326 {
1327     ASSERT(!RunLoop::isMain());
1328
1329     SQLiteStatement statement(m_database, makeString("SELECT isPrevalent, hadUserInteraction FROM ObservedDomains WHERE registrableDomain = ", domain.string()));
1330     if (statement.prepare() != SQLITE_OK
1331         || statement.step() != SQLITE_ROW) {
1332         return CookieTreatmentResult::Allow;
1333     }
1334     
1335     bool isPrevalent = !!statement.getColumnInt(0);
1336     if (!isPrevalent)
1337         return CookieTreatmentResult::Allow;
1338
1339     bool hadUserInteraction = statement.getColumnInt(1) ? true : false;
1340     return hadUserInteraction ? CookieTreatmentResult::BlockAndKeep : CookieTreatmentResult::BlockAndPurge;
1341 }
1342     
1343 StorageAccessPromptWasShown ResourceLoadStatisticsDatabaseStore::hasUserGrantedStorageAccessThroughPrompt(unsigned requestingDomainID, const RegistrableDomain& firstPartyDomain) const
1344 {
1345     ASSERT(!RunLoop::isMain());
1346
1347     auto firstPartyPrimaryDomainID = domainID(firstPartyDomain);
1348
1349     SQLiteStatement statement(m_database, makeString("SELECT COUNT(*) FROM StorageAccessUnderTopFrameDomains WHERE domainID = ", String::number(requestingDomainID), " AND topLevelDomainID = ", String::number(firstPartyPrimaryDomainID)));
1350     if (statement.prepare() != SQLITE_OK
1351         || statement.step() != SQLITE_ROW)
1352         return StorageAccessPromptWasShown::No;
1353
1354     return !statement.getColumnInt(0) ? StorageAccessPromptWasShown::Yes : StorageAccessPromptWasShown::No;
1355 }
1356
1357 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::domainsToBlockAndDeleteCookiesFor() const
1358 {
1359     ASSERT(!RunLoop::isMain());
1360     
1361     Vector<RegistrableDomain> results;
1362     SQLiteStatement statement(m_database, "SELECT registrableDomain FROM ObservedDomains WHERE isPrevalent = 1 AND hadUserInteraction = 0"_s);
1363     if (statement.prepare() != SQLITE_OK)
1364         return results;
1365     
1366     while (statement.step() == SQLITE_ROW)
1367         results.append(RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(0)));
1368     
1369     return results;
1370 }
1371
1372 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::domainsToBlockButKeepCookiesFor() const
1373 {
1374     ASSERT(!RunLoop::isMain());
1375
1376     Vector<RegistrableDomain> results;
1377     SQLiteStatement statement(m_database, "SELECT registrableDomain FROM ObservedDomains WHERE isPrevalent = 1 AND hadUserInteraction = 1"_s);
1378     if (statement.prepare() != SQLITE_OK)
1379         return results;
1380     
1381     while (statement.step() == SQLITE_ROW)
1382         results.append(RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(0)));
1383
1384     return results;
1385 }
1386
1387 void ResourceLoadStatisticsDatabaseStore::updateCookieBlocking(CompletionHandler<void()>&& completionHandler)
1388 {
1389     ASSERT(!RunLoop::isMain());
1390
1391     auto domainsToBlockAndDeleteCookiesFor = this->domainsToBlockAndDeleteCookiesFor();
1392     auto domainsToBlockButKeepCookiesFor = this->domainsToBlockButKeepCookiesFor();
1393
1394     if (domainsToBlockAndDeleteCookiesFor.isEmpty() && domainsToBlockButKeepCookiesFor.isEmpty()) {
1395         completionHandler();
1396         return;
1397     }
1398
1399     RegistrableDomainsToBlockCookiesFor domainsToBlock { domainsToBlockAndDeleteCookiesFor, domainsToBlockButKeepCookiesFor };
1400
1401     if (debugLoggingEnabled() && !domainsToBlockAndDeleteCookiesFor.isEmpty() && !domainsToBlockButKeepCookiesFor.isEmpty())
1402         debugLogDomainsInBatches("block", domainsToBlock);
1403
1404     RunLoop::main().dispatch([weakThis = makeWeakPtr(*this), store = makeRef(store()), domainsToBlock = crossThreadCopy(domainsToBlock), completionHandler = WTFMove(completionHandler)] () mutable {
1405         store->callUpdatePrevalentDomainsToBlockCookiesForHandler(domainsToBlock, [weakThis = WTFMove(weakThis), store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1406             store->statisticsQueue().dispatch([weakThis = WTFMove(weakThis), completionHandler = WTFMove(completionHandler)]() mutable {
1407                 completionHandler();
1408                 if (!weakThis)
1409                     return;
1410                 RELEASE_LOG_INFO_IF(weakThis->debugLoggingEnabled(), ITPDebug, "Done updating cookie blocking.");
1411             });
1412         });
1413     });
1414 }
1415
1416 Vector<ResourceLoadStatisticsDatabaseStore::PrevalentDomainData> ResourceLoadStatisticsDatabaseStore::prevalentDomains() const
1417 {
1418     ASSERT(!RunLoop::isMain());
1419     
1420     Vector<PrevalentDomainData> results;
1421     SQLiteStatement statement(m_database, "SELECT domainID, registrableDomain, mostRecentUserInteractionTime, hadUserInteraction, grandfathered FROM ObservedDomains WHERE isPrevalent = 1"_s);
1422     if (statement.prepare() != SQLITE_OK)
1423         return results;
1424     
1425     while (statement.step() == SQLITE_ROW) {
1426         results.append({ static_cast<unsigned>(statement.getColumnInt(0))
1427             , RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(1))
1428             , WallTime::fromRawSeconds(statement.getColumnDouble(2))
1429             , statement.getColumnInt(3) ? true : false
1430             , statement.getColumnInt(4) ? true : false
1431         });
1432     }
1433     
1434     return results;
1435 }
1436
1437 Vector<unsigned> ResourceLoadStatisticsDatabaseStore::findExpiredUserInteractions() const
1438 {
1439     ASSERT(!RunLoop::isMain());
1440
1441     Vector<unsigned> results;
1442     Optional<Seconds> expirationDateTime = statisticsEpirationTime();
1443     if (!expirationDateTime)
1444         return results;
1445
1446     if (m_findExpiredUserInteractionStatement.bindDouble(1, expirationDateTime.value().value()) != SQLITE_OK)
1447         return results;
1448
1449     while (m_findExpiredUserInteractionStatement.step() == SQLITE_ROW)
1450         results.append(m_findExpiredUserInteractionStatement.getColumnInt(0));
1451
1452     int resetResult = m_findExpiredUserInteractionStatement.reset();
1453     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1454
1455     return results;
1456 }
1457
1458 void ResourceLoadStatisticsDatabaseStore::clearExpiredUserInteractions()
1459 {
1460     ASSERT(!RunLoop::isMain());
1461
1462     auto expiredRecords = findExpiredUserInteractions();
1463     if (expiredRecords.isEmpty())
1464         return;
1465
1466     auto expiredRecordIDs = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(expiredRecords.begin(), expiredRecords.end()));
1467
1468     SQLiteStatement clearExpiredInteraction(m_database, makeString("UPDATE ObservedDomains SET mostRecentUserInteractionTime = 0, hadUserInteraction = 1 WHERE domainID IN (", expiredRecordIDs, ")"));
1469     if (clearExpiredInteraction.prepare() != SQLITE_OK)
1470         return;
1471
1472     SQLiteStatement removeStorageAccess(m_database, makeString("DELETE FROM StorageAccessUnderTopFrameDomains ", expiredRecordIDs, ")"));
1473     if (removeStorageAccess.prepare() != SQLITE_OK)
1474         return;
1475
1476     if (clearExpiredInteraction.step() != SQLITE_DONE
1477         || removeStorageAccess.step() != SQLITE_DONE) {
1478         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::logUserInteraction failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1479         ASSERT_NOT_REACHED();
1480     }
1481 }
1482
1483 void ResourceLoadStatisticsDatabaseStore::clearGrandfathering(Vector<unsigned>&& domainIDsToClear)
1484 {
1485     ASSERT(!RunLoop::isMain());
1486
1487     if (domainIDsToClear.isEmpty())
1488         return;
1489
1490     auto listToClear = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(domainIDsToClear.begin(), domainIDsToClear.end()));
1491
1492     SQLiteStatement clearGrandfatheringStatement(m_database, makeString("UPDATE ObservedDomains SET grandfathered = 0 WHERE domainID IN (", listToClear, ")"));
1493     if (clearGrandfatheringStatement.prepare() != SQLITE_OK)
1494         return;
1495     
1496     if (clearGrandfatheringStatement.step() != SQLITE_DONE) {
1497         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::clearGrandfathering failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1498         ASSERT_NOT_REACHED();
1499     }
1500 }
1501
1502 bool ResourceLoadStatisticsDatabaseStore::shouldRemoveAllWebsiteDataFor(const PrevalentDomainData& resourceStatistic, bool shouldCheckForGrandfathering) const
1503 {
1504     return !resourceStatistic.hadUserInteraction && (!shouldCheckForGrandfathering || !resourceStatistic.grandfathered);
1505 }
1506
1507 bool ResourceLoadStatisticsDatabaseStore::shouldRemoveAllButCookiesFor(const PrevalentDomainData& resourceStatistic, bool shouldCheckForGrandfathering) const
1508 {
1509     UNUSED_PARAM(resourceStatistic);
1510     UNUSED_PARAM(shouldCheckForGrandfathering);
1511     return false;
1512 }
1513
1514 HashMap<RegistrableDomain, WebsiteDataToRemove> ResourceLoadStatisticsDatabaseStore::registrableDomainsToRemoveWebsiteDataFor()
1515 {
1516     ASSERT(!RunLoop::isMain());
1517
1518     bool shouldCheckForGrandfathering = endOfGrandfatheringTimestamp() > WallTime::now();
1519     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && endOfGrandfatheringTimestamp();
1520
1521     if (shouldClearGrandfathering)
1522         clearEndOfGrandfatheringTimeStamp();
1523
1524     clearExpiredUserInteractions();
1525     
1526     HashMap<RegistrableDomain, WebsiteDataToRemove> domainsToRemoveWebsiteDataFor;
1527
1528     Vector<PrevalentDomainData> prevalentDomains = this->prevalentDomains();
1529     Vector<unsigned> domainIDsToClearGrandfathering;
1530     for (auto& statistic : prevalentDomains) {
1531         if (shouldRemoveAllWebsiteDataFor(statistic, shouldCheckForGrandfathering))
1532             domainsToRemoveWebsiteDataFor.add(statistic.registerableDomain, WebsiteDataToRemove::All);
1533         else if (shouldRemoveAllButCookiesFor(statistic, shouldCheckForGrandfathering))
1534             domainsToRemoveWebsiteDataFor.add(statistic.registerableDomain, WebsiteDataToRemove::AllButCookies);
1535
1536         if (shouldClearGrandfathering && statistic.grandfathered)
1537             domainIDsToClearGrandfathering.append(statistic.domainID);
1538     }
1539
1540     clearGrandfathering(WTFMove(domainIDsToClearGrandfathering));
1541     
1542     return domainsToRemoveWebsiteDataFor;
1543 }
1544
1545 void ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded()
1546 {
1547     ASSERT(!RunLoop::isMain());
1548
1549     unsigned count = 0;
1550     if (m_observedDomainCount.step() == SQLITE_ROW)
1551         count = m_observedDomainCount.getColumnInt(0);
1552
1553     int resetResult = m_observedDomainCount.reset();
1554     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1555
1556     if (count <= parameters().maxStatisticsEntries)
1557         return;
1558
1559     ASSERT(parameters().pruneEntriesDownTo <= parameters().maxStatisticsEntries);
1560
1561     size_t countLeftToPrune = count - parameters().pruneEntriesDownTo;
1562     ASSERT(countLeftToPrune);
1563
1564     SQLiteStatement recordsToPrune(m_database, makeString("SELECT domainID FROM ObservedDomains ORDER BY hadUserInteraction, isPrevalent, lastSeen DESC LIMIT ", String::number(countLeftToPrune)));
1565     if (recordsToPrune.prepare() != SQLITE_OK) {
1566         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded failed, error message: %{public}s", this, m_database.lastErrorMsg());
1567         ASSERT_NOT_REACHED();
1568         return;
1569     }
1570
1571     Vector<unsigned> entriesToPrune;
1572     while (recordsToPrune.step() == SQLITE_ROW)
1573         entriesToPrune.append(recordsToPrune.getColumnInt(0));
1574
1575     auto listToPrune = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(entriesToPrune.begin(), entriesToPrune.end()));
1576
1577     SQLiteStatement pruneCommand(m_database, makeString("DELETE from ObservedDomains WHERE domainID IN (", listToPrune, ")"));
1578     if (pruneCommand.prepare() != SQLITE_OK
1579         || pruneCommand.step() != SQLITE_DONE) {
1580         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded failed, error message: %{public}s", this, m_database.lastErrorMsg());
1581         ASSERT_NOT_REACHED();
1582         return;
1583     }
1584 }
1585
1586 void ResourceLoadStatisticsDatabaseStore::updateLastSeen(const RegistrableDomain& domain, WallTime lastSeen)
1587 {
1588     ASSERT(!RunLoop::isMain());
1589
1590     if (m_updateLastSeenStatement.bindDouble(1, lastSeen.secondsSinceEpoch().value()) != SQLITE_OK
1591         || m_updateLastSeenStatement.bindText(2, domain.string()) != SQLITE_OK
1592         || m_updateLastSeenStatement.step() != SQLITE_DONE) {
1593         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::updateLastSeen failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1594         ASSERT_NOT_REACHED();
1595         return;
1596     }
1597     
1598     int resetResult = m_updateLastSeenStatement.reset();
1599     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1600 }
1601
1602 void ResourceLoadStatisticsDatabaseStore::setLastSeen(const RegistrableDomain& domain, Seconds seconds)
1603 {
1604     ASSERT(!RunLoop::isMain());
1605
1606     ensureResourceStatisticsForRegistrableDomain(domain);
1607     updateLastSeen(domain, WallTime::fromRawSeconds(seconds.seconds()));
1608 }
1609
1610 void ResourceLoadStatisticsDatabaseStore::setPrevalentResource(const RegistrableDomain& domain)
1611 {
1612     ASSERT(!RunLoop::isMain());
1613
1614     if (shouldSkip(domain))
1615         return;
1616
1617     ensureResourceStatisticsForRegistrableDomain(domain);
1618     setPrevalentResource(domain, ResourceLoadPrevalence::High);
1619 }
1620
1621 void ResourceLoadStatisticsDatabaseStore::setVeryPrevalentResource(const RegistrableDomain& domain)
1622 {
1623     ASSERT(!RunLoop::isMain());
1624
1625     if (shouldSkip(domain))
1626         return;
1627     
1628     ensureResourceStatisticsForRegistrableDomain(domain);
1629     setPrevalentResource(domain, ResourceLoadPrevalence::VeryHigh);
1630 }
1631
1632 } // namespace WebKit
1633
1634 #endif