88fdb7e6d2c64d2cd93d85371df6a1d1c0172ece
[WebKit-https.git] / Source / WebKit / NetworkProcess / Classifier / ResourceLoadStatisticsDatabaseStore.cpp
1 /*
2  * Copyright (C) 2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ResourceLoadStatisticsDatabaseStore.h"
28
29 #if ENABLE(RESOURCE_LOAD_STATISTICS)
30
31 #include "Logging.h"
32 #include "NetworkSession.h"
33 #include "PluginProcessManager.h"
34 #include "PluginProcessProxy.h"
35 #include "ResourceLoadStatisticsMemoryStore.h"
36 #include "StorageAccessStatus.h"
37 #include "WebProcessProxy.h"
38 #include "WebResourceLoadStatisticsTelemetry.h"
39 #include "WebsiteDataStore.h"
40 #include <WebCore/DocumentStorageAccess.h>
41 #include <WebCore/KeyedCoding.h>
42 #include <WebCore/NetworkStorageSession.h>
43 #include <WebCore/ResourceLoadStatistics.h>
44 #include <WebCore/SQLiteDatabase.h>
45 #include <WebCore/SQLiteStatement.h>
46 #include <WebCore/UserGestureIndicator.h>
47 #include <wtf/CallbackAggregator.h>
48 #include <wtf/DateMath.h>
49 #include <wtf/HashMap.h>
50 #include <wtf/MathExtras.h>
51 #include <wtf/StdSet.h>
52 #include <wtf/text/StringBuilder.h>
53
54 namespace WebKit {
55 using namespace WebCore;
56
57 #if PLATFORM(COCOA)
58 #define RELEASE_LOG_IF_ALLOWED(sessionID, fmt, ...) RELEASE_LOG_IF(sessionID.isAlwaysOnLoggingAllowed(), Network, "%p - ResourceLoadStatisticsDatabaseStore::" fmt, this, ##__VA_ARGS__)
59 #define RELEASE_LOG_ERROR_IF_ALLOWED(sessionID, fmt, ...) RELEASE_LOG_ERROR_IF(sessionID.isAlwaysOnLoggingAllowed(), Network, "%p - ResourceLoadStatisticsDatabaseStore::" fmt, this, ##__VA_ARGS__)
60 #else
61 #define RELEASE_LOG_IF_ALLOWED(sessionID, fmt, ...)  ((void)0)
62 #define RELEASE_LOG_ERROR_IF_ALLOWED(sessionID, fmt, ...)  ((void)0)
63 #endif
64
65 constexpr auto observedDomainCountQuery = "SELECT COUNT(*) FROM ObservedDomains"_s;
66 constexpr auto insertObservedDomainQuery = "INSERT INTO ObservedDomains (registrableDomain, lastSeen, hadUserInteraction,"
67     "mostRecentUserInteractionTime, grandfathered, isPrevalent, isVeryPrevalent, dataRecordsRemoved, timesAccessedAsFirstPartyDueToUserInteraction,"
68     "timesAccessedAsFirstPartyDueToStorageAccessAPI) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"_s;
69 constexpr auto insertTopLevelDomainQuery = "INSERT INTO TopLevelDomains VALUES (?)"_s;
70 constexpr auto domainIDFromStringQuery = "SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
71 constexpr auto storageAccessUnderTopFrameDomainsQuery = "INSERT INTO StorageAccessUnderTopFrameDomains (domainID, topLevelDomainID) "
72     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
73 constexpr auto storageAccessUnderTopFrameDomainsExistsQuery = "SELECT EXISTS (SELECT 1 FROM StorageAccessUnderTopFrameDomains WHERE domainID = ? "
74 "AND topLevelDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
75 constexpr auto topFrameUniqueRedirectsToQuery = "INSERT INTO TopFrameUniqueRedirectsTo (sourceDomainID, toDomainID) SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
76 constexpr auto topFrameUniqueRedirectsToExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameUniqueRedirectsTo WHERE sourceDomainID = ? "
77     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
78 constexpr auto topFrameUniqueRedirectsFromQuery = "INSERT INTO TopFrameUniqueRedirectsFrom (targetDomainID, fromDomainID) "
79     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
80 constexpr auto topFrameUniqueRedirectsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameUniqueRedirectsFrom WHERE targetDomainID = ? "
81     "AND fromDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
82 constexpr auto topFrameLinkDecorationsFromQuery = "INSERT INTO TopFrameLinkDecorationsFrom (fromDomainID, toDomainID) "
83     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
84 constexpr auto topFrameLinkDecorationsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameLinkDecorationsFrom WHERE fromDomainID = ? "
85     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
86 constexpr auto subframeUnderTopFrameDomainsQuery = "INSERT INTO SubframeUnderTopFrameDomains (subFrameDomainID, topFrameDomainID) "
87     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
88 constexpr auto subframeUnderTopFrameDomainExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubframeUnderTopFrameDomains WHERE subFrameDomainID = ? "
89     "AND topFrameDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
90 constexpr auto subresourceUnderTopFrameDomainsQuery = "INSERT INTO SubresourceUnderTopFrameDomains (subresourceDomainID, topFrameDomainID) "
91     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
92 constexpr auto subresourceUnderTopFrameDomainExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUnderTopFrameDomains "
93     "WHERE subresourceDomainID = ? AND topFrameDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
94 constexpr auto subresourceUniqueRedirectsToQuery = "INSERT INTO SubresourceUniqueRedirectsTo (subresourceDomainID, toDomainID) "
95     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
96 constexpr auto subresourceUniqueRedirectsToExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUniqueRedirectsTo WHERE subresourceDomainID = ? "
97     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
98 constexpr auto subresourceUniqueRedirectsFromQuery = "INSERT INTO SubresourceUniqueRedirectsFrom (subresourceDomainID, fromDomainID) "
99     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
100 constexpr auto subresourceUniqueRedirectsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUniqueRedirectsFrom WHERE subresourceDomainID = ? "
101     "AND fromDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
102 constexpr auto mostRecentUserInteractionQuery = "UPDATE ObservedDomains SET hadUserInteraction = ?, mostRecentUserInteractionTime = ? "
103     "WHERE registrableDomain = ?"_s;
104 constexpr auto updateLastSeenQuery = "UPDATE ObservedDomains SET lastSeen = ? WHERE registrableDomain = ?"_s;
105 constexpr auto updateDataRecordsRemovedQuery = "UPDATE ObservedDomains SET dataRecordsRemoved = ? WHERE registrableDomain = ?"_s;
106 constexpr auto updatePrevalentResourceQuery = "UPDATE ObservedDomains SET isPrevalent = ? WHERE registrableDomain = ?"_s;
107 constexpr auto isPrevalentResourceQuery = "SELECT isPrevalent FROM ObservedDomains WHERE registrableDomain = ?"_s;
108 constexpr auto updateVeryPrevalentResourceQuery = "UPDATE ObservedDomains SET isVeryPrevalent = ? WHERE registrableDomain = ?"_s;
109 constexpr auto isVeryPrevalentResourceQuery = "SELECT isVeryPrevalent FROM ObservedDomains WHERE registrableDomain = ?"_s;
110 constexpr auto clearPrevalentResourceQuery = "UPDATE ObservedDomains SET isPrevalent = 0, isVeryPrevalent = 0 WHERE registrableDomain = ?"_s;
111 constexpr auto hadUserInteractionQuery = "SELECT hadUserInteraction, mostRecentUserInteractionTime FROM ObservedDomains WHERE registrableDomain = ?"_s;
112 constexpr auto updateGrandfatheredQuery = "UPDATE ObservedDomains SET grandfathered = ? WHERE registrableDomain = ?"_s;
113 constexpr auto isGrandfatheredQuery = "SELECT grandfathered FROM ObservedDomains WHERE registrableDomain = ?"_s;
114 constexpr auto findExpiredUserInteractionQuery = "SELECT domainID FROM ObservedDomains WHERE hadUserInteraction = 1 AND mostRecentUserInteractionTime < ?"_s;
115
116 constexpr auto createObservedDomain = "CREATE TABLE ObservedDomains ("
117     "domainID INTEGER PRIMARY KEY, registrableDomain TEXT NOT NULL UNIQUE ON CONFLICT FAIL, lastSeen REAL NOT NULL, "
118     "hadUserInteraction INTEGER NOT NULL, mostRecentUserInteractionTime REAL NOT NULL, grandfathered INTEGER NOT NULL, "
119     "isPrevalent INTEGER NOT NULL, isVeryPrevalent INTEGER NOT NULL, dataRecordsRemoved INTEGER NOT NULL,"
120     "timesAccessedAsFirstPartyDueToUserInteraction INTEGER NOT NULL, timesAccessedAsFirstPartyDueToStorageAccessAPI INTEGER NOT NULL);"_s;
121
122 enum {
123     DomainIDIndex,
124     RegistrableDomainIndex,
125     LastSeenIndex,
126     HadUserInteractionIndex,
127     MostRecentUserInteractionTimeIndex,
128     GrandfatheredIndex,
129     IsPrevalentIndex,
130     IsVeryPrevalentIndex,
131     DataRecordsRemovedIndex,
132     TimesAccessedAsFirstPartyDueToUserInteractionIndex,
133     TimesAccessedAsFirstPartyDueToStorageAccessAPIIndex
134 };
135
136 constexpr auto createTopLevelDomains = "CREATE TABLE TopLevelDomains ("
137     "topLevelDomainID INTEGER PRIMARY KEY, CONSTRAINT fkDomainID FOREIGN KEY(topLevelDomainID) "
138     "REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
139     
140 constexpr auto createStorageAccessUnderTopFrameDomains = "CREATE TABLE StorageAccessUnderTopFrameDomains ("
141     "domainID INTEGER NOT NULL, topLevelDomainID INTEGER NOT NULL ON CONFLICT FAIL, "
142     "CONSTRAINT fkDomainID FOREIGN KEY(domainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
143     "FOREIGN KEY(topLevelDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
144     
145 constexpr auto createTopFrameUniqueRedirectsTo = "CREATE TABLE TopFrameUniqueRedirectsTo ("
146     "sourceDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
147     "FOREIGN KEY(sourceDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
148     "FOREIGN KEY(toDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
149     
150 constexpr auto createTopFrameUniqueRedirectsFrom = "CREATE TABLE TopFrameUniqueRedirectsFrom ("
151     "targetDomainID INTEGER NOT NULL, fromDomainID INTEGER NOT NULL, "
152     "FOREIGN KEY(targetDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
153     "FOREIGN KEY(fromDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
154
155 constexpr auto createTopFrameLinkDecorationsFrom = "CREATE TABLE TopFrameLinkDecorationsFrom ("
156     "fromDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
157     "FOREIGN KEY(fromDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
158     "FOREIGN KEY(toDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
159
160 constexpr auto createSubframeUnderTopFrameDomains = "CREATE TABLE SubframeUnderTopFrameDomains ("
161     "subFrameDomainID INTEGER NOT NULL, topFrameDomainID INTEGER NOT NULL, "
162     "FOREIGN KEY(subFrameDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
163     "FOREIGN KEY(topFrameDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
164     
165 constexpr auto createSubresourceUnderTopFrameDomains = "CREATE TABLE SubresourceUnderTopFrameDomains ("
166     "subresourceDomainID INTEGER NOT NULL, topFrameDomainID INTEGER NOT NULL, "
167     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
168     "FOREIGN KEY(topFrameDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
169     
170 constexpr auto createSubresourceUniqueRedirectsTo = "CREATE TABLE SubresourceUniqueRedirectsTo ("
171     "subresourceDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
172     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
173     "FOREIGN KEY(toDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
174     
175 constexpr auto createSubresourceUniqueRedirectsFrom = "CREATE TABLE SubresourceUniqueRedirectsFrom ("
176     "subresourceDomainID INTEGER NOT NULL, fromDomainID INTEGER NOT NULL, "
177     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
178     "FOREIGN KEY(fromDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
179     
180 ResourceLoadStatisticsDatabaseStore::ResourceLoadStatisticsDatabaseStore(WebResourceLoadStatisticsStore& store, WorkQueue& workQueue, ShouldIncludeLocalhost shouldIncludeLocalhost, const String& storageDirectoryPath, PAL::SessionID sessionID)
181     : ResourceLoadStatisticsStore(store, workQueue, shouldIncludeLocalhost)
182     , m_storageDirectoryPath(storageDirectoryPath + "/observations.db")
183     , m_observedDomainCount(m_database, observedDomainCountQuery)
184     , m_insertObservedDomainStatement(m_database, insertObservedDomainQuery)
185     , m_insertTopLevelDomainStatement(m_database, insertTopLevelDomainQuery)
186     , m_domainIDFromStringStatement(m_database, domainIDFromStringQuery)
187     , m_storageAccessUnderTopFrameDomainsStatement(m_database, storageAccessUnderTopFrameDomainsQuery)
188     , m_storageAccessUnderTopFrameDomainsExistsStatement(m_database, storageAccessUnderTopFrameDomainsExistsQuery)
189     , m_topFrameUniqueRedirectsTo(m_database, topFrameUniqueRedirectsToQuery)
190     , m_topFrameUniqueRedirectsToExists(m_database, topFrameUniqueRedirectsToExistsQuery)
191     , m_topFrameUniqueRedirectsFrom(m_database, topFrameUniqueRedirectsFromQuery)
192     , m_topFrameUniqueRedirectsFromExists(m_database, topFrameUniqueRedirectsFromExistsQuery)
193     , m_topFrameLinkDecorationsFrom(m_database, topFrameLinkDecorationsFromQuery)
194     , m_topFrameLinkDecorationsFromExists(m_database, topFrameLinkDecorationsFromExistsQuery)
195     , m_subframeUnderTopFrameDomains(m_database, subframeUnderTopFrameDomainsQuery)
196     , m_subframeUnderTopFrameDomainExists(m_database, subframeUnderTopFrameDomainExistsQuery)
197     , m_subresourceUnderTopFrameDomains(m_database, subresourceUnderTopFrameDomainsQuery)
198     , m_subresourceUnderTopFrameDomainExists(m_database, subresourceUnderTopFrameDomainExistsQuery)
199     , m_subresourceUniqueRedirectsTo(m_database, subresourceUniqueRedirectsToQuery)
200     , m_subresourceUniqueRedirectsToExists(m_database, subresourceUniqueRedirectsToExistsQuery)
201     , m_subresourceUniqueRedirectsFrom(m_database, subresourceUniqueRedirectsFromQuery)
202     , m_subresourceUniqueRedirectsFromExists(m_database, subresourceUniqueRedirectsFromExistsQuery)
203     , m_mostRecentUserInteractionStatement(m_database, mostRecentUserInteractionQuery)
204     , m_updateLastSeenStatement(m_database, updateLastSeenQuery)
205     , m_updateDataRecordsRemovedStatement(m_database, updateDataRecordsRemovedQuery)
206     , m_updatePrevalentResourceStatement(m_database, updatePrevalentResourceQuery)
207     , m_isPrevalentResourceStatement(m_database, isPrevalentResourceQuery)
208     , m_updateVeryPrevalentResourceStatement(m_database, updateVeryPrevalentResourceQuery)
209     , m_isVeryPrevalentResourceStatement(m_database, isVeryPrevalentResourceQuery)
210     , m_clearPrevalentResourceStatement(m_database, clearPrevalentResourceQuery)
211     , m_hadUserInteractionStatement(m_database, hadUserInteractionQuery)
212     , m_updateGrandfatheredStatement(m_database, updateGrandfatheredQuery)
213     , m_isGrandfatheredStatement(m_database, isGrandfatheredQuery)
214     , m_findExpiredUserInteractionStatement(m_database, findExpiredUserInteractionQuery)
215     , m_sessionID(sessionID)
216 {
217     ASSERT(!RunLoop::isMain());
218
219     if (!m_database.open(m_storageDirectoryPath)) {
220         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::open failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
221         ASSERT_NOT_REACHED();
222         return;
223     }
224     
225     // Since we are using a workerQueue, the sequential dispatch blocks may be called by different threads.
226     m_database.disableThreadingChecks();
227
228     if (!m_database.tableExists("ObservedDomains"_s)) {
229         if (!createSchema()) {
230             RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::createSchema failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
231             ASSERT_NOT_REACHED();
232             return;
233         }
234     }
235     
236     if (!m_database.turnOnIncrementalAutoVacuum())
237         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::turnOnIncrementalAutoVacuum failed, error message: %{public}s", this, m_database.lastErrorMsg());
238
239     if (!prepareStatements()) {
240         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::prepareStatements failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
241         ASSERT_NOT_REACHED();
242         return;
243     }
244
245     workQueue.dispatchAfter(5_s, [weakThis = makeWeakPtr(*this)] {
246         if (weakThis)
247             weakThis->calculateAndSubmitTelemetry();
248     });
249 }
250
251 bool ResourceLoadStatisticsDatabaseStore::isEmpty() const
252 {
253     ASSERT(!RunLoop::isMain());
254     
255     bool result = false;
256     if (m_observedDomainCount.step() == SQLITE_ROW)
257         result = !m_observedDomainCount.getColumnInt(0);
258     
259     int resetResult = m_observedDomainCount.reset();
260     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
261     
262     return result;
263 }
264
265 bool ResourceLoadStatisticsDatabaseStore::createSchema()
266 {
267     ASSERT(!RunLoop::isMain());
268
269     if (!m_database.executeCommand(createObservedDomain)) {
270         LOG_ERROR("Could not create ObservedDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
271         return false;
272     }
273
274     if (!m_database.executeCommand(createTopLevelDomains)) {
275         LOG_ERROR("Could not create TopLevelDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
276         return false;
277     }
278
279     if (!m_database.executeCommand(createStorageAccessUnderTopFrameDomains)) {
280         LOG_ERROR("Could not create StorageAccessUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
281         return false;
282     }
283
284     if (!m_database.executeCommand(createTopFrameUniqueRedirectsTo)) {
285         LOG_ERROR("Could not create TopFrameUniqueRedirectsTo table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
286         return false;
287     }
288
289     if (!m_database.executeCommand(createTopFrameUniqueRedirectsFrom)) {
290         LOG_ERROR("Could not create TopFrameUniqueRedirectsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
291         return false;
292     }
293
294     if (!m_database.executeCommand(createTopFrameLinkDecorationsFrom)) {
295         LOG_ERROR("Could not create TopFrameLinkDecorationsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
296         return false;
297     }
298     
299     if (!m_database.executeCommand(createSubframeUnderTopFrameDomains)) {
300         LOG_ERROR("Could not create SubframeUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
301         return false;
302     }
303
304     if (!m_database.executeCommand(createSubresourceUnderTopFrameDomains)) {
305         LOG_ERROR("Could not create SubresourceUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
306         return false;
307     }
308
309     if (!m_database.executeCommand(createSubresourceUniqueRedirectsTo)) {
310         LOG_ERROR("Could not create SubresourceUniqueRedirectsTo table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
311         return false;
312     }
313
314     if (!m_database.executeCommand(createSubresourceUniqueRedirectsFrom)) {
315         LOG_ERROR("Could not create SubresourceUniqueRedirectsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
316         return false;
317     }
318
319     return true;
320 }
321
322 bool ResourceLoadStatisticsDatabaseStore::prepareStatements()
323 {
324     ASSERT(!RunLoop::isMain());
325         
326     if (m_observedDomainCount.prepare() != SQLITE_OK
327         || m_insertObservedDomainStatement.prepare() != SQLITE_OK
328         || m_insertTopLevelDomainStatement.prepare() != SQLITE_OK
329         || m_domainIDFromStringStatement.prepare() != SQLITE_OK
330         || m_storageAccessUnderTopFrameDomainsStatement.prepare() != SQLITE_OK
331         || m_storageAccessUnderTopFrameDomainsExistsStatement.prepare() != SQLITE_OK
332         || m_topFrameUniqueRedirectsTo.prepare() != SQLITE_OK
333         || m_topFrameUniqueRedirectsToExists.prepare() != SQLITE_OK
334         || m_topFrameUniqueRedirectsFrom.prepare() != SQLITE_OK
335         || m_topFrameUniqueRedirectsFromExists.prepare() != SQLITE_OK
336         || m_subframeUnderTopFrameDomains.prepare() != SQLITE_OK
337         || m_subframeUnderTopFrameDomainExists.prepare() != SQLITE_OK
338         || m_subresourceUnderTopFrameDomains.prepare() != SQLITE_OK
339         || m_subresourceUnderTopFrameDomainExists.prepare() != SQLITE_OK
340         || m_subresourceUniqueRedirectsTo.prepare() != SQLITE_OK
341         || m_subresourceUniqueRedirectsToExists.prepare() != SQLITE_OK
342         || m_subresourceUniqueRedirectsFrom.prepare() != SQLITE_OK
343         || m_subresourceUniqueRedirectsFromExists.prepare() != SQLITE_OK
344         || m_updateLastSeenStatement.prepare() != SQLITE_OK
345         || m_updateDataRecordsRemovedStatement.prepare() != SQLITE_OK
346         || m_mostRecentUserInteractionStatement.prepare() != SQLITE_OK
347         || m_updatePrevalentResourceStatement.prepare() != SQLITE_OK
348         || m_isPrevalentResourceStatement.prepare() != SQLITE_OK
349         || m_updateVeryPrevalentResourceStatement.prepare() != SQLITE_OK
350         || m_isVeryPrevalentResourceStatement.prepare() != SQLITE_OK
351         || m_clearPrevalentResourceStatement.prepare() != SQLITE_OK
352         || m_hadUserInteractionStatement.prepare() != SQLITE_OK
353         || m_updateGrandfatheredStatement.prepare() != SQLITE_OK
354         || m_isGrandfatheredStatement.prepare() != SQLITE_OK
355         || m_findExpiredUserInteractionStatement.prepare() != SQLITE_OK
356         || m_topFrameLinkDecorationsFrom.prepare() != SQLITE_OK
357         || m_topFrameLinkDecorationsFromExists.prepare() != SQLITE_OK) {
358         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::prepareStatements failed to prepare, error message: %{public}s", this, m_database.lastErrorMsg());
359         ASSERT_NOT_REACHED();
360         return false;
361     }
362
363     return true;
364 }
365
366 bool ResourceLoadStatisticsDatabaseStore::insertObservedDomain(const ResourceLoadStatistics& loadStatistics)
367 {
368     ASSERT(!RunLoop::isMain());
369
370     if (domainID(loadStatistics.registrableDomain)) {
371         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "ResourceLoadStatisticsDatabaseStore::insertObservedDomain can only be called on domains not in the database.");
372         ASSERT_NOT_REACHED();
373         return false;
374     }
375
376     if (m_insertObservedDomainStatement.bindText(RegistrableDomainIndex, loadStatistics.registrableDomain.string()) != SQLITE_OK
377         || m_insertObservedDomainStatement.bindDouble(LastSeenIndex, loadStatistics.lastSeen.secondsSinceEpoch().value()) != SQLITE_OK
378         || m_insertObservedDomainStatement.bindInt(HadUserInteractionIndex, loadStatistics.hadUserInteraction) != SQLITE_OK
379         || m_insertObservedDomainStatement.bindDouble(MostRecentUserInteractionTimeIndex, loadStatistics.mostRecentUserInteractionTime.secondsSinceEpoch().value()) != SQLITE_OK
380         || m_insertObservedDomainStatement.bindInt(GrandfatheredIndex, loadStatistics.grandfathered) != SQLITE_OK
381         || m_insertObservedDomainStatement.bindInt(IsPrevalentIndex, loadStatistics.isPrevalentResource) != SQLITE_OK
382         || m_insertObservedDomainStatement.bindInt(IsVeryPrevalentIndex, loadStatistics.isVeryPrevalentResource) != SQLITE_OK
383         || m_insertObservedDomainStatement.bindInt(DataRecordsRemovedIndex, loadStatistics.dataRecordsRemoved) != SQLITE_OK
384         || m_insertObservedDomainStatement.bindInt(TimesAccessedAsFirstPartyDueToUserInteractionIndex, loadStatistics.timesAccessedAsFirstPartyDueToUserInteraction) != SQLITE_OK
385         || m_insertObservedDomainStatement.bindInt(TimesAccessedAsFirstPartyDueToStorageAccessAPIIndex, loadStatistics.timesAccessedAsFirstPartyDueToStorageAccessAPI) != SQLITE_OK) {
386         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::insertObservedDomain failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
387         ASSERT_NOT_REACHED();
388         return false;
389     }
390
391     if (m_insertObservedDomainStatement.step() != SQLITE_DONE) {
392         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::insertObservedDomain failed to commit, error message: %{private}s", this, m_database.lastErrorMsg());
393         ASSERT_NOT_REACHED();
394         return false;
395     }
396
397     int resetResult = m_insertObservedDomainStatement.reset();
398     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
399
400     return true;
401 }
402
403 bool ResourceLoadStatisticsDatabaseStore::relationshipExists(WebCore::SQLiteStatement& statement, Optional<unsigned> firstDomainID, const RegistrableDomain& secondDomain) const
404 {
405     if (!firstDomainID)
406         return false;
407     
408     ASSERT(!RunLoop::isMain());
409
410     if (statement.bindInt(1, *firstDomainID) != SQLITE_OK
411         || statement.bindText(2, secondDomain.string()) != SQLITE_OK
412         || statement.step() != SQLITE_ROW) {
413         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_insertDomainRelationshipStatement failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
414         ASSERT_NOT_REACHED();
415         return false;
416     }
417     bool relationshipExists = !!statement.getColumnInt(0);
418
419     int resetResult = statement.reset();
420     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
421     
422     return relationshipExists;
423 }
424
425 bool ResourceLoadStatisticsDatabaseStore::insertDomainRelationship(WebCore::SQLiteStatement& statement, unsigned domainID, const RegistrableDomain& topFrame)
426 {
427     ASSERT(!RunLoop::isMain());
428
429     if (statement.bindInt(1, domainID) != SQLITE_OK
430         || statement.bindText(2, topFrame.string()) != SQLITE_OK
431         || statement.step() != SQLITE_DONE) {
432         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_insertDomainRelationshipStatement failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
433         ASSERT_NOT_REACHED();
434         return false;
435     }
436
437     int resetResult = statement.reset();
438     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
439     
440     return true;
441 }
442
443 Optional<unsigned> ResourceLoadStatisticsDatabaseStore::domainID(const RegistrableDomain& domain) const
444 {
445     ASSERT(!RunLoop::isMain());
446
447     unsigned domainID = 0;
448
449     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK) {
450         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::domainIDFromString failed, error message: %{private}s", this, m_database.lastErrorMsg());
451         return WTF::nullopt;
452     }
453     
454     if (m_domainIDFromStringStatement.step() != SQLITE_ROW) {
455         int resetResult = m_domainIDFromStringStatement.reset();
456         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
457         return WTF::nullopt;
458     }
459
460     domainID = m_domainIDFromStringStatement.getColumnInt(0);
461
462     int resetResult = m_domainIDFromStringStatement.reset();
463     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
464     
465     return domainID;
466 }
467
468 void ResourceLoadStatisticsDatabaseStore::insertDomainRelationships(const ResourceLoadStatistics& loadStatistics)
469 {
470     ASSERT(!RunLoop::isMain());
471     auto registrableDomainID = domainID(loadStatistics.registrableDomain);
472     
473     if (!registrableDomainID)
474         return;
475     
476     for (auto& topFrameDomain : loadStatistics.storageAccessUnderTopFrameDomains) {
477         if (!relationshipExists(m_storageAccessUnderTopFrameDomainsExistsStatement, registrableDomainID.value(), topFrameDomain))
478             insertDomainRelationship(m_storageAccessUnderTopFrameDomainsStatement, registrableDomainID.value(), topFrameDomain);
479     }
480
481     for (auto& toDomain : loadStatistics.topFrameUniqueRedirectsTo) {
482         if (!relationshipExists(m_topFrameUniqueRedirectsToExists, registrableDomainID.value(), toDomain))
483             insertDomainRelationship(m_topFrameUniqueRedirectsTo, registrableDomainID.value(), toDomain);
484     }
485     
486     for (auto& fromDomain : loadStatistics.topFrameUniqueRedirectsFrom) {
487         if (!relationshipExists(m_topFrameUniqueRedirectsFromExists, registrableDomainID.value(), fromDomain))
488             insertDomainRelationship(m_topFrameUniqueRedirectsFrom, registrableDomainID.value(), fromDomain);
489     }
490     
491     for (auto& topFrameDomain : loadStatistics.subframeUnderTopFrameDomains) {
492         if (!relationshipExists(m_subframeUnderTopFrameDomainExists, registrableDomainID.value(), topFrameDomain))
493             insertDomainRelationship(m_subframeUnderTopFrameDomains, registrableDomainID.value(), topFrameDomain);
494     }
495     
496     for (auto& topFrameDomain : loadStatistics.subresourceUnderTopFrameDomains) {
497         if (!relationshipExists(m_subresourceUnderTopFrameDomainExists, registrableDomainID.value(), topFrameDomain))
498             insertDomainRelationship(m_subresourceUnderTopFrameDomains, registrableDomainID.value(), topFrameDomain);
499     }
500     
501     for (auto& toDomain : loadStatistics.subresourceUniqueRedirectsTo) {
502         if (!relationshipExists(m_subresourceUniqueRedirectsToExists, registrableDomainID.value(), toDomain))
503             insertDomainRelationship(m_subresourceUniqueRedirectsTo, registrableDomainID.value(), toDomain);
504     }
505     
506     for (auto& fromDomain : loadStatistics.subresourceUniqueRedirectsFrom) {
507         if (!relationshipExists(m_subresourceUniqueRedirectsFromExists, registrableDomainID.value(), fromDomain))
508             insertDomainRelationship(m_subresourceUniqueRedirectsFrom, registrableDomainID.value(), fromDomain);
509     }
510 }
511
512 void ResourceLoadStatisticsDatabaseStore::populateFromMemoryStore(const ResourceLoadStatisticsMemoryStore& memoryStore)
513 {
514     ASSERT(!RunLoop::isMain());
515
516     if (!isEmpty())
517         return;
518
519     auto& statisticsMap = memoryStore.data();
520     for (const auto& statistic : statisticsMap)
521         insertObservedDomain(statistic.value);
522
523     // Make a separate pass for inter-domain relationships so we
524     // can refer to the ObservedDomain table entries
525     for (auto& statistic : statisticsMap)
526         insertDomainRelationships(statistic.value);
527 }
528
529 void ResourceLoadStatisticsDatabaseStore::merge(WebCore::SQLiteStatement& current, const ResourceLoadStatistics& other)
530 {
531     ASSERT(!RunLoop::isMain());
532
533     auto currentRegistrableDomain = current.getColumnText(RegistrableDomainIndex);
534     auto currentLastSeen = current.getColumnDouble(LastSeenIndex);
535     auto currentMostRecentUserInteraction = current.getColumnDouble(MostRecentUserInteractionTimeIndex);
536     bool currentGrandfathered = current.getColumnInt(GrandfatheredIndex);
537     bool currentIsPrevalent = current.getColumnInt(IsPrevalentIndex);
538     bool currentIsVeryPrevalent = current.getColumnInt(IsVeryPrevalentIndex);
539     unsigned currentDataRecordsRemoved = current.getColumnInt(DataRecordsRemovedIndex);
540
541     ASSERT(currentRegistrableDomain == other.registrableDomain.string());
542
543     if (WallTime::fromRawSeconds(currentLastSeen) < other.lastSeen)
544         updateLastSeen(other.registrableDomain, other.lastSeen);
545
546     if (!other.hadUserInteraction) {
547         // If most recent user interaction time has been reset do so here too.
548         if (!other.mostRecentUserInteractionTime)
549             setUserInteraction(other.registrableDomain, false, { });
550     } else
551         setUserInteraction(other.registrableDomain, true, std::max(WallTime::fromRawSeconds(currentMostRecentUserInteraction), other.mostRecentUserInteractionTime));
552
553     if (other.grandfathered && !currentGrandfathered)
554         setGrandfathered(other.registrableDomain, true);
555     if (other.isPrevalentResource && !currentIsPrevalent)
556         setPrevalentResource(other.registrableDomain);
557     if (other.isVeryPrevalentResource && !currentIsVeryPrevalent)
558         setVeryPrevalentResource(other.registrableDomain);
559     if (other.dataRecordsRemoved > currentDataRecordsRemoved)
560         updateDataRecordsRemoved(other.registrableDomain, other.dataRecordsRemoved);
561 }
562
563 void ResourceLoadStatisticsDatabaseStore::mergeStatistic(const ResourceLoadStatistics& statistic)
564 {
565     ASSERT(!RunLoop::isMain());
566
567     SQLiteStatement fetchOldStatisticData(m_database, "SELECT * FROM ObservedDomains where registrableDomain = ?");
568     if (fetchOldStatisticData.prepare() != SQLITE_OK
569         || fetchOldStatisticData.bindText(1, statistic.registrableDomain.string()) != SQLITE_OK
570         || fetchOldStatisticData.step() != SQLITE_ROW) {
571         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::mergeStatistic. Statement failed to bind or domain was not found, error message: %{private}s", this, m_database.lastErrorMsg());
572         ASSERT_NOT_REACHED();
573         return;
574     }
575
576     merge(fetchOldStatisticData, statistic);
577 }
578
579 void ResourceLoadStatisticsDatabaseStore::mergeStatistics(Vector<ResourceLoadStatistics>&& statistics)
580 {
581     ASSERT(!RunLoop::isMain());
582
583     for (auto& statistic : statistics) {
584         if (!domainID(statistic.registrableDomain))
585             insertObservedDomain(statistic);
586         else
587             mergeStatistic(statistic);
588     }
589
590     // Make a separate pass for inter-domain relationships so we
591     // can refer to the ObservedDomain table entries.
592     for (auto& statistic : statistics)
593         insertDomainRelationships(statistic);
594 }
595
596 void ResourceLoadStatisticsDatabaseStore::calculateAndSubmitTelemetry() const
597 {
598     ASSERT(!RunLoop::isMain());
599
600     // FIXME(195088): Implement for Database version.
601 }
602
603 static String domainsToString(const HashSet<RegistrableDomain>& domains)
604 {
605     StringBuilder builder;
606     for (const auto& domainName : domains) {
607         if (!builder.isEmpty())
608             builder.appendLiteral(", ");
609         builder.append('"');
610         builder.append(domainName.string());
611         builder.append('"');
612     }
613
614     return builder.toString();
615 }
616
617 void ResourceLoadStatisticsDatabaseStore::incrementRecordsDeletedCountForDomains(HashSet<RegistrableDomain>&& domains)
618 {
619     ASSERT(!RunLoop::isMain());
620
621     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET dataRecordsRemoved = dataRecordsRemoved + 1 WHERE registrableDomain IN (", domainsToString(domains), ")"));
622     if (domainsToUpdateStatement.prepare() != SQLITE_OK
623         || domainsToUpdateStatement.step() != SQLITE_DONE) {
624         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::incrementStatisticsForDomains failed, error message: %{private}s", this, m_database.lastErrorMsg());
625         ASSERT_NOT_REACHED();
626     }
627 }
628
629 unsigned ResourceLoadStatisticsDatabaseStore::recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(unsigned primaryDomainID, StdSet<unsigned>& nonPrevalentRedirectionSources, unsigned numberOfRecursiveCalls)
630 {
631     ASSERT(!RunLoop::isMain());
632
633     if (numberOfRecursiveCalls >= maxNumberOfRecursiveCallsInRedirectTraceBack) {
634         RELEASE_LOG(ResourceLoadStatistics, "Hit %u recursive calls in redirect backtrace. Returning early.", maxNumberOfRecursiveCallsInRedirectTraceBack);
635         return numberOfRecursiveCalls;
636     }
637     
638     ++numberOfRecursiveCalls;
639
640     StdSet<unsigned> newlyIdentifiedDomains;
641     SQLiteStatement findSubresources(m_database, "SELECT SubresourceUniqueRedirectsFrom.fromDomainID from SubresourceUniqueRedirectsFrom INNER JOIN ObservedDomains ON ObservedDomains.domainID = SubresourceUniqueRedirectsFrom.fromDomainID WHERE subresourceDomainID = ? AND ObservedDomains.isPrevalent = 0"_s);
642     if (findSubresources.prepare() != SQLITE_OK
643         || findSubresources.bindInt(1, primaryDomainID) != SQLITE_OK) {
644         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain failed, error message: %{private}s", this, m_database.lastErrorMsg());
645         ASSERT_NOT_REACHED();
646         return 0;
647     }
648     
649     while (findSubresources.step() == SQLITE_ROW) {
650         int newDomainID = findSubresources.getColumnInt(0);
651         auto insertResult = nonPrevalentRedirectionSources.insert(newDomainID);
652         if (insertResult.second)
653             newlyIdentifiedDomains.insert(newDomainID);
654     }
655
656     SQLiteStatement findTopFrames(m_database, "SELECT TopFrameUniqueRedirectsFrom.fromDomainID from TopFrameUniqueRedirectsFrom INNER JOIN ObservedDomains ON ObservedDomains.domainID = TopFrameUniqueRedirectsFrom.fromDomainID WHERE targetDomainID = ? AND ObservedDomains.isPrevalent = 0"_s);
657     if (findTopFrames.prepare() != SQLITE_OK
658         || findTopFrames.bindInt(1, primaryDomainID) != SQLITE_OK) {
659         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain failed, error message: %{private}s", this, m_database.lastErrorMsg());
660         ASSERT_NOT_REACHED();
661         return 0;
662     }
663     
664     while (findTopFrames.step() == SQLITE_ROW) {
665         int newDomainID = findTopFrames.getColumnInt(0);
666         auto insertResult = nonPrevalentRedirectionSources.insert(newDomainID);
667         if (insertResult.second)
668             newlyIdentifiedDomains.insert(newDomainID);
669     }
670
671     if (newlyIdentifiedDomains.empty())
672         return numberOfRecursiveCalls;
673
674     for (auto domainID : newlyIdentifiedDomains)
675         numberOfRecursiveCalls = recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(domainID, nonPrevalentRedirectionSources, numberOfRecursiveCalls);
676
677     return numberOfRecursiveCalls;
678 }
679
680 template <typename IteratorType>
681 static String buildList(const WTF::IteratorRange<IteratorType>& values)
682 {
683     StringBuilder builder;
684     for (auto domainID : values) {
685         if (!builder.isEmpty())
686             builder.appendLiteral(", ");
687         builder.appendNumber(domainID);
688     }
689     return builder.toString();
690 }
691
692 void ResourceLoadStatisticsDatabaseStore::markAsPrevalentIfHasRedirectedToPrevalent()
693 {
694     ASSERT(!RunLoop::isMain());
695
696     StdSet<unsigned> prevalentDueToRedirect;
697     SQLiteStatement subresourceRedirectStatement(m_database, "SELECT DISTINCT SubresourceUniqueRedirectsTo.subresourceDomainID FROM SubresourceUniqueRedirectsTo JOIN ObservedDomains ON ObservedDomains.domainID = SubresourceUniqueRedirectsTo.toDomainID AND ObservedDomains.isPrevalent = 1"_s);
698     if (subresourceRedirectStatement.prepare() == SQLITE_OK) {
699         while (subresourceRedirectStatement.step() == SQLITE_ROW)
700             prevalentDueToRedirect.insert(subresourceRedirectStatement.getColumnInt(0));
701     }
702
703     SQLiteStatement topFrameRedirectStatement(m_database, "SELECT DISTINCT TopFrameUniqueRedirectsTo.sourceDomainID FROM TopFrameUniqueRedirectsTo JOIN ObservedDomains ON ObservedDomains.domainID = TopFrameUniqueRedirectsTo.toDomainID AND ObservedDomains.isPrevalent = 1"_s);
704     if (topFrameRedirectStatement.prepare() == SQLITE_OK) {
705         while (topFrameRedirectStatement.step() == SQLITE_ROW)
706             prevalentDueToRedirect.insert(topFrameRedirectStatement.getColumnInt(0));
707     }
708
709     SQLiteStatement markPrevalentStatement(m_database, makeString("UPDATE ObservedDomains SET isPrevalent = 1 WHERE domainID IN (", buildList(WTF::IteratorRange<StdSet<unsigned>::iterator>(prevalentDueToRedirect.begin(), prevalentDueToRedirect.end())), ")"));
710     if (markPrevalentStatement.prepare() != SQLITE_OK
711         || markPrevalentStatement.step() != SQLITE_DONE) {
712         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::markAsPrevalentIfHasRedirectedToPrevalent failed to execute, error message: %{private}s", this, m_database.lastErrorMsg());
713         ASSERT_NOT_REACHED();
714     }
715 }
716
717 HashMap<unsigned, ResourceLoadStatisticsDatabaseStore::NotVeryPrevalentResources> ResourceLoadStatisticsDatabaseStore::findNotVeryPrevalentResources()
718 {
719     ASSERT(!RunLoop::isMain());
720
721     HashMap<unsigned, NotVeryPrevalentResources> results;
722     SQLiteStatement notVeryPrevalentResourcesStatement(m_database, "SELECT domainID, registrableDomain, isPrevalent FROM ObservedDomains WHERE isVeryPrevalent = 0"_s);
723     if (notVeryPrevalentResourcesStatement.prepare() == SQLITE_OK) {
724         while (notVeryPrevalentResourcesStatement.step() == SQLITE_ROW) {
725             unsigned key = static_cast<unsigned>(notVeryPrevalentResourcesStatement.getColumnInt(0));
726             NotVeryPrevalentResources value({ RegistrableDomain::uncheckedCreateFromRegistrableDomainString(notVeryPrevalentResourcesStatement.getColumnText(1))
727                 , notVeryPrevalentResourcesStatement.getColumnInt(2) ? ResourceLoadPrevalence::High : ResourceLoadPrevalence::Low
728                 , 0, 0, 0, 0 });
729             results.add(key, value);
730         }
731     }
732
733     StringBuilder builder;
734     for (auto value : results.keys()) {
735         if (!builder.isEmpty())
736             builder.appendLiteral(", ");
737         builder.appendNumber(value);
738     }
739
740     auto domainIDsOfInterest = builder.toString();
741
742     SQLiteStatement subresourceUnderTopFrameDomainsStatement(m_database, makeString("SELECT subresourceDomainID, COUNT(topFrameDomainID) FROM SubresourceUnderTopFrameDomains WHERE subresourceDomainID IN (", domainIDsOfInterest, ") GROUP BY subresourceDomainID"));
743     if (subresourceUnderTopFrameDomainsStatement.prepare() == SQLITE_OK) {
744         while (subresourceUnderTopFrameDomainsStatement.step() == SQLITE_ROW) {
745             unsigned domainID = static_cast<unsigned>(subresourceUnderTopFrameDomainsStatement.getColumnInt(0));
746             auto result = results.find(domainID);
747             if (result != results.end())
748                 result->value.subresourceUnderTopFrameDomainsCount = static_cast<unsigned>(subresourceUnderTopFrameDomainsStatement.getColumnInt(1));
749         }
750     }
751
752     SQLiteStatement subresourceUniqueRedirectsToCountStatement(m_database, makeString("SELECT subresourceDomainID, COUNT(toDomainID) FROM SubresourceUniqueRedirectsTo WHERE subresourceDomainID IN (", domainIDsOfInterest, ") GROUP BY subresourceDomainID"));
753     if (subresourceUniqueRedirectsToCountStatement.prepare() == SQLITE_OK) {
754         while (subresourceUniqueRedirectsToCountStatement.step() == SQLITE_ROW) {
755             unsigned domainID = static_cast<unsigned>(subresourceUniqueRedirectsToCountStatement.getColumnInt(0));
756             auto result = results.find(domainID);
757             if (result != results.end())
758                 result->value.subresourceUniqueRedirectsToCount = static_cast<unsigned>(subresourceUniqueRedirectsToCountStatement.getColumnInt(1));
759         }
760     }
761
762     SQLiteStatement subframeUnderTopFrameDomainsCountStatement(m_database, makeString("SELECT subframeDomainID, COUNT(topFrameDomainID) FROM SubframeUnderTopFrameDomains WHERE subframeDomainID IN (", domainIDsOfInterest, ") GROUP BY subframeDomainID"));
763     if (subframeUnderTopFrameDomainsCountStatement.prepare() == SQLITE_OK) {
764         while (subframeUnderTopFrameDomainsCountStatement.step() == SQLITE_ROW) {
765             unsigned domainID = static_cast<unsigned>(subframeUnderTopFrameDomainsCountStatement.getColumnInt(0));
766             auto result = results.find(domainID);
767             if (result != results.end())
768                 result->value.subframeUnderTopFrameDomainsCount = static_cast<unsigned>(subframeUnderTopFrameDomainsCountStatement.getColumnInt(1));
769         }
770     }
771
772     SQLiteStatement topFrameUniqueRedirectsToCountStatement(m_database, makeString("SELECT sourceDomainID, COUNT(toDomainID) FROM TopFrameUniqueRedirectsTo WHERE sourceDomainID IN (", domainIDsOfInterest, ") GROUP BY sourceDomainID"));
773     if (topFrameUniqueRedirectsToCountStatement.prepare() == SQLITE_OK) {
774         while (topFrameUniqueRedirectsToCountStatement.step() == SQLITE_ROW) {
775             unsigned domainID = static_cast<unsigned>(topFrameUniqueRedirectsToCountStatement.getColumnInt(0));
776             auto result = results.find(domainID);
777             if (result != results.end())
778                 result->value.topFrameUniqueRedirectsToCount = static_cast<unsigned>(topFrameUniqueRedirectsToCountStatement.getColumnInt(1));
779         }
780     }
781
782     return results;
783 }
784
785 void ResourceLoadStatisticsDatabaseStore::reclassifyResources()
786 {
787     ASSERT(!RunLoop::isMain());
788
789     auto notVeryPrevalentResources = findNotVeryPrevalentResources();
790
791     for (auto& resourceStatistic : notVeryPrevalentResources.values()) {
792         if (shouldSkip(resourceStatistic.registrableDomain))
793             continue;
794
795         auto newPrevalence = classifier().calculateResourcePrevalence(resourceStatistic.subresourceUnderTopFrameDomainsCount, resourceStatistic.subresourceUniqueRedirectsToCount, resourceStatistic.subframeUnderTopFrameDomainsCount, resourceStatistic.topFrameUniqueRedirectsToCount, resourceStatistic.prevalence);
796         if (newPrevalence != resourceStatistic.prevalence)
797             setPrevalentResource(resourceStatistic.registrableDomain, newPrevalence);
798     }
799 }
800
801 void ResourceLoadStatisticsDatabaseStore::classifyPrevalentResources()
802 {
803     ASSERT(!RunLoop::isMain());
804     ensurePrevalentResourcesForDebugMode();
805     markAsPrevalentIfHasRedirectedToPrevalent();
806     reclassifyResources();
807 }
808
809 void ResourceLoadStatisticsDatabaseStore::syncStorageIfNeeded()
810 {
811     ASSERT(!RunLoop::isMain());
812     m_database.runVacuumCommand();
813 }
814
815 void ResourceLoadStatisticsDatabaseStore::syncStorageImmediately()
816 {
817     ASSERT(!RunLoop::isMain());
818     m_database.runVacuumCommand();
819 }
820
821 void ResourceLoadStatisticsDatabaseStore::hasStorageAccess(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain, Optional<FrameIdentifier> frameID, PageIdentifier pageID, CompletionHandler<void(bool)>&& completionHandler)
822 {
823     ASSERT(!RunLoop::isMain());
824
825     ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
826
827     switch (cookieTreatmentForOrigin(subFrameDomain)) {
828     case CookieTreatmentResult::BlockAndPurge:
829         completionHandler(false);
830         return;
831     case CookieTreatmentResult::Allow:
832         // We should only return true if the context has asked for and been granted access.
833         completionHandler(false);
834         return;
835     case CookieTreatmentResult::BlockAndKeep:
836         // Do nothing. The below dispatch will complete the task.
837         break;
838     };
839
840     RunLoop::main().dispatch([store = makeRef(store()), subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, completionHandler = WTFMove(completionHandler)]() mutable {
841         store->callHasStorageAccessForFrameHandler(subFrameDomain, topFrameDomain, frameID.value(), pageID, [store = store.copyRef(), completionHandler = WTFMove(completionHandler)](bool result) mutable {
842             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler), result] () mutable {
843                 completionHandler(result);
844             });
845         });
846     });
847 }
848
849 void ResourceLoadStatisticsDatabaseStore::requestStorageAccess(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, FrameIdentifier frameID, PageIdentifier pageID, CompletionHandler<void(StorageAccessStatus)>&& completionHandler)
850 {
851     ASSERT(!RunLoop::isMain());
852
853     auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
854
855     switch (cookieTreatmentForOrigin(subFrameDomain)) {
856     case CookieTreatmentResult::BlockAndPurge: {
857         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "Cannot grant storage access to %{private}s since its cookies are blocked in third-party contexts and it has not received user interaction as first-party.", subFrameDomain.string().utf8().data());
858         completionHandler(StorageAccessStatus::CannotRequestAccess);
859         }
860         return;
861     case CookieTreatmentResult::BlockAndKeep: {
862         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "No need to grant storage access to %{private}s since its cookies are not blocked in third-party contexts.", subFrameDomain.string().utf8().data());
863         completionHandler(StorageAccessStatus::HasAccess);
864         }
865         return;
866     case CookieTreatmentResult::Allow:
867         // Do nothing
868         break;
869     };
870
871     auto userWasPromptedEarlier = hasUserGrantedStorageAccessThroughPrompt(subFrameStatus.second, topFrameDomain);
872     if (userWasPromptedEarlier == StorageAccessPromptWasShown::No) {
873         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "About to ask the user whether they want to grant storage access to %{private}s under %{private}s or not.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
874         completionHandler(StorageAccessStatus::RequiresUserPrompt);
875         return;
876     }
877
878     if (userWasPromptedEarlier == StorageAccessPromptWasShown::Yes)
879         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "Storage access was granted to %{private}s under %{private}s.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
880
881     SQLiteStatement incrementStorageAccess(m_database, "UPDATE ObservedDomains SET timesAccessedAsFirstPartyDueToStorageAccessAPI = timesAccessedAsFirstPartyDueToStorageAccessAPI + 1 WHERE domainID = ?");
882     if (incrementStorageAccess.prepare() != SQLITE_OK
883         || incrementStorageAccess.bindInt(1, subFrameStatus.second) != SQLITE_OK
884         || incrementStorageAccess.step() != SQLITE_DONE) {
885         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::requestStorageAccess failed, error message: %{private}s", this, m_database.lastErrorMsg());
886         ASSERT_NOT_REACHED();
887         return;
888     }
889     
890     grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, userWasPromptedEarlier, [completionHandler = WTFMove(completionHandler)] (StorageAccessWasGranted wasGranted) mutable {
891         completionHandler(wasGranted == StorageAccessWasGranted::Yes ? StorageAccessStatus::HasAccess : StorageAccessStatus::CannotRequestAccess);
892     });
893 }
894
895 void ResourceLoadStatisticsDatabaseStore::requestStorageAccessUnderOpener(DomainInNeedOfStorageAccess&& domainInNeedOfStorageAccess, PageIdentifier openerPageID, OpenerDomain&& openerDomain)
896 {
897     ASSERT(domainInNeedOfStorageAccess != openerDomain);
898     ASSERT(!RunLoop::isMain());
899
900     if (domainInNeedOfStorageAccess == openerDomain)
901         return;
902
903     ensureResourceStatisticsForRegistrableDomain(domainInNeedOfStorageAccess);
904
905     if (cookieTreatmentForOrigin(domainInNeedOfStorageAccess) != CookieTreatmentResult::Allow)
906         return;
907
908     RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "[Temporary combatibility fix] Storage access was granted for %{private}s under opener page from %{private}s, with user interaction in the opened window.", domainInNeedOfStorageAccess.string().utf8().data(), openerDomain.string().utf8().data());
909     grantStorageAccessInternal(WTFMove(domainInNeedOfStorageAccess), WTFMove(openerDomain), WTF::nullopt, openerPageID, StorageAccessPromptWasShown::No, [](StorageAccessWasGranted) { });
910 }
911
912 void ResourceLoadStatisticsDatabaseStore::grantStorageAccess(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, FrameIdentifier frameID, PageIdentifier pageID, StorageAccessPromptWasShown promptWasShown, CompletionHandler<void(StorageAccessWasGranted)>&& completionHandler)
913 {
914     ASSERT(!RunLoop::isMain());
915
916     if (promptWasShown == StorageAccessPromptWasShown::Yes) {
917         auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
918         ASSERT(subFrameStatus.first == AddedRecord::No);
919         ASSERT(hasHadUserInteraction(subFrameDomain, OperatingDatesWindow::Long));
920         insertDomainRelationship(m_storageAccessUnderTopFrameDomainsStatement, subFrameStatus.second, topFrameDomain);
921     }
922
923     grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, promptWasShown, WTFMove(completionHandler));
924 }
925
926 void ResourceLoadStatisticsDatabaseStore::grantStorageAccessInternal(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, Optional<FrameIdentifier> frameID, PageIdentifier pageID, StorageAccessPromptWasShown promptWasShownNowOrEarlier, CompletionHandler<void(StorageAccessWasGranted)>&& completionHandler)
927 {
928     ASSERT(!RunLoop::isMain());
929
930     if (subFrameDomain == topFrameDomain) {
931         completionHandler(StorageAccessWasGranted::Yes);
932         return;
933     }
934
935     if (promptWasShownNowOrEarlier == StorageAccessPromptWasShown::Yes) {
936 #ifndef NDEBUG
937         auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
938         ASSERT(subFrameStatus.first == AddedRecord::No);
939         ASSERT(hasHadUserInteraction(subFrameDomain, OperatingDatesWindow::Long));
940         ASSERT(hasUserGrantedStorageAccessThroughPrompt(subFrameStatus.second, topFrameDomain) == StorageAccessPromptWasShown::Yes);
941 #endif
942         setUserInteraction(subFrameDomain, true, WallTime::now());
943     }
944
945     RunLoop::main().dispatch([subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, store = makeRef(store()), completionHandler = WTFMove(completionHandler)]() mutable {
946         store->callGrantStorageAccessHandler(subFrameDomain, topFrameDomain, frameID, pageID, [completionHandler = WTFMove(completionHandler), store = store.copyRef()](StorageAccessWasGranted wasGranted) mutable {
947             store->statisticsQueue().dispatch([wasGranted, completionHandler = WTFMove(completionHandler)] () mutable {
948                 completionHandler(wasGranted);
949             });
950         });
951     });
952
953 }
954
955 void ResourceLoadStatisticsDatabaseStore::grandfatherDataForDomains(const HashSet<RegistrableDomain>& domains)
956 {
957     ASSERT(!RunLoop::isMain());
958
959     for (auto& registrableDomain : domains)
960         ensureResourceStatisticsForRegistrableDomain(registrableDomain);
961
962     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET grandfathered = 1 WHERE registrableDomain IN (", domainsToString(domains), ")"));
963     if (domainsToUpdateStatement.prepare() != SQLITE_OK
964         || domainsToUpdateStatement.step() != SQLITE_DONE) {
965         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::grandfatherDataForDomains failed, error message: %{private}s", this, m_database.lastErrorMsg());
966         ASSERT_NOT_REACHED();
967     }
968 }
969
970 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::ensurePrevalentResourcesForDebugMode()
971 {
972     ASSERT(!RunLoop::isMain());
973
974     if (!debugModeEnabled())
975         return { };
976
977     Vector<RegistrableDomain> primaryDomainsToBlock;
978     primaryDomainsToBlock.reserveInitialCapacity(2);
979
980     ensureResourceStatisticsForRegistrableDomain(debugStaticPrevalentResource());
981     setPrevalentResource(debugStaticPrevalentResource(), ResourceLoadPrevalence::High);
982     primaryDomainsToBlock.uncheckedAppend(debugStaticPrevalentResource());
983
984     if (!debugManualPrevalentResource().isEmpty()) {
985         ensureResourceStatisticsForRegistrableDomain(debugManualPrevalentResource());
986         setPrevalentResource(debugManualPrevalentResource(), ResourceLoadPrevalence::High);
987         primaryDomainsToBlock.uncheckedAppend(debugManualPrevalentResource());
988         RELEASE_LOG_INFO(ITPDebug, "Did set %{private}s as prevalent resource for the purposes of ITP Debug Mode.", debugManualPrevalentResource().string().utf8().data());
989     }
990
991     return primaryDomainsToBlock;
992 }
993
994 void ResourceLoadStatisticsDatabaseStore::logFrameNavigation(const RegistrableDomain& targetDomain, const RegistrableDomain& topFrameDomain, const RegistrableDomain& sourceDomain, bool isRedirect, bool isMainFrame)
995 {
996     ASSERT(!RunLoop::isMain());
997
998     bool areTargetAndTopFrameDomainsSameSite = targetDomain == topFrameDomain;
999     bool areTargetAndSourceDomainsSameSite = targetDomain == sourceDomain;
1000
1001     bool statisticsWereUpdated = false;
1002     if (!isMainFrame && !(areTargetAndTopFrameDomainsSameSite || areTargetAndSourceDomainsSameSite)) {
1003         auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
1004         updateLastSeen(targetDomain, ResourceLoadStatistics::reduceTimeResolution(WallTime::now()));
1005         ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1006         if (!relationshipExists(m_subframeUnderTopFrameDomainExists, targetResult.second, topFrameDomain)) {
1007             insertDomainRelationship(m_subframeUnderTopFrameDomains, targetResult.second, topFrameDomain);
1008             statisticsWereUpdated = true;
1009         }
1010     }
1011
1012     if (isRedirect && !areTargetAndSourceDomainsSameSite) {
1013         if (isMainFrame) {
1014             auto redirectingDomainResult = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
1015             auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
1016
1017             if (!relationshipExists(m_topFrameUniqueRedirectsToExists, redirectingDomainResult.second, targetDomain)) {
1018                 insertDomainRelationship(m_topFrameUniqueRedirectsTo, redirectingDomainResult.second, targetDomain);
1019                 statisticsWereUpdated = true;
1020             }
1021
1022             if (!relationshipExists(m_topFrameUniqueRedirectsFromExists, targetResult.second, sourceDomain)) {
1023                 insertDomainRelationship(m_topFrameUniqueRedirectsFrom, targetResult.second, sourceDomain);
1024                 statisticsWereUpdated = true;
1025             }
1026         } else {
1027             auto redirectingDomainResult = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
1028             auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
1029             if (!relationshipExists(m_subresourceUniqueRedirectsToExists, redirectingDomainResult.second, targetDomain)) {
1030                 insertDomainRelationship(m_subresourceUniqueRedirectsTo, redirectingDomainResult.second, targetDomain);
1031                 statisticsWereUpdated = true;
1032             }
1033
1034             if (!relationshipExists(m_subresourceUniqueRedirectsFromExists, targetResult.second, sourceDomain)) {
1035                 insertDomainRelationship(m_subresourceUniqueRedirectsFrom, targetResult.second, sourceDomain);
1036                 statisticsWereUpdated = true;
1037             }
1038         }
1039     }
1040
1041     if (statisticsWereUpdated)
1042         scheduleStatisticsProcessingRequestIfNecessary();
1043 }
1044
1045 void ResourceLoadStatisticsDatabaseStore::logCrossSiteLoadWithLinkDecoration(const NavigatedFromDomain& fromDomain, const NavigatedToDomain& toDomain)
1046 {
1047     ASSERT(!RunLoop::isMain());
1048     ASSERT(fromDomain != toDomain);
1049
1050     auto fromDomainResult = ensureResourceStatisticsForRegistrableDomain(fromDomain);
1051
1052     if (!relationshipExists(m_topFrameLinkDecorationsFromExists, fromDomainResult.second, toDomain)) {
1053         insertDomainRelationship(m_topFrameLinkDecorationsFrom, fromDomainResult.second, toDomain);
1054         scheduleStatisticsProcessingRequestIfNecessary();
1055     }
1056 }
1057
1058 void ResourceLoadStatisticsDatabaseStore::setUserInteraction(const RegistrableDomain& domain, bool hadUserInteraction, WallTime mostRecentInteraction)
1059 {
1060     ASSERT(!RunLoop::isMain());
1061
1062     if (m_mostRecentUserInteractionStatement.bindInt(1, hadUserInteraction) != SQLITE_OK
1063         || m_mostRecentUserInteractionStatement.bindDouble(2, mostRecentInteraction.secondsSinceEpoch().value()) != SQLITE_OK
1064         || m_mostRecentUserInteractionStatement.bindText(3, domain.string()) != SQLITE_OK
1065         || m_mostRecentUserInteractionStatement.step() != SQLITE_DONE) {
1066         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::setUserInteraction, error message: %{private}s", this, m_database.lastErrorMsg());
1067         ASSERT_NOT_REACHED();
1068         return;
1069     }
1070
1071     int resetResult = m_mostRecentUserInteractionStatement.reset();
1072     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1073 }
1074
1075 void ResourceLoadStatisticsDatabaseStore::logUserInteraction(const TopFrameDomain& domain)
1076 {
1077     ASSERT(!RunLoop::isMain());
1078
1079     ensureResourceStatisticsForRegistrableDomain(domain);
1080     setUserInteraction(domain, true, WallTime::now());
1081 }
1082
1083 void ResourceLoadStatisticsDatabaseStore::clearUserInteraction(const RegistrableDomain& domain)
1084 {
1085     ASSERT(!RunLoop::isMain());
1086
1087     auto targetResult = ensureResourceStatisticsForRegistrableDomain(domain);
1088     setUserInteraction(domain, false, { });
1089
1090     SQLiteStatement removeStorageAccess(m_database, "DELETE FROM StorageAccessUnderTopFrameDomains WHERE domainID = ?");
1091     if (removeStorageAccess.prepare() != SQLITE_OK
1092         || removeStorageAccess.bindInt(1, targetResult.second) != SQLITE_OK
1093         || removeStorageAccess.step() != SQLITE_DONE) {
1094         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::logUserInteraction failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
1095         ASSERT_NOT_REACHED();
1096     }
1097 }
1098
1099 bool ResourceLoadStatisticsDatabaseStore::hasHadUserInteraction(const RegistrableDomain& domain, OperatingDatesWindow operatingDatesWindow)
1100 {
1101     ASSERT(!RunLoop::isMain());
1102
1103     if (m_hadUserInteractionStatement.bindText(1, domain.string()) != SQLITE_OK
1104         || m_hadUserInteractionStatement.step() != SQLITE_ROW) {
1105         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_hadUserInteractionStatement failed, error message: %{private}s", this, m_database.lastErrorMsg());
1106
1107         int resetResult = m_hadUserInteractionStatement.reset();
1108         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1109         return false;
1110     }
1111
1112     bool hadUserInteraction = !!m_hadUserInteractionStatement.getColumnInt(0);
1113     if (!hadUserInteraction) {
1114         int resetResult = m_hadUserInteractionStatement.reset();
1115         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1116         return false;
1117     }
1118     
1119     WallTime mostRecentUserInteractionTime = WallTime::fromRawSeconds(m_hadUserInteractionStatement.getColumnDouble(1));
1120
1121     if (hasStatisticsExpired(mostRecentUserInteractionTime, operatingDatesWindow)) {
1122         // Drop privacy sensitive data because we no longer need it.
1123         // Set timestamp to 0 so that statistics merge will know
1124         // it has been reset as opposed to its default -1.
1125         clearUserInteraction(domain);
1126         hadUserInteraction = false;
1127     }
1128     
1129     int resetResult = m_hadUserInteractionStatement.reset();
1130     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1131
1132     return hadUserInteraction;
1133 }
1134
1135 void ResourceLoadStatisticsDatabaseStore::setPrevalentResource(const RegistrableDomain& domain, ResourceLoadPrevalence newPrevalence)
1136 {
1137     ASSERT(!RunLoop::isMain());
1138     if (shouldSkip(domain))
1139         return;
1140
1141     if (m_updatePrevalentResourceStatement.bindInt(1, 1) != SQLITE_OK
1142         || m_updatePrevalentResourceStatement.bindText(2, domain.string()) != SQLITE_OK
1143         || m_updatePrevalentResourceStatement.step() != SQLITE_DONE) {
1144         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_updatePrevalentResourceStatement failed, error message: %{private}s", this, m_database.lastErrorMsg());
1145         ASSERT_NOT_REACHED();
1146         return;
1147     }
1148     
1149     int resetResult = m_updatePrevalentResourceStatement.reset();
1150     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1151
1152     if (newPrevalence == ResourceLoadPrevalence::VeryHigh) {
1153         if (m_updateVeryPrevalentResourceStatement.bindInt(1, 1) != SQLITE_OK
1154             || m_updateVeryPrevalentResourceStatement.bindText(2, domain.string()) != SQLITE_OK
1155             || m_updateVeryPrevalentResourceStatement.step() != SQLITE_DONE) {
1156             RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_updateVeryPrevalentResourceStatement failed, error message: %{private}s", this, m_database.lastErrorMsg());
1157             ASSERT_NOT_REACHED();
1158             return;
1159         }
1160
1161         int resetResult = m_updateVeryPrevalentResourceStatement.reset();
1162         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1163     }
1164
1165     StdSet<unsigned> nonPrevalentRedirectionSources;
1166     recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(domainID(domain).value(), nonPrevalentRedirectionSources, 0);
1167     setDomainsAsPrevalent(WTFMove(nonPrevalentRedirectionSources));
1168 }
1169
1170 void ResourceLoadStatisticsDatabaseStore::setDomainsAsPrevalent(StdSet<unsigned>&& domains)
1171 {
1172     ASSERT(!RunLoop::isMain());
1173
1174     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET isPrevalent = 1 WHERE domainID IN (", buildList(WTF::IteratorRange<StdSet<unsigned>::iterator>(domains.begin(), domains.end())), ")"));
1175     if (domainsToUpdateStatement.prepare() != SQLITE_OK
1176         || domainsToUpdateStatement.step() != SQLITE_DONE) {
1177         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::setDomainsAsPrevalent failed, error message: %{private}s", this, m_database.lastErrorMsg());
1178         ASSERT_NOT_REACHED();
1179         return;
1180     }
1181 }
1182
1183 void ResourceLoadStatisticsDatabaseStore::dumpResourceLoadStatistics(CompletionHandler<void(const String&)>&& completionHandler)
1184 {
1185     ASSERT(!RunLoop::isMain());
1186
1187     // FIXME(195088): Implement SQLite-based dumping routines.
1188     ASSERT_NOT_REACHED();
1189     completionHandler({ });
1190 }
1191
1192 bool ResourceLoadStatisticsDatabaseStore::predicateValueForDomain(WebCore::SQLiteStatement& predicateStatement, const RegistrableDomain& domain) const
1193 {
1194     ASSERT(!RunLoop::isMain());
1195     
1196     if (predicateStatement.bindText(1, domain.string()) != SQLITE_OK
1197         || predicateStatement.step() != SQLITE_ROW) {
1198
1199         int resetResult = predicateStatement.reset();
1200         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1201
1202         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::predicateValueForDomain failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
1203         return false;
1204     }
1205
1206     bool result = !!predicateStatement.getColumnInt(0);
1207     int resetResult = predicateStatement.reset();
1208     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1209     return result;
1210 }
1211
1212 bool ResourceLoadStatisticsDatabaseStore::isPrevalentResource(const RegistrableDomain& domain) const
1213 {
1214     ASSERT(!RunLoop::isMain());
1215
1216     if (shouldSkip(domain))
1217         return false;
1218
1219     return predicateValueForDomain(m_isPrevalentResourceStatement, domain);
1220 }
1221
1222 bool ResourceLoadStatisticsDatabaseStore::isVeryPrevalentResource(const RegistrableDomain& domain) const
1223 {
1224     ASSERT(!RunLoop::isMain());
1225
1226     if (shouldSkip(domain))
1227         return false;
1228
1229     return predicateValueForDomain(m_isVeryPrevalentResourceStatement, domain);
1230 }
1231
1232 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsSubresourceUnder(const SubResourceDomain& subresourceDomain, const TopFrameDomain& topFrameDomain) const
1233 {
1234     ASSERT(!RunLoop::isMain());
1235
1236     return relationshipExists(m_subresourceUnderTopFrameDomainExists, domainID(subresourceDomain), topFrameDomain);
1237 }
1238
1239 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsSubFrameUnder(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain) const
1240 {
1241     ASSERT(!RunLoop::isMain());
1242
1243     return relationshipExists(m_subframeUnderTopFrameDomainExists, domainID(subFrameDomain), topFrameDomain);
1244 }
1245
1246 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsRedirectingTo(const RedirectedFromDomain& redirectedFromDomain, const RedirectedToDomain& redirectedToDomain) const
1247 {
1248     ASSERT(!RunLoop::isMain());
1249
1250     return relationshipExists(m_subresourceUniqueRedirectsToExists, domainID(redirectedFromDomain), redirectedToDomain);
1251 }
1252
1253 void ResourceLoadStatisticsDatabaseStore::clearPrevalentResource(const RegistrableDomain& domain)
1254 {
1255     ASSERT(!RunLoop::isMain());
1256
1257     ensureResourceStatisticsForRegistrableDomain(domain);
1258     
1259     if (m_clearPrevalentResourceStatement.bindText(1, domain.string()) != SQLITE_OK
1260         || m_clearPrevalentResourceStatement.step() != SQLITE_DONE) {
1261         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::clearPrevalentResource, error message: %{private}s", this, m_database.lastErrorMsg());
1262         ASSERT_NOT_REACHED();
1263         return;
1264     }
1265     
1266     int resetResult = m_clearPrevalentResourceStatement.reset();
1267     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1268 }
1269
1270 void ResourceLoadStatisticsDatabaseStore::setGrandfathered(const RegistrableDomain& domain, bool value)
1271 {
1272     ASSERT(!RunLoop::isMain());
1273
1274     ensureResourceStatisticsForRegistrableDomain(domain);
1275     
1276     if (m_updateGrandfatheredStatement.bindInt(1, value) != SQLITE_OK
1277         || m_updateGrandfatheredStatement.bindText(2, domain.string()) != SQLITE_OK
1278         || m_updateGrandfatheredStatement.step() != SQLITE_DONE) {
1279         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::setGrandfathered failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
1280         ASSERT_NOT_REACHED();
1281         return;
1282     }
1283     
1284     int resetResult = m_updateGrandfatheredStatement.reset();
1285     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1286 }
1287
1288 bool ResourceLoadStatisticsDatabaseStore::isGrandfathered(const RegistrableDomain& domain) const
1289 {
1290     ASSERT(!RunLoop::isMain());
1291
1292     return predicateValueForDomain(m_isGrandfatheredStatement, domain);
1293 }
1294
1295 void ResourceLoadStatisticsDatabaseStore::setSubframeUnderTopFrameDomain(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain)
1296 {
1297     ASSERT(!RunLoop::isMain());
1298
1299     auto result = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
1300
1301     // For consistency, make sure we also have a statistics entry for the top frame domain.
1302     ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1303
1304     insertDomainRelationship(m_subframeUnderTopFrameDomains, result.second, topFrameDomain);
1305 }
1306
1307 void ResourceLoadStatisticsDatabaseStore::setSubresourceUnderTopFrameDomain(const SubResourceDomain& subresourceDomain, const TopFrameDomain& topFrameDomain)
1308 {
1309     ASSERT(!RunLoop::isMain());
1310
1311     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1312
1313     // For consistency, make sure we also have a statistics entry for the top frame domain.
1314     ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1315
1316     insertDomainRelationship(m_subresourceUnderTopFrameDomains, result.second, topFrameDomain);
1317 }
1318
1319 void ResourceLoadStatisticsDatabaseStore::setSubresourceUniqueRedirectTo(const SubResourceDomain& subresourceDomain, const RedirectDomain& redirectDomain)
1320 {
1321     ASSERT(!RunLoop::isMain());
1322
1323     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1324
1325     // For consistency, make sure we also have a statistics entry for the redirect domain.
1326     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1327
1328     insertDomainRelationship(m_subresourceUniqueRedirectsTo, result.second, redirectDomain);
1329 }
1330
1331 void ResourceLoadStatisticsDatabaseStore::setSubresourceUniqueRedirectFrom(const SubResourceDomain& subresourceDomain, const RedirectDomain& redirectDomain)
1332 {
1333     ASSERT(!RunLoop::isMain());
1334
1335     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1336
1337     // For consistency, make sure we also have a statistics entry for the redirect domain.
1338     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1339
1340     insertDomainRelationship(m_subresourceUniqueRedirectsFrom, result.second, redirectDomain);
1341 }
1342
1343 void ResourceLoadStatisticsDatabaseStore::setTopFrameUniqueRedirectTo(const TopFrameDomain& topFrameDomain, const RedirectDomain& redirectDomain)
1344 {
1345     ASSERT(!RunLoop::isMain());
1346
1347     auto result = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1348
1349     // For consistency, make sure we also have a statistics entry for the redirect domain.
1350     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1351
1352     insertDomainRelationship(m_topFrameUniqueRedirectsTo, result.second, redirectDomain);
1353 }
1354
1355 void ResourceLoadStatisticsDatabaseStore::setTopFrameUniqueRedirectFrom(const TopFrameDomain& topFrameDomain, const RedirectDomain& redirectDomain)
1356 {
1357     ASSERT(!RunLoop::isMain());
1358
1359     auto result = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1360
1361     // For consistency, make sure we also have a statistics entry for the redirect domain.
1362     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1363
1364     insertDomainRelationship(m_topFrameUniqueRedirectsFrom, result.second, redirectDomain);
1365 }
1366
1367 std::pair<ResourceLoadStatisticsDatabaseStore::AddedRecord, unsigned> ResourceLoadStatisticsDatabaseStore::ensureResourceStatisticsForRegistrableDomain(const RegistrableDomain& domain)
1368 {
1369     ASSERT(!RunLoop::isMain());
1370
1371     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK) {
1372         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::ensureResourceStatisticsForRegistrableDomain failed, error message: %{private}s", this, m_database.lastErrorMsg());
1373         ASSERT_NOT_REACHED();
1374         return { AddedRecord::No, 0 };
1375     }
1376     
1377     if (m_domainIDFromStringStatement.step() == SQLITE_ROW) {
1378         unsigned domainID = m_domainIDFromStringStatement.getColumnInt(0);
1379
1380         int resetResult = m_domainIDFromStringStatement.reset();
1381         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1382         return { AddedRecord::No, domainID };
1383     }
1384
1385     int resetResult = m_domainIDFromStringStatement.reset();
1386     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1387
1388     ResourceLoadStatistics newObservation(domain);
1389     insertObservedDomain(newObservation);
1390
1391     return { AddedRecord::Yes, domainID(domain).value() };
1392 }
1393
1394 void ResourceLoadStatisticsDatabaseStore::clearDatabaseContents()
1395 {
1396     m_database.clearAllTables();
1397
1398     if (!createSchema()) {
1399         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::clearDatabaseContents failed, error message: %{private}s, database path: %{private}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
1400         ASSERT_NOT_REACHED();
1401         return;
1402     }
1403 }
1404
1405 void ResourceLoadStatisticsDatabaseStore::clear(CompletionHandler<void()>&& completionHandler)
1406 {
1407     ASSERT(!RunLoop::isMain());
1408
1409     clearDatabaseContents();
1410     clearOperatingDates();
1411
1412     auto callbackAggregator = CallbackAggregator::create(WTFMove(completionHandler));
1413
1414     removeAllStorageAccess([callbackAggregator = callbackAggregator.copyRef()] { });
1415
1416     auto registrableDomainsToBlockAndDeleteCookiesFor = ensurePrevalentResourcesForDebugMode();
1417     RegistrableDomainsToBlockCookiesFor domainsToBlock { registrableDomainsToBlockAndDeleteCookiesFor, { } };
1418     updateCookieBlockingForDomains(domainsToBlock, [callbackAggregator = callbackAggregator.copyRef()] { });
1419 }
1420
1421 ResourceLoadStatisticsDatabaseStore::CookieTreatmentResult ResourceLoadStatisticsDatabaseStore::cookieTreatmentForOrigin(const RegistrableDomain& domain) const
1422 {
1423     ASSERT(!RunLoop::isMain());
1424
1425     SQLiteStatement statement(m_database, "SELECT isPrevalent, hadUserInteraction FROM ObservedDomains WHERE registrableDomain = ?");
1426     if (statement.prepare() != SQLITE_OK
1427         || statement.bindText(1, domain.string()) != SQLITE_OK) {
1428         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::cookieTreatmentForOrigin failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
1429         ASSERT_NOT_REACHED();
1430     }
1431     
1432     if (statement.step() != SQLITE_ROW)
1433         return CookieTreatmentResult::Allow;
1434     
1435     bool isPrevalent = !!statement.getColumnInt(0);
1436     if (!isPrevalent)
1437         return CookieTreatmentResult::Allow;
1438
1439     bool hadUserInteraction = statement.getColumnInt(1) ? true : false;
1440     return hadUserInteraction ? CookieTreatmentResult::BlockAndKeep : CookieTreatmentResult::BlockAndPurge;
1441 }
1442     
1443 StorageAccessPromptWasShown ResourceLoadStatisticsDatabaseStore::hasUserGrantedStorageAccessThroughPrompt(unsigned requestingDomainID, const RegistrableDomain& firstPartyDomain) const
1444 {
1445     ASSERT(!RunLoop::isMain());
1446
1447     auto firstPartyPrimaryDomainID = domainID(firstPartyDomain).value();
1448
1449     SQLiteStatement statement(m_database, "SELECT COUNT(*) FROM StorageAccessUnderTopFrameDomains WHERE domainID = ? AND topLevelDomainID = ?");
1450     if (statement.prepare() != SQLITE_OK
1451         || statement.bindInt(1, requestingDomainID) != SQLITE_OK
1452         || statement.bindInt(2, firstPartyPrimaryDomainID) != SQLITE_OK
1453         || statement.step() != SQLITE_ROW)
1454         return StorageAccessPromptWasShown::No;
1455
1456     return !statement.getColumnInt(0) ? StorageAccessPromptWasShown::Yes : StorageAccessPromptWasShown::No;
1457 }
1458
1459 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::domainsToBlockAndDeleteCookiesFor() const
1460 {
1461     ASSERT(!RunLoop::isMain());
1462     
1463     Vector<RegistrableDomain> results;
1464     SQLiteStatement statement(m_database, "SELECT registrableDomain FROM ObservedDomains WHERE isPrevalent = 1 AND hadUserInteraction = 0"_s);
1465     if (statement.prepare() != SQLITE_OK)
1466         return results;
1467     
1468     while (statement.step() == SQLITE_ROW)
1469         results.append(RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(0)));
1470     
1471     return results;
1472 }
1473
1474 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::domainsToBlockButKeepCookiesFor() const
1475 {
1476     ASSERT(!RunLoop::isMain());
1477
1478     Vector<RegistrableDomain> results;
1479     SQLiteStatement statement(m_database, "SELECT registrableDomain FROM ObservedDomains WHERE isPrevalent = 1 AND hadUserInteraction = 1"_s);
1480     if (statement.prepare() != SQLITE_OK)
1481         return results;
1482     
1483     while (statement.step() == SQLITE_ROW)
1484         results.append(RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(0)));
1485
1486     return results;
1487 }
1488
1489 void ResourceLoadStatisticsDatabaseStore::updateCookieBlocking(CompletionHandler<void()>&& completionHandler)
1490 {
1491     ASSERT(!RunLoop::isMain());
1492
1493     auto domainsToBlockAndDeleteCookiesFor = this->domainsToBlockAndDeleteCookiesFor();
1494     auto domainsToBlockButKeepCookiesFor = this->domainsToBlockButKeepCookiesFor();
1495
1496     if (domainsToBlockAndDeleteCookiesFor.isEmpty() && domainsToBlockButKeepCookiesFor.isEmpty()) {
1497         completionHandler();
1498         return;
1499     }
1500
1501     RegistrableDomainsToBlockCookiesFor domainsToBlock { domainsToBlockAndDeleteCookiesFor, domainsToBlockButKeepCookiesFor };
1502
1503     if (debugLoggingEnabled() && !domainsToBlockAndDeleteCookiesFor.isEmpty() && !domainsToBlockButKeepCookiesFor.isEmpty())
1504         debugLogDomainsInBatches("block", domainsToBlock);
1505
1506     RunLoop::main().dispatch([weakThis = makeWeakPtr(*this), store = makeRef(store()), domainsToBlock = crossThreadCopy(domainsToBlock), completionHandler = WTFMove(completionHandler)] () mutable {
1507         store->callUpdatePrevalentDomainsToBlockCookiesForHandler(domainsToBlock, [weakThis = WTFMove(weakThis), store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1508             store->statisticsQueue().dispatch([weakThis = WTFMove(weakThis), completionHandler = WTFMove(completionHandler)]() mutable {
1509                 completionHandler();
1510                 if (!weakThis)
1511                     return;
1512                 RELEASE_LOG_INFO_IF(weakThis->debugLoggingEnabled(), ITPDebug, "Done updating cookie blocking.");
1513             });
1514         });
1515     });
1516 }
1517
1518 Vector<ResourceLoadStatisticsDatabaseStore::PrevalentDomainData> ResourceLoadStatisticsDatabaseStore::prevalentDomains() const
1519 {
1520     ASSERT(!RunLoop::isMain());
1521     
1522     Vector<PrevalentDomainData> results;
1523     SQLiteStatement statement(m_database, "SELECT domainID, registrableDomain, mostRecentUserInteractionTime, hadUserInteraction, grandfathered FROM ObservedDomains WHERE isPrevalent = 1"_s);
1524     if (statement.prepare() != SQLITE_OK)
1525         return results;
1526     
1527     while (statement.step() == SQLITE_ROW) {
1528         results.append({ static_cast<unsigned>(statement.getColumnInt(0))
1529             , RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(1))
1530             , WallTime::fromRawSeconds(statement.getColumnDouble(2))
1531             , statement.getColumnInt(3) ? true : false
1532             , statement.getColumnInt(4) ? true : false
1533         });
1534     }
1535     
1536     return results;
1537 }
1538
1539 Vector<unsigned> ResourceLoadStatisticsDatabaseStore::findExpiredUserInteractions() const
1540 {
1541     ASSERT(!RunLoop::isMain());
1542
1543     Vector<unsigned> results;
1544     Optional<Seconds> expirationDateTime = statisticsEpirationTime();
1545     if (!expirationDateTime)
1546         return results;
1547
1548     if (m_findExpiredUserInteractionStatement.bindDouble(1, expirationDateTime.value().value()) != SQLITE_OK)
1549         return results;
1550
1551     while (m_findExpiredUserInteractionStatement.step() == SQLITE_ROW)
1552         results.append(m_findExpiredUserInteractionStatement.getColumnInt(0));
1553
1554     int resetResult = m_findExpiredUserInteractionStatement.reset();
1555     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1556
1557     return results;
1558 }
1559
1560 void ResourceLoadStatisticsDatabaseStore::clearExpiredUserInteractions()
1561 {
1562     ASSERT(!RunLoop::isMain());
1563
1564     auto expiredRecords = findExpiredUserInteractions();
1565     if (expiredRecords.isEmpty())
1566         return;
1567
1568     auto expiredRecordIDs = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(expiredRecords.begin(), expiredRecords.end()));
1569
1570     SQLiteStatement clearExpiredInteraction(m_database, makeString("UPDATE ObservedDomains SET mostRecentUserInteractionTime = 0, hadUserInteraction = 1 WHERE domainID IN (", expiredRecordIDs, ")"));
1571     if (clearExpiredInteraction.prepare() != SQLITE_OK)
1572         return;
1573
1574     SQLiteStatement removeStorageAccess(m_database, makeString("DELETE FROM StorageAccessUnderTopFrameDomains ", expiredRecordIDs, ")"));
1575     if (removeStorageAccess.prepare() != SQLITE_OK)
1576         return;
1577
1578     if (clearExpiredInteraction.step() != SQLITE_DONE
1579         || removeStorageAccess.step() != SQLITE_DONE) {
1580         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::clearExpiredUserInteractions statement(s) failed to step, error message: %{private}s", this, m_database.lastErrorMsg());
1581         ASSERT_NOT_REACHED();
1582     }
1583 }
1584
1585 void ResourceLoadStatisticsDatabaseStore::clearGrandfathering(Vector<unsigned>&& domainIDsToClear)
1586 {
1587     ASSERT(!RunLoop::isMain());
1588
1589     if (domainIDsToClear.isEmpty())
1590         return;
1591
1592     auto listToClear = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(domainIDsToClear.begin(), domainIDsToClear.end()));
1593
1594     SQLiteStatement clearGrandfatheringStatement(m_database, makeString("UPDATE ObservedDomains SET grandfathered = 0 WHERE domainID IN (", listToClear, ")"));
1595     if (clearGrandfatheringStatement.prepare() != SQLITE_OK)
1596         return;
1597     
1598     if (clearGrandfatheringStatement.step() != SQLITE_DONE) {
1599         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::clearGrandfathering failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
1600         ASSERT_NOT_REACHED();
1601     }
1602 }
1603
1604 bool ResourceLoadStatisticsDatabaseStore::hasHadUnexpiredRecentUserInteraction(const PrevalentDomainData& resourceStatistic, OperatingDatesWindow operatingDatesWindow)
1605 {
1606     if (resourceStatistic.hadUserInteraction && hasStatisticsExpired(resourceStatistic.mostRecentUserInteractionTime, operatingDatesWindow)) {
1607         clearUserInteraction(resourceStatistic.registrableDomain);
1608         return false;
1609     }
1610
1611     return resourceStatistic.hadUserInteraction;
1612 }
1613
1614 bool ResourceLoadStatisticsDatabaseStore::shouldRemoveAllWebsiteDataFor(const PrevalentDomainData& resourceStatistic, bool shouldCheckForGrandfathering)
1615 {
1616     return !hasHadUnexpiredRecentUserInteraction(resourceStatistic, OperatingDatesWindow::Long) && (!shouldCheckForGrandfathering || !resourceStatistic.grandfathered);
1617 }
1618
1619 bool ResourceLoadStatisticsDatabaseStore::shouldRemoveAllButCookiesFor(const PrevalentDomainData& resourceStatistic, bool shouldCheckForGrandfathering) const
1620 {
1621     UNUSED_PARAM(resourceStatistic);
1622     UNUSED_PARAM(shouldCheckForGrandfathering);
1623     return false;
1624 }
1625
1626 Vector<std::pair<RegistrableDomain, WebsiteDataToRemove>> ResourceLoadStatisticsDatabaseStore::registrableDomainsToRemoveWebsiteDataFor()
1627 {
1628     ASSERT(!RunLoop::isMain());
1629
1630     bool shouldCheckForGrandfathering = endOfGrandfatheringTimestamp() > WallTime::now();
1631     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && endOfGrandfatheringTimestamp();
1632
1633     if (shouldClearGrandfathering)
1634         clearEndOfGrandfatheringTimeStamp();
1635
1636     clearExpiredUserInteractions();
1637     
1638     Vector<std::pair<RegistrableDomain, WebsiteDataToRemove>> domainsToRemoveWebsiteDataFor;
1639
1640     Vector<PrevalentDomainData> prevalentDomains = this->prevalentDomains();
1641     Vector<unsigned> domainIDsToClearGrandfathering;
1642     for (auto& statistic : prevalentDomains) {
1643         if (shouldRemoveAllWebsiteDataFor(statistic, shouldCheckForGrandfathering))
1644             domainsToRemoveWebsiteDataFor.append(std::make_pair(statistic.registrableDomain, WebsiteDataToRemove::All));
1645         else if (shouldRemoveAllButCookiesFor(statistic, shouldCheckForGrandfathering))
1646             domainsToRemoveWebsiteDataFor.append(std::make_pair(statistic.registrableDomain, WebsiteDataToRemove::AllButCookies));
1647
1648         if (shouldClearGrandfathering && statistic.grandfathered)
1649             domainIDsToClearGrandfathering.append(statistic.domainID);
1650     }
1651
1652     clearGrandfathering(WTFMove(domainIDsToClearGrandfathering));
1653     
1654     return domainsToRemoveWebsiteDataFor;
1655 }
1656
1657 void ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded()
1658 {
1659     ASSERT(!RunLoop::isMain());
1660
1661     unsigned count = 0;
1662     if (m_observedDomainCount.step() == SQLITE_ROW)
1663         count = m_observedDomainCount.getColumnInt(0);
1664
1665     int resetResult = m_observedDomainCount.reset();
1666     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1667
1668     if (count <= parameters().maxStatisticsEntries)
1669         return;
1670
1671     ASSERT(parameters().pruneEntriesDownTo <= parameters().maxStatisticsEntries);
1672
1673     size_t countLeftToPrune = count - parameters().pruneEntriesDownTo;
1674     ASSERT(countLeftToPrune);
1675
1676     SQLiteStatement recordsToPrune(m_database, "SELECT domainID FROM ObservedDomains ORDER BY hadUserInteraction, isPrevalent, lastSeen LIMIT ?");
1677     if (recordsToPrune.prepare() != SQLITE_OK
1678         || recordsToPrune.bindInt(1, countLeftToPrune) != SQLITE_OK) {
1679         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded failed, error message: %{private}s", this, m_database.lastErrorMsg());
1680         ASSERT_NOT_REACHED();
1681         return;
1682     }
1683
1684     Vector<unsigned> entriesToPrune;
1685     while (recordsToPrune.step() == SQLITE_ROW)
1686         entriesToPrune.append(recordsToPrune.getColumnInt(0));
1687
1688     auto listToPrune = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(entriesToPrune.begin(), entriesToPrune.end()));
1689
1690     SQLiteStatement pruneCommand(m_database, makeString("DELETE from ObservedDomains WHERE domainID IN (", listToPrune, ")"));
1691     if (pruneCommand.prepare() != SQLITE_OK
1692         || pruneCommand.step() != SQLITE_DONE) {
1693         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded failed, error message: %{private}s", this, m_database.lastErrorMsg());
1694         ASSERT_NOT_REACHED();
1695         return;
1696     }
1697 }
1698
1699 void ResourceLoadStatisticsDatabaseStore::updateLastSeen(const RegistrableDomain& domain, WallTime lastSeen)
1700 {
1701     ASSERT(!RunLoop::isMain());
1702
1703     if (m_updateLastSeenStatement.bindDouble(1, lastSeen.secondsSinceEpoch().value()) != SQLITE_OK
1704         || m_updateLastSeenStatement.bindText(2, domain.string()) != SQLITE_OK
1705         || m_updateLastSeenStatement.step() != SQLITE_DONE) {
1706         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::updateLastSeen failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
1707         ASSERT_NOT_REACHED();
1708         return;
1709     }
1710     
1711     int resetResult = m_updateLastSeenStatement.reset();
1712     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1713 }
1714
1715 void ResourceLoadStatisticsDatabaseStore::setLastSeen(const RegistrableDomain& domain, Seconds seconds)
1716 {
1717     ASSERT(!RunLoop::isMain());
1718
1719     ensureResourceStatisticsForRegistrableDomain(domain);
1720     updateLastSeen(domain, WallTime::fromRawSeconds(seconds.seconds()));
1721 }
1722
1723 void ResourceLoadStatisticsDatabaseStore::setPrevalentResource(const RegistrableDomain& domain)
1724 {
1725     ASSERT(!RunLoop::isMain());
1726
1727     if (shouldSkip(domain))
1728         return;
1729
1730     ensureResourceStatisticsForRegistrableDomain(domain);
1731     setPrevalentResource(domain, ResourceLoadPrevalence::High);
1732 }
1733
1734 void ResourceLoadStatisticsDatabaseStore::setVeryPrevalentResource(const RegistrableDomain& domain)
1735 {
1736     ASSERT(!RunLoop::isMain());
1737
1738     if (shouldSkip(domain))
1739         return;
1740     
1741     ensureResourceStatisticsForRegistrableDomain(domain);
1742     setPrevalentResource(domain, ResourceLoadPrevalence::VeryHigh);
1743 }
1744
1745 void ResourceLoadStatisticsDatabaseStore::updateDataRecordsRemoved(const RegistrableDomain& domain, int value)
1746 {
1747     ASSERT(!RunLoop::isMain());
1748
1749     if (m_updateDataRecordsRemovedStatement.bindInt(1, value) != SQLITE_OK
1750         || m_updateDataRecordsRemovedStatement.bindText(2, domain.string()) != SQLITE_OK
1751         || m_updateDataRecordsRemovedStatement.step() != SQLITE_DONE) {
1752         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::updateDataRecordsRemoved failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
1753         ASSERT_NOT_REACHED();
1754         return;
1755     }
1756
1757     int resetResult = m_updateDataRecordsRemovedStatement.reset();
1758     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1759 }
1760
1761 } // namespace WebKit
1762
1763 #endif