Domain relationships in the ITP Database should be inserted in a single query and...
[WebKit-https.git] / Source / WebKit / NetworkProcess / Classifier / ResourceLoadStatisticsDatabaseStore.cpp
1 /*
2  * Copyright (C) 2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ResourceLoadStatisticsDatabaseStore.h"
28
29 #if ENABLE(RESOURCE_LOAD_STATISTICS)
30
31 #include "Logging.h"
32 #include "NetworkSession.h"
33 #include "PluginProcessManager.h"
34 #include "PluginProcessProxy.h"
35 #include "ResourceLoadStatisticsMemoryStore.h"
36 #include "StorageAccessStatus.h"
37 #include "WebProcessProxy.h"
38 #include "WebResourceLoadStatisticsTelemetry.h"
39 #include "WebsiteDataStore.h"
40 #include <WebCore/DocumentStorageAccess.h>
41 #include <WebCore/KeyedCoding.h>
42 #include <WebCore/NetworkStorageSession.h>
43 #include <WebCore/ResourceLoadStatistics.h>
44 #include <WebCore/SQLiteDatabase.h>
45 #include <WebCore/SQLiteStatement.h>
46 #include <WebCore/UserGestureIndicator.h>
47 #include <wtf/CallbackAggregator.h>
48 #include <wtf/DateMath.h>
49 #include <wtf/HashMap.h>
50 #include <wtf/MathExtras.h>
51 #include <wtf/StdSet.h>
52 #include <wtf/text/StringBuilder.h>
53
54 namespace WebKit {
55 using namespace WebCore;
56
57 #if PLATFORM(COCOA)
58 #define RELEASE_LOG_IF_ALLOWED(sessionID, fmt, ...) RELEASE_LOG_IF(sessionID.isAlwaysOnLoggingAllowed(), Network, "%p - ResourceLoadStatisticsDatabaseStore::" fmt, this, ##__VA_ARGS__)
59 #define RELEASE_LOG_ERROR_IF_ALLOWED(sessionID, fmt, ...) RELEASE_LOG_ERROR_IF(sessionID.isAlwaysOnLoggingAllowed(), Network, "%p - ResourceLoadStatisticsDatabaseStore::" fmt, this, ##__VA_ARGS__)
60 #else
61 #define RELEASE_LOG_IF_ALLOWED(sessionID, fmt, ...)  ((void)0)
62 #define RELEASE_LOG_ERROR_IF_ALLOWED(sessionID, fmt, ...)  ((void)0)
63 #endif
64
65 // COUNT Queries
66 constexpr auto observedDomainCountQuery = "SELECT COUNT(*) FROM ObservedDomains"_s;
67 constexpr auto countSubframeUnderTopFrameQuery = "SELECT COUNT(*) FROM SubframeUnderTopFrameDomains WHERE subFrameDomainID = ? AND topFrameDomainID = ?;"_s;
68 constexpr auto countSubresourceUnderTopFrameQuery = "SELECT COUNT(*) FROM SubresourceUnderTopFrameDomains WHERE subresourceDomainID = ? AND topFrameDomainID = ?;"_s;
69 constexpr auto countSubresourceUniqueRedirectsToQuery = "SELECT COUNT(*) FROM SubresourceUniqueRedirectsTo WHERE subresourceDomainID = ? AND toDomainID = ?;"_s;
70
71 // INSERT Queries
72 constexpr auto insertObservedDomainQuery = "INSERT INTO ObservedDomains (registrableDomain, lastSeen, hadUserInteraction,"
73     "mostRecentUserInteractionTime, grandfathered, isPrevalent, isVeryPrevalent, dataRecordsRemoved, timesAccessedAsFirstPartyDueToUserInteraction,"
74     "timesAccessedAsFirstPartyDueToStorageAccessAPI) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"_s;
75 constexpr auto insertTopLevelDomainQuery = "INSERT INTO TopLevelDomains VALUES (?)"_s;
76 constexpr auto storageAccessUnderTopFrameDomainsQuery = "INSERT OR IGNORE INTO StorageAccessUnderTopFrameDomains (domainID, topLevelDomainID) SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain in ( "_s;
77 constexpr auto topFrameUniqueRedirectsToQuery = "INSERT OR IGNORE into TopFrameUniqueRedirectsTo (sourceDomainID, toDomainID) SELECT ?, domainID FROM ObservedDomains where registrableDomain in ( "_s;
78 constexpr auto subframeUnderTopFrameDomainsQuery = "INSERT OR IGNORE into SubframeUnderTopFrameDomains (subFrameDomainID, topFrameDomainID) SELECT ?, domainID FROM ObservedDomains where registrableDomain in ( "_s;
79 constexpr auto topFrameUniqueRedirectsFromQuery = "INSERT OR IGNORE INTO TopFrameUniqueRedirectsFrom (targetDomainID, fromDomainID) SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain in ( "_s;
80 constexpr auto topFrameLinkDecorationsFromQuery = "INSERT OR IGNORE INTO TopFrameLinkDecorationsFrom (fromDomainID, toDomainID) SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain in ( "_s;
81 constexpr auto subresourceUnderTopFrameDomainsQuery = "INSERT OR IGNORE INTO SubresourceUnderTopFrameDomains (subresourceDomainID, topFrameDomainID) SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain in ( "_s;
82 constexpr auto subresourceUniqueRedirectsToQuery = "INSERT OR IGNORE INTO SubresourceUniqueRedirectsTo (subresourceDomainID, toDomainID) SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain in ( "_s;
83 constexpr auto subresourceUniqueRedirectsFromQuery = "INSERT OR IGNORE INTO SubresourceUniqueRedirectsFrom (subresourceDomainID, fromDomainID) SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain in ( "_s;
84
85 // EXISTS Queries
86 constexpr auto subframeUnderTopFrameDomainExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubframeUnderTopFrameDomains WHERE subFrameDomainID = ? "
87     "AND topFrameDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
88 constexpr auto subresourceUnderTopFrameDomainExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUnderTopFrameDomains "
89     "WHERE subresourceDomainID = ? AND topFrameDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
90 constexpr auto subresourceUniqueRedirectsToExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUniqueRedirectsTo WHERE subresourceDomainID = ? "
91     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
92 constexpr auto topFrameLinkDecorationsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameLinkDecorationsFrom WHERE fromDomainID = ? "
93 "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
94
95 // UPDATE Queries
96 constexpr auto mostRecentUserInteractionQuery = "UPDATE ObservedDomains SET hadUserInteraction = ?, mostRecentUserInteractionTime = ? "
97     "WHERE registrableDomain = ?"_s;
98 constexpr auto updateLastSeenQuery = "UPDATE ObservedDomains SET lastSeen = ? WHERE registrableDomain = ?"_s;
99 constexpr auto updateDataRecordsRemovedQuery = "UPDATE ObservedDomains SET dataRecordsRemoved = ? WHERE registrableDomain = ?"_s;
100 constexpr auto updatePrevalentResourceQuery = "UPDATE ObservedDomains SET isPrevalent = ? WHERE registrableDomain = ?"_s;
101 constexpr auto updateVeryPrevalentResourceQuery = "UPDATE ObservedDomains SET isVeryPrevalent = ? WHERE registrableDomain = ?"_s;
102 constexpr auto clearPrevalentResourceQuery = "UPDATE ObservedDomains SET isPrevalent = 0, isVeryPrevalent = 0 WHERE registrableDomain = ?"_s;
103 constexpr auto updateGrandfatheredQuery = "UPDATE ObservedDomains SET grandfathered = ? WHERE registrableDomain = ?"_s;
104
105
106 // SELECT Queries
107 constexpr auto domainIDFromStringQuery = "SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
108 constexpr auto isPrevalentResourceQuery = "SELECT isPrevalent FROM ObservedDomains WHERE registrableDomain = ?"_s;
109 constexpr auto isVeryPrevalentResourceQuery = "SELECT isVeryPrevalent FROM ObservedDomains WHERE registrableDomain = ?"_s;
110 constexpr auto hadUserInteractionQuery = "SELECT hadUserInteraction, mostRecentUserInteractionTime FROM ObservedDomains WHERE registrableDomain = ?"_s;
111 constexpr auto isGrandfatheredQuery = "SELECT grandfathered FROM ObservedDomains WHERE registrableDomain = ?"_s;
112 constexpr auto findExpiredUserInteractionQuery = "SELECT domainID FROM ObservedDomains WHERE hadUserInteraction = 1 AND mostRecentUserInteractionTime < ?"_s;
113
114 // CREATE TABLE Queries
115 constexpr auto createObservedDomain = "CREATE TABLE ObservedDomains ("
116     "domainID INTEGER PRIMARY KEY, registrableDomain TEXT NOT NULL UNIQUE ON CONFLICT FAIL, lastSeen REAL NOT NULL, "
117     "hadUserInteraction INTEGER NOT NULL, mostRecentUserInteractionTime REAL NOT NULL, grandfathered INTEGER NOT NULL, "
118     "isPrevalent INTEGER NOT NULL, isVeryPrevalent INTEGER NOT NULL, dataRecordsRemoved INTEGER NOT NULL,"
119     "timesAccessedAsFirstPartyDueToUserInteraction INTEGER NOT NULL, timesAccessedAsFirstPartyDueToStorageAccessAPI INTEGER NOT NULL);"_s;
120
121 enum {
122     DomainIDIndex,
123     RegistrableDomainIndex,
124     LastSeenIndex,
125     HadUserInteractionIndex,
126     MostRecentUserInteractionTimeIndex,
127     GrandfatheredIndex,
128     IsPrevalentIndex,
129     IsVeryPrevalentIndex,
130     DataRecordsRemovedIndex,
131     TimesAccessedAsFirstPartyDueToUserInteractionIndex,
132     TimesAccessedAsFirstPartyDueToStorageAccessAPIIndex
133 };
134
135 constexpr auto createTopLevelDomains = "CREATE TABLE TopLevelDomains ("
136     "topLevelDomainID INTEGER PRIMARY KEY, CONSTRAINT fkDomainID FOREIGN KEY(topLevelDomainID) "
137     "REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
138     
139 constexpr auto createStorageAccessUnderTopFrameDomains = "CREATE TABLE StorageAccessUnderTopFrameDomains ("
140     "domainID INTEGER NOT NULL, topLevelDomainID INTEGER NOT NULL ON CONFLICT FAIL, "
141     "CONSTRAINT fkDomainID FOREIGN KEY(domainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
142     "FOREIGN KEY(topLevelDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
143     
144 constexpr auto createTopFrameUniqueRedirectsTo = "CREATE TABLE TopFrameUniqueRedirectsTo ("
145     "sourceDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
146     "FOREIGN KEY(sourceDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
147     "FOREIGN KEY(toDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
148     
149 constexpr auto createTopFrameUniqueRedirectsFrom = "CREATE TABLE TopFrameUniqueRedirectsFrom ("
150     "targetDomainID INTEGER NOT NULL, fromDomainID INTEGER NOT NULL, "
151     "FOREIGN KEY(targetDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
152     "FOREIGN KEY(fromDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
153
154 constexpr auto createTopFrameLinkDecorationsFrom = "CREATE TABLE TopFrameLinkDecorationsFrom ("
155     "fromDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
156     "FOREIGN KEY(fromDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
157     "FOREIGN KEY(toDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
158
159 constexpr auto createSubframeUnderTopFrameDomains = "CREATE TABLE SubframeUnderTopFrameDomains ("
160     "subFrameDomainID INTEGER NOT NULL, topFrameDomainID INTEGER NOT NULL, "
161     "FOREIGN KEY(subFrameDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
162     "FOREIGN KEY(topFrameDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
163     
164 constexpr auto createSubresourceUnderTopFrameDomains = "CREATE TABLE SubresourceUnderTopFrameDomains ("
165     "subresourceDomainID INTEGER NOT NULL, topFrameDomainID INTEGER NOT NULL, "
166     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
167     "FOREIGN KEY(topFrameDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
168     
169 constexpr auto createSubresourceUniqueRedirectsTo = "CREATE TABLE SubresourceUniqueRedirectsTo ("
170     "subresourceDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
171     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
172     "FOREIGN KEY(toDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
173     
174 constexpr auto createSubresourceUniqueRedirectsFrom = "CREATE TABLE SubresourceUniqueRedirectsFrom ("
175     "subresourceDomainID INTEGER NOT NULL, fromDomainID INTEGER NOT NULL, "
176     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
177     "FOREIGN KEY(fromDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
178
179 // CREATE UNIQUE INDEX Queries
180 constexpr auto createUniqueIndexStorageAccessUnderTopFrameDomains = "CREATE UNIQUE INDEX IF NOT EXISTS StorageAccessUnderTopFrameDomains_domainID_topLevelDomainID on StorageAccessUnderTopFrameDomains ( domainID, topLevelDomainID );"_s;
181 constexpr auto createUniqueIndexTopFrameUniqueRedirectsTo = "CREATE UNIQUE INDEX IF NOT EXISTS TopFrameUniqueRedirectsTo_sourceDomainID_toDomainID on TopFrameUniqueRedirectsTo ( sourceDomainID, toDomainID );"_s;
182 constexpr auto createUniqueIndexTopFrameUniqueRedirectsFrom = "CREATE UNIQUE INDEX IF NOT EXISTS TopFrameUniqueRedirectsFrom_targetDomainID_fromDomainID on TopFrameUniqueRedirectsFrom ( targetDomainID, fromDomainID );"_s;
183 constexpr auto createUniqueIndexTopFrameLinkDecorationsFrom = "CREATE UNIQUE INDEX IF NOT EXISTS TopFrameLinkDecorationsFrom_fromDomainID_toDomainID on TopFrameLinkDecorationsFrom ( fromDomainID, toDomainID );"_s;
184 constexpr auto createUniqueIndexSubframeUnderTopFrameDomains = "CREATE UNIQUE INDEX IF NOT EXISTS SubframeUnderTopFrameDomains_subFrameDomainID_topFrameDomainID on SubframeUnderTopFrameDomains ( subFrameDomainID, topFrameDomainID );"_s;
185 constexpr auto createUniqueIndexSubresourceUnderTopFrameDomains = "CREATE UNIQUE INDEX IF NOT EXISTS SubresourceUnderTopFrameDomains_subresourceDomainID_topFrameDomainID on SubresourceUnderTopFrameDomains ( subresourceDomainID, topFrameDomainID );"_s;
186 constexpr auto createUniqueIndexSubresourceUniqueRedirectsTo = "CREATE UNIQUE INDEX IF NOT EXISTS SubresourceUniqueRedirectsTo_subresourceDomainID_toDomainID on SubresourceUniqueRedirectsTo ( subresourceDomainID, toDomainID );"_s;
187 constexpr auto createUniqueIndexSubresourceUniqueRedirectsFrom = "CREATE UNIQUE INDEX IF NOT EXISTS SubresourceUniqueRedirectsFrom_subresourceDomainID_fromDomainID on SubresourceUnderTopFrameDomains ( subresourceDomainID, fromDomainID );"_s;
188
189     
190 ResourceLoadStatisticsDatabaseStore::ResourceLoadStatisticsDatabaseStore(WebResourceLoadStatisticsStore& store, WorkQueue& workQueue, ShouldIncludeLocalhost shouldIncludeLocalhost, const String& storageDirectoryPath, PAL::SessionID sessionID)
191     : ResourceLoadStatisticsStore(store, workQueue, shouldIncludeLocalhost)
192     , m_storageDirectoryPath(storageDirectoryPath + "/observations.db")
193     , m_observedDomainCount(m_database, observedDomainCountQuery)
194     , m_insertObservedDomainStatement(m_database, insertObservedDomainQuery)
195     , m_insertTopLevelDomainStatement(m_database, insertTopLevelDomainQuery)
196     , m_domainIDFromStringStatement(m_database, domainIDFromStringQuery)
197     , m_topFrameLinkDecorationsFromExists(m_database, topFrameLinkDecorationsFromExistsQuery)
198     , m_subframeUnderTopFrameDomainExists(m_database, subframeUnderTopFrameDomainExistsQuery)
199     , m_subresourceUnderTopFrameDomainExists(m_database, subresourceUnderTopFrameDomainExistsQuery)
200     , m_subresourceUniqueRedirectsToExists(m_database, subresourceUniqueRedirectsToExistsQuery)
201     , m_mostRecentUserInteractionStatement(m_database, mostRecentUserInteractionQuery)
202     , m_updateLastSeenStatement(m_database, updateLastSeenQuery)
203     , m_updateDataRecordsRemovedStatement(m_database, updateDataRecordsRemovedQuery)
204     , m_updatePrevalentResourceStatement(m_database, updatePrevalentResourceQuery)
205     , m_isPrevalentResourceStatement(m_database, isPrevalentResourceQuery)
206     , m_updateVeryPrevalentResourceStatement(m_database, updateVeryPrevalentResourceQuery)
207     , m_isVeryPrevalentResourceStatement(m_database, isVeryPrevalentResourceQuery)
208     , m_clearPrevalentResourceStatement(m_database, clearPrevalentResourceQuery)
209     , m_hadUserInteractionStatement(m_database, hadUserInteractionQuery)
210     , m_updateGrandfatheredStatement(m_database, updateGrandfatheredQuery)
211     , m_isGrandfatheredStatement(m_database, isGrandfatheredQuery)
212     , m_findExpiredUserInteractionStatement(m_database, findExpiredUserInteractionQuery)
213     , m_sessionID(sessionID)
214 {
215     ASSERT(!RunLoop::isMain());
216
217     if (!m_database.open(m_storageDirectoryPath)) {
218         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::open failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
219         ASSERT_NOT_REACHED();
220         return;
221     }
222     
223     // Since we are using a workerQueue, the sequential dispatch blocks may be called by different threads.
224     m_database.disableThreadingChecks();
225
226     if (!m_database.tableExists("ObservedDomains"_s)) {
227         if (!createSchema()) {
228             RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::createSchema failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
229             ASSERT_NOT_REACHED();
230             return;
231         }
232     }
233     
234     if (!m_database.turnOnIncrementalAutoVacuum())
235         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::turnOnIncrementalAutoVacuum failed, error message: %{public}s", this, m_database.lastErrorMsg());
236
237     if (!prepareStatements()) {
238         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::prepareStatements failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
239         ASSERT_NOT_REACHED();
240         return;
241     }
242
243     workQueue.dispatchAfter(5_s, [weakThis = makeWeakPtr(*this)] {
244         if (weakThis)
245             weakThis->calculateAndSubmitTelemetry();
246     });
247 }
248
249 bool ResourceLoadStatisticsDatabaseStore::isEmpty() const
250 {
251     ASSERT(!RunLoop::isMain());
252     
253     bool result = false;
254     if (m_observedDomainCount.step() == SQLITE_ROW)
255         result = !m_observedDomainCount.getColumnInt(0);
256     
257     int resetResult = m_observedDomainCount.reset();
258     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
259     
260     return result;
261 }
262
263 bool ResourceLoadStatisticsDatabaseStore::createUniqueIndices()
264 {
265     if (!m_database.executeCommand(createUniqueIndexStorageAccessUnderTopFrameDomains)
266         || !m_database.executeCommand(createUniqueIndexTopFrameUniqueRedirectsTo)
267         || !m_database.executeCommand(createUniqueIndexTopFrameUniqueRedirectsFrom)
268         || !m_database.executeCommand(createUniqueIndexTopFrameLinkDecorationsFrom)
269         || !m_database.executeCommand(createUniqueIndexSubframeUnderTopFrameDomains)
270         || !m_database.executeCommand(createUniqueIndexSubresourceUnderTopFrameDomains)
271         || !m_database.executeCommand(createUniqueIndexSubresourceUniqueRedirectsTo)
272         || !m_database.executeCommand(createUniqueIndexSubresourceUnderTopFrameDomains)) {
273         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::createUniqueIndices failed to execute, error message: %{public}s", this, m_database.lastErrorMsg());
274         return false;
275     }
276     return true;
277 }
278
279 bool ResourceLoadStatisticsDatabaseStore::createSchema()
280 {
281     ASSERT(!RunLoop::isMain());
282
283     if (!m_database.executeCommand(createObservedDomain)) {
284         LOG_ERROR("Could not create ObservedDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
285         return false;
286     }
287
288     if (!m_database.executeCommand(createTopLevelDomains)) {
289         LOG_ERROR("Could not create TopLevelDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
290         return false;
291     }
292
293     if (!m_database.executeCommand(createStorageAccessUnderTopFrameDomains)) {
294         LOG_ERROR("Could not create StorageAccessUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
295         return false;
296     }
297
298     if (!m_database.executeCommand(createTopFrameUniqueRedirectsTo)) {
299         LOG_ERROR("Could not create TopFrameUniqueRedirectsTo table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
300         return false;
301     }
302
303     if (!m_database.executeCommand(createTopFrameUniqueRedirectsFrom)) {
304         LOG_ERROR("Could not create TopFrameUniqueRedirectsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
305         return false;
306     }
307
308     if (!m_database.executeCommand(createTopFrameLinkDecorationsFrom)) {
309         LOG_ERROR("Could not create TopFrameLinkDecorationsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
310         return false;
311     }
312     
313     if (!m_database.executeCommand(createSubframeUnderTopFrameDomains)) {
314         LOG_ERROR("Could not create SubframeUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
315         return false;
316     }
317
318     if (!m_database.executeCommand(createSubresourceUnderTopFrameDomains)) {
319         LOG_ERROR("Could not create SubresourceUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
320         return false;
321     }
322
323     if (!m_database.executeCommand(createSubresourceUniqueRedirectsTo)) {
324         LOG_ERROR("Could not create SubresourceUniqueRedirectsTo table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
325         return false;
326     }
327
328     if (!m_database.executeCommand(createSubresourceUniqueRedirectsFrom)) {
329         LOG_ERROR("Could not create SubresourceUniqueRedirectsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
330         return false;
331     }
332     
333     if (!createUniqueIndices())
334         return false;
335
336     return true;
337 }
338
339 bool ResourceLoadStatisticsDatabaseStore::prepareStatements()
340 {
341     ASSERT(!RunLoop::isMain());
342         
343     if (m_observedDomainCount.prepare() != SQLITE_OK
344         || m_insertObservedDomainStatement.prepare() != SQLITE_OK
345         || m_insertTopLevelDomainStatement.prepare() != SQLITE_OK
346         || m_domainIDFromStringStatement.prepare() != SQLITE_OK
347         || m_subframeUnderTopFrameDomainExists.prepare() != SQLITE_OK
348         || m_subresourceUnderTopFrameDomainExists.prepare() != SQLITE_OK
349         || m_subresourceUniqueRedirectsToExists.prepare() != SQLITE_OK
350         || m_updateLastSeenStatement.prepare() != SQLITE_OK
351         || m_updateDataRecordsRemovedStatement.prepare() != SQLITE_OK
352         || m_mostRecentUserInteractionStatement.prepare() != SQLITE_OK
353         || m_updatePrevalentResourceStatement.prepare() != SQLITE_OK
354         || m_isPrevalentResourceStatement.prepare() != SQLITE_OK
355         || m_updateVeryPrevalentResourceStatement.prepare() != SQLITE_OK
356         || m_isVeryPrevalentResourceStatement.prepare() != SQLITE_OK
357         || m_clearPrevalentResourceStatement.prepare() != SQLITE_OK
358         || m_hadUserInteractionStatement.prepare() != SQLITE_OK
359         || m_updateGrandfatheredStatement.prepare() != SQLITE_OK
360         || m_isGrandfatheredStatement.prepare() != SQLITE_OK
361         || m_findExpiredUserInteractionStatement.prepare() != SQLITE_OK
362         || m_topFrameLinkDecorationsFromExists.prepare() != SQLITE_OK) {
363         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::prepareStatements failed to prepare, error message: %{public}s", this, m_database.lastErrorMsg());
364         ASSERT_NOT_REACHED();
365         return false;
366     }
367
368     return true;
369 }
370
371 bool ResourceLoadStatisticsDatabaseStore::insertObservedDomain(const ResourceLoadStatistics& loadStatistics)
372 {
373     ASSERT(!RunLoop::isMain());
374
375     if (domainID(loadStatistics.registrableDomain)) {
376         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "ResourceLoadStatisticsDatabaseStore::insertObservedDomain can only be called on domains not in the database.");
377         ASSERT_NOT_REACHED();
378         return false;
379     }
380
381     if (m_insertObservedDomainStatement.bindText(RegistrableDomainIndex, loadStatistics.registrableDomain.string()) != SQLITE_OK
382         || m_insertObservedDomainStatement.bindDouble(LastSeenIndex, loadStatistics.lastSeen.secondsSinceEpoch().value()) != SQLITE_OK
383         || m_insertObservedDomainStatement.bindInt(HadUserInteractionIndex, loadStatistics.hadUserInteraction) != SQLITE_OK
384         || m_insertObservedDomainStatement.bindDouble(MostRecentUserInteractionTimeIndex, loadStatistics.mostRecentUserInteractionTime.secondsSinceEpoch().value()) != SQLITE_OK
385         || m_insertObservedDomainStatement.bindInt(GrandfatheredIndex, loadStatistics.grandfathered) != SQLITE_OK
386         || m_insertObservedDomainStatement.bindInt(IsPrevalentIndex, loadStatistics.isPrevalentResource) != SQLITE_OK
387         || m_insertObservedDomainStatement.bindInt(IsVeryPrevalentIndex, loadStatistics.isVeryPrevalentResource) != SQLITE_OK
388         || m_insertObservedDomainStatement.bindInt(DataRecordsRemovedIndex, loadStatistics.dataRecordsRemoved) != SQLITE_OK
389         || m_insertObservedDomainStatement.bindInt(TimesAccessedAsFirstPartyDueToUserInteractionIndex, loadStatistics.timesAccessedAsFirstPartyDueToUserInteraction) != SQLITE_OK
390         || m_insertObservedDomainStatement.bindInt(TimesAccessedAsFirstPartyDueToStorageAccessAPIIndex, loadStatistics.timesAccessedAsFirstPartyDueToStorageAccessAPI) != SQLITE_OK) {
391         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::insertObservedDomain failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
392         ASSERT_NOT_REACHED();
393         return false;
394     }
395
396     if (m_insertObservedDomainStatement.step() != SQLITE_DONE) {
397         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::insertObservedDomain failed to commit, error message: %{private}s", this, m_database.lastErrorMsg());
398         ASSERT_NOT_REACHED();
399         return false;
400     }
401
402     int resetResult = m_insertObservedDomainStatement.reset();
403     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
404
405     return true;
406 }
407
408 bool ResourceLoadStatisticsDatabaseStore::relationshipExists(WebCore::SQLiteStatement& statement, Optional<unsigned> firstDomainID, const RegistrableDomain& secondDomain) const
409 {
410     if (!firstDomainID)
411         return false;
412     
413     ASSERT(!RunLoop::isMain());
414
415     if (statement.bindInt(1, *firstDomainID) != SQLITE_OK
416         || statement.bindText(2, secondDomain.string()) != SQLITE_OK
417         || statement.step() != SQLITE_ROW) {
418         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_insertDomainRelationshipStatement failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
419         ASSERT_NOT_REACHED();
420         return false;
421     }
422     bool relationshipExists = !!statement.getColumnInt(0);
423
424     int resetResult = statement.reset();
425     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
426     
427     return relationshipExists;
428 }
429
430 bool ResourceLoadStatisticsDatabaseStore::insertDomainRelationship(WebCore::SQLiteStatement& statement, unsigned domainID, const RegistrableDomain& topFrame)
431 {
432     ASSERT(!RunLoop::isMain());
433
434     if (statement.bindInt(1, domainID) != SQLITE_OK
435         || statement.bindText(2, topFrame.string()) != SQLITE_OK
436         || statement.step() != SQLITE_DONE) {
437         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_insertDomainRelationshipStatement failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
438         ASSERT_NOT_REACHED();
439         return false;
440     }
441
442     int resetResult = statement.reset();
443     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
444     
445     return true;
446 }
447
448 Optional<unsigned> ResourceLoadStatisticsDatabaseStore::domainID(const RegistrableDomain& domain) const
449 {
450     ASSERT(!RunLoop::isMain());
451
452     unsigned domainID = 0;
453
454     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK) {
455         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::domainIDFromString failed, error message: %{private}s", this, m_database.lastErrorMsg());
456         return WTF::nullopt;
457     }
458     
459     if (m_domainIDFromStringStatement.step() != SQLITE_ROW) {
460         int resetResult = m_domainIDFromStringStatement.reset();
461         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
462         return WTF::nullopt;
463     }
464
465     domainID = m_domainIDFromStringStatement.getColumnInt(0);
466
467     int resetResult = m_domainIDFromStringStatement.reset();
468     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
469     
470     return domainID;
471 }
472
473 String ResourceLoadStatisticsDatabaseStore::ensureAndMakeDomainList(const HashSet<RegistrableDomain>& subframeUnderTopFrameDomains)
474 {
475     StringBuilder builder;
476     
477     for (auto& topFrameResource : subframeUnderTopFrameDomains) {
478         
479         // Insert query will fail if top frame domain is not already in the database
480         ensureResourceStatisticsForRegistrableDomain(topFrameResource);
481         
482         if (!builder.isEmpty())
483             builder.appendLiteral(", ");
484         builder.append('"');
485         builder.append(topFrameResource.string());
486         builder.append('"');
487     }
488
489     return builder.toString();
490 }
491
492 void ResourceLoadStatisticsDatabaseStore::insertDomainRelationshipList(const String& statement, const HashSet<RegistrableDomain>& subframeUnderTopFrameDomains, unsigned domainID)
493 {
494     SQLiteStatement insertRelationshipStatement(m_database, makeString(statement, ensureAndMakeDomainList(subframeUnderTopFrameDomains), " );"));
495     
496     if (insertRelationshipStatement.prepare() != SQLITE_OK
497         || insertRelationshipStatement.bindInt(1, domainID) != SQLITE_OK
498         || insertRelationshipStatement.step() != SQLITE_DONE) {
499         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::insertDomainRelationshipList failed, error message: %{private}s", this, m_database.lastErrorMsg());
500         ASSERT_NOT_REACHED();
501     }
502 }
503
504 void ResourceLoadStatisticsDatabaseStore::insertDomainRelationships(const ResourceLoadStatistics& loadStatistics)
505 {
506     ASSERT(!RunLoop::isMain());
507     auto registrableDomainID = domainID(loadStatistics.registrableDomain);
508     
509     if (!registrableDomainID)
510         return;
511
512     insertDomainRelationshipList(storageAccessUnderTopFrameDomainsQuery, loadStatistics.storageAccessUnderTopFrameDomains, registrableDomainID.value());
513     insertDomainRelationshipList(topFrameUniqueRedirectsToQuery, loadStatistics.topFrameUniqueRedirectsTo, registrableDomainID.value());
514     insertDomainRelationshipList(topFrameUniqueRedirectsFromQuery, loadStatistics.topFrameUniqueRedirectsFrom, registrableDomainID.value());
515     insertDomainRelationshipList(subframeUnderTopFrameDomainsQuery, loadStatistics.subframeUnderTopFrameDomains, registrableDomainID.value());
516     insertDomainRelationshipList(subresourceUnderTopFrameDomainsQuery, loadStatistics.subresourceUnderTopFrameDomains, registrableDomainID.value());
517     insertDomainRelationshipList(subresourceUniqueRedirectsToQuery, loadStatistics.subresourceUniqueRedirectsTo, registrableDomainID.value());
518     insertDomainRelationshipList(subresourceUniqueRedirectsFromQuery, loadStatistics.subresourceUniqueRedirectsFrom, registrableDomainID.value());
519 }
520
521 void ResourceLoadStatisticsDatabaseStore::populateFromMemoryStore(const ResourceLoadStatisticsMemoryStore& memoryStore)
522 {
523     ASSERT(!RunLoop::isMain());
524
525     if (!isEmpty())
526         return;
527
528     auto& statisticsMap = memoryStore.data();
529     for (const auto& statistic : statisticsMap)
530         insertObservedDomain(statistic.value);
531
532     // Make a separate pass for inter-domain relationships so we
533     // can refer to the ObservedDomain table entries
534     for (auto& statistic : statisticsMap)
535         insertDomainRelationships(statistic.value);
536 }
537
538 void ResourceLoadStatisticsDatabaseStore::merge(WebCore::SQLiteStatement& current, const ResourceLoadStatistics& other)
539 {
540     ASSERT(!RunLoop::isMain());
541
542     auto currentRegistrableDomain = current.getColumnText(RegistrableDomainIndex);
543     auto currentLastSeen = current.getColumnDouble(LastSeenIndex);
544     auto currentMostRecentUserInteraction = current.getColumnDouble(MostRecentUserInteractionTimeIndex);
545     bool currentGrandfathered = current.getColumnInt(GrandfatheredIndex);
546     bool currentIsPrevalent = current.getColumnInt(IsPrevalentIndex);
547     bool currentIsVeryPrevalent = current.getColumnInt(IsVeryPrevalentIndex);
548     unsigned currentDataRecordsRemoved = current.getColumnInt(DataRecordsRemovedIndex);
549
550     ASSERT(currentRegistrableDomain == other.registrableDomain.string());
551
552     if (WallTime::fromRawSeconds(currentLastSeen) < other.lastSeen)
553         updateLastSeen(other.registrableDomain, other.lastSeen);
554
555     if (!other.hadUserInteraction) {
556         // If most recent user interaction time has been reset do so here too.
557         if (!other.mostRecentUserInteractionTime)
558             setUserInteraction(other.registrableDomain, false, { });
559     } else
560         setUserInteraction(other.registrableDomain, true, std::max(WallTime::fromRawSeconds(currentMostRecentUserInteraction), other.mostRecentUserInteractionTime));
561
562     if (other.grandfathered && !currentGrandfathered)
563         setGrandfathered(other.registrableDomain, true);
564     if (other.isPrevalentResource && !currentIsPrevalent)
565         setPrevalentResource(other.registrableDomain);
566     if (other.isVeryPrevalentResource && !currentIsVeryPrevalent)
567         setVeryPrevalentResource(other.registrableDomain);
568     if (other.dataRecordsRemoved > currentDataRecordsRemoved)
569         updateDataRecordsRemoved(other.registrableDomain, other.dataRecordsRemoved);
570 }
571
572 void ResourceLoadStatisticsDatabaseStore::mergeStatistic(const ResourceLoadStatistics& statistic)
573 {
574     ASSERT(!RunLoop::isMain());
575
576     SQLiteStatement fetchOldStatisticData(m_database, "SELECT * FROM ObservedDomains where registrableDomain = ?");
577     if (fetchOldStatisticData.prepare() != SQLITE_OK
578         || fetchOldStatisticData.bindText(1, statistic.registrableDomain.string()) != SQLITE_OK
579         || fetchOldStatisticData.step() != SQLITE_ROW) {
580         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::mergeStatistic. Statement failed to bind or domain was not found, error message: %{private}s", this, m_database.lastErrorMsg());
581         ASSERT_NOT_REACHED();
582         return;
583     }
584
585     merge(fetchOldStatisticData, statistic);
586 }
587
588 void ResourceLoadStatisticsDatabaseStore::mergeStatistics(Vector<ResourceLoadStatistics>&& statistics)
589 {
590     ASSERT(!RunLoop::isMain());
591
592     for (auto& statistic : statistics) {
593         if (!domainID(statistic.registrableDomain))
594             insertObservedDomain(statistic);
595         else
596             mergeStatistic(statistic);
597     }
598
599     // Make a separate pass for inter-domain relationships so we
600     // can refer to the ObservedDomain table entries.
601     for (auto& statistic : statistics)
602         insertDomainRelationships(statistic);
603 }
604
605 void ResourceLoadStatisticsDatabaseStore::calculateAndSubmitTelemetry() const
606 {
607     ASSERT(!RunLoop::isMain());
608
609     // FIXME(195088): Implement for Database version.
610 }
611
612 static String domainsToString(const HashSet<RegistrableDomain>& domains)
613 {
614     StringBuilder builder;
615     for (const auto& domainName : domains) {
616         if (!builder.isEmpty())
617             builder.appendLiteral(", ");
618         builder.append('"');
619         builder.append(domainName.string());
620         builder.append('"');
621     }
622
623     return builder.toString();
624 }
625
626 void ResourceLoadStatisticsDatabaseStore::incrementRecordsDeletedCountForDomains(HashSet<RegistrableDomain>&& domains)
627 {
628     ASSERT(!RunLoop::isMain());
629
630     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET dataRecordsRemoved = dataRecordsRemoved + 1 WHERE registrableDomain IN (", domainsToString(domains), ")"));
631     if (domainsToUpdateStatement.prepare() != SQLITE_OK
632         || domainsToUpdateStatement.step() != SQLITE_DONE) {
633         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::incrementStatisticsForDomains failed, error message: %{private}s", this, m_database.lastErrorMsg());
634         ASSERT_NOT_REACHED();
635     }
636 }
637
638 unsigned ResourceLoadStatisticsDatabaseStore::recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(unsigned primaryDomainID, StdSet<unsigned>& nonPrevalentRedirectionSources, unsigned numberOfRecursiveCalls)
639 {
640     ASSERT(!RunLoop::isMain());
641
642     if (numberOfRecursiveCalls >= maxNumberOfRecursiveCallsInRedirectTraceBack) {
643         RELEASE_LOG(ResourceLoadStatistics, "Hit %u recursive calls in redirect backtrace. Returning early.", maxNumberOfRecursiveCallsInRedirectTraceBack);
644         return numberOfRecursiveCalls;
645     }
646     
647     ++numberOfRecursiveCalls;
648
649     StdSet<unsigned> newlyIdentifiedDomains;
650     SQLiteStatement findSubresources(m_database, "SELECT SubresourceUniqueRedirectsFrom.fromDomainID from SubresourceUniqueRedirectsFrom INNER JOIN ObservedDomains ON ObservedDomains.domainID = SubresourceUniqueRedirectsFrom.fromDomainID WHERE subresourceDomainID = ? AND ObservedDomains.isPrevalent = 0"_s);
651     if (findSubresources.prepare() != SQLITE_OK
652         || findSubresources.bindInt(1, primaryDomainID) != SQLITE_OK) {
653         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain failed, error message: %{private}s", this, m_database.lastErrorMsg());
654         ASSERT_NOT_REACHED();
655         return 0;
656     }
657     
658     while (findSubresources.step() == SQLITE_ROW) {
659         int newDomainID = findSubresources.getColumnInt(0);
660         auto insertResult = nonPrevalentRedirectionSources.insert(newDomainID);
661         if (insertResult.second)
662             newlyIdentifiedDomains.insert(newDomainID);
663     }
664
665     SQLiteStatement findTopFrames(m_database, "SELECT TopFrameUniqueRedirectsFrom.fromDomainID from TopFrameUniqueRedirectsFrom INNER JOIN ObservedDomains ON ObservedDomains.domainID = TopFrameUniqueRedirectsFrom.fromDomainID WHERE targetDomainID = ? AND ObservedDomains.isPrevalent = 0"_s);
666     if (findTopFrames.prepare() != SQLITE_OK
667         || findTopFrames.bindInt(1, primaryDomainID) != SQLITE_OK) {
668         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain failed, error message: %{private}s", this, m_database.lastErrorMsg());
669         ASSERT_NOT_REACHED();
670         return 0;
671     }
672     
673     while (findTopFrames.step() == SQLITE_ROW) {
674         int newDomainID = findTopFrames.getColumnInt(0);
675         auto insertResult = nonPrevalentRedirectionSources.insert(newDomainID);
676         if (insertResult.second)
677             newlyIdentifiedDomains.insert(newDomainID);
678     }
679
680     if (newlyIdentifiedDomains.empty())
681         return numberOfRecursiveCalls;
682
683     for (auto domainID : newlyIdentifiedDomains)
684         numberOfRecursiveCalls = recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(domainID, nonPrevalentRedirectionSources, numberOfRecursiveCalls);
685
686     return numberOfRecursiveCalls;
687 }
688
689 template <typename IteratorType>
690 static String buildList(const WTF::IteratorRange<IteratorType>& values)
691 {
692     StringBuilder builder;
693     for (auto domainID : values) {
694         if (!builder.isEmpty())
695             builder.appendLiteral(", ");
696         builder.appendNumber(domainID);
697     }
698     return builder.toString();
699 }
700
701 void ResourceLoadStatisticsDatabaseStore::markAsPrevalentIfHasRedirectedToPrevalent()
702 {
703     ASSERT(!RunLoop::isMain());
704
705     StdSet<unsigned> prevalentDueToRedirect;
706     SQLiteStatement subresourceRedirectStatement(m_database, "SELECT DISTINCT SubresourceUniqueRedirectsTo.subresourceDomainID FROM SubresourceUniqueRedirectsTo JOIN ObservedDomains ON ObservedDomains.domainID = SubresourceUniqueRedirectsTo.toDomainID AND ObservedDomains.isPrevalent = 1"_s);
707     if (subresourceRedirectStatement.prepare() == SQLITE_OK) {
708         while (subresourceRedirectStatement.step() == SQLITE_ROW)
709             prevalentDueToRedirect.insert(subresourceRedirectStatement.getColumnInt(0));
710     }
711
712     SQLiteStatement topFrameRedirectStatement(m_database, "SELECT DISTINCT TopFrameUniqueRedirectsTo.sourceDomainID FROM TopFrameUniqueRedirectsTo JOIN ObservedDomains ON ObservedDomains.domainID = TopFrameUniqueRedirectsTo.toDomainID AND ObservedDomains.isPrevalent = 1"_s);
713     if (topFrameRedirectStatement.prepare() == SQLITE_OK) {
714         while (topFrameRedirectStatement.step() == SQLITE_ROW)
715             prevalentDueToRedirect.insert(topFrameRedirectStatement.getColumnInt(0));
716     }
717
718     SQLiteStatement markPrevalentStatement(m_database, makeString("UPDATE ObservedDomains SET isPrevalent = 1 WHERE domainID IN (", buildList(WTF::IteratorRange<StdSet<unsigned>::iterator>(prevalentDueToRedirect.begin(), prevalentDueToRedirect.end())), ")"));
719     if (markPrevalentStatement.prepare() != SQLITE_OK
720         || markPrevalentStatement.step() != SQLITE_DONE) {
721         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::markAsPrevalentIfHasRedirectedToPrevalent failed to execute, error message: %{private}s", this, m_database.lastErrorMsg());
722         ASSERT_NOT_REACHED();
723     }
724 }
725
726 HashMap<unsigned, ResourceLoadStatisticsDatabaseStore::NotVeryPrevalentResources> ResourceLoadStatisticsDatabaseStore::findNotVeryPrevalentResources()
727 {
728     ASSERT(!RunLoop::isMain());
729
730     HashMap<unsigned, NotVeryPrevalentResources> results;
731     SQLiteStatement notVeryPrevalentResourcesStatement(m_database, "SELECT domainID, registrableDomain, isPrevalent FROM ObservedDomains WHERE isVeryPrevalent = 0"_s);
732     if (notVeryPrevalentResourcesStatement.prepare() == SQLITE_OK) {
733         while (notVeryPrevalentResourcesStatement.step() == SQLITE_ROW) {
734             unsigned key = static_cast<unsigned>(notVeryPrevalentResourcesStatement.getColumnInt(0));
735             NotVeryPrevalentResources value({ RegistrableDomain::uncheckedCreateFromRegistrableDomainString(notVeryPrevalentResourcesStatement.getColumnText(1))
736                 , notVeryPrevalentResourcesStatement.getColumnInt(2) ? ResourceLoadPrevalence::High : ResourceLoadPrevalence::Low
737                 , 0, 0, 0, 0 });
738             results.add(key, value);
739         }
740     }
741
742     StringBuilder builder;
743     for (auto value : results.keys()) {
744         if (!builder.isEmpty())
745             builder.appendLiteral(", ");
746         builder.appendNumber(value);
747     }
748
749     auto domainIDsOfInterest = builder.toString();
750
751     SQLiteStatement subresourceUnderTopFrameDomainsStatement(m_database, makeString("SELECT subresourceDomainID, COUNT(topFrameDomainID) FROM SubresourceUnderTopFrameDomains WHERE subresourceDomainID IN (", domainIDsOfInterest, ") GROUP BY subresourceDomainID"));
752     if (subresourceUnderTopFrameDomainsStatement.prepare() == SQLITE_OK) {
753         while (subresourceUnderTopFrameDomainsStatement.step() == SQLITE_ROW) {
754             unsigned domainID = static_cast<unsigned>(subresourceUnderTopFrameDomainsStatement.getColumnInt(0));
755             auto result = results.find(domainID);
756             if (result != results.end())
757                 result->value.subresourceUnderTopFrameDomainsCount = static_cast<unsigned>(subresourceUnderTopFrameDomainsStatement.getColumnInt(1));
758         }
759     }
760
761     SQLiteStatement subresourceUniqueRedirectsToCountStatement(m_database, makeString("SELECT subresourceDomainID, COUNT(toDomainID) FROM SubresourceUniqueRedirectsTo WHERE subresourceDomainID IN (", domainIDsOfInterest, ") GROUP BY subresourceDomainID"));
762     if (subresourceUniqueRedirectsToCountStatement.prepare() == SQLITE_OK) {
763         while (subresourceUniqueRedirectsToCountStatement.step() == SQLITE_ROW) {
764             unsigned domainID = static_cast<unsigned>(subresourceUniqueRedirectsToCountStatement.getColumnInt(0));
765             auto result = results.find(domainID);
766             if (result != results.end())
767                 result->value.subresourceUniqueRedirectsToCount = static_cast<unsigned>(subresourceUniqueRedirectsToCountStatement.getColumnInt(1));
768         }
769     }
770
771     SQLiteStatement subframeUnderTopFrameDomainsCountStatement(m_database, makeString("SELECT subframeDomainID, COUNT(topFrameDomainID) FROM SubframeUnderTopFrameDomains WHERE subframeDomainID IN (", domainIDsOfInterest, ") GROUP BY subframeDomainID"));
772     if (subframeUnderTopFrameDomainsCountStatement.prepare() == SQLITE_OK) {
773         while (subframeUnderTopFrameDomainsCountStatement.step() == SQLITE_ROW) {
774             unsigned domainID = static_cast<unsigned>(subframeUnderTopFrameDomainsCountStatement.getColumnInt(0));
775             auto result = results.find(domainID);
776             if (result != results.end())
777                 result->value.subframeUnderTopFrameDomainsCount = static_cast<unsigned>(subframeUnderTopFrameDomainsCountStatement.getColumnInt(1));
778         }
779     }
780
781     SQLiteStatement topFrameUniqueRedirectsToCountStatement(m_database, makeString("SELECT sourceDomainID, COUNT(toDomainID) FROM TopFrameUniqueRedirectsTo WHERE sourceDomainID IN (", domainIDsOfInterest, ") GROUP BY sourceDomainID"));
782     if (topFrameUniqueRedirectsToCountStatement.prepare() == SQLITE_OK) {
783         while (topFrameUniqueRedirectsToCountStatement.step() == SQLITE_ROW) {
784             unsigned domainID = static_cast<unsigned>(topFrameUniqueRedirectsToCountStatement.getColumnInt(0));
785             auto result = results.find(domainID);
786             if (result != results.end())
787                 result->value.topFrameUniqueRedirectsToCount = static_cast<unsigned>(topFrameUniqueRedirectsToCountStatement.getColumnInt(1));
788         }
789     }
790
791     return results;
792 }
793
794 void ResourceLoadStatisticsDatabaseStore::reclassifyResources()
795 {
796     ASSERT(!RunLoop::isMain());
797
798     auto notVeryPrevalentResources = findNotVeryPrevalentResources();
799
800     for (auto& resourceStatistic : notVeryPrevalentResources.values()) {
801         if (shouldSkip(resourceStatistic.registrableDomain))
802             continue;
803
804         auto newPrevalence = classifier().calculateResourcePrevalence(resourceStatistic.subresourceUnderTopFrameDomainsCount, resourceStatistic.subresourceUniqueRedirectsToCount, resourceStatistic.subframeUnderTopFrameDomainsCount, resourceStatistic.topFrameUniqueRedirectsToCount, resourceStatistic.prevalence);
805         if (newPrevalence != resourceStatistic.prevalence)
806             setPrevalentResource(resourceStatistic.registrableDomain, newPrevalence);
807     }
808 }
809
810 void ResourceLoadStatisticsDatabaseStore::classifyPrevalentResources()
811 {
812     ASSERT(!RunLoop::isMain());
813     ensurePrevalentResourcesForDebugMode();
814     markAsPrevalentIfHasRedirectedToPrevalent();
815     reclassifyResources();
816 }
817
818 void ResourceLoadStatisticsDatabaseStore::syncStorageIfNeeded()
819 {
820     ASSERT(!RunLoop::isMain());
821     m_database.runVacuumCommand();
822 }
823
824 void ResourceLoadStatisticsDatabaseStore::syncStorageImmediately()
825 {
826     ASSERT(!RunLoop::isMain());
827     m_database.runVacuumCommand();
828 }
829
830 void ResourceLoadStatisticsDatabaseStore::hasStorageAccess(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain, Optional<FrameIdentifier> frameID, PageIdentifier pageID, CompletionHandler<void(bool)>&& completionHandler)
831 {
832     ASSERT(!RunLoop::isMain());
833
834     ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
835
836     switch (cookieTreatmentForOrigin(subFrameDomain)) {
837     case CookieTreatmentResult::BlockAndPurge:
838         completionHandler(false);
839         return;
840     case CookieTreatmentResult::Allow:
841         // We should only return true if the context has asked for and been granted access.
842         completionHandler(false);
843         return;
844     case CookieTreatmentResult::BlockAndKeep:
845         // Do nothing. The below dispatch will complete the task.
846         break;
847     };
848
849     RunLoop::main().dispatch([store = makeRef(store()), subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, completionHandler = WTFMove(completionHandler)]() mutable {
850         store->callHasStorageAccessForFrameHandler(subFrameDomain, topFrameDomain, frameID.value(), pageID, [store = store.copyRef(), completionHandler = WTFMove(completionHandler)](bool result) mutable {
851             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler), result] () mutable {
852                 completionHandler(result);
853             });
854         });
855     });
856 }
857
858 void ResourceLoadStatisticsDatabaseStore::requestStorageAccess(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, FrameIdentifier frameID, PageIdentifier pageID, CompletionHandler<void(StorageAccessStatus)>&& completionHandler)
859 {
860     ASSERT(!RunLoop::isMain());
861
862     auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
863
864     switch (cookieTreatmentForOrigin(subFrameDomain)) {
865     case CookieTreatmentResult::BlockAndPurge: {
866         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "Cannot grant storage access to %{private}s since its cookies are blocked in third-party contexts and it has not received user interaction as first-party.", subFrameDomain.string().utf8().data());
867         completionHandler(StorageAccessStatus::CannotRequestAccess);
868         }
869         return;
870     case CookieTreatmentResult::BlockAndKeep: {
871         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "No need to grant storage access to %{private}s since its cookies are not blocked in third-party contexts.", subFrameDomain.string().utf8().data());
872         completionHandler(StorageAccessStatus::HasAccess);
873         }
874         return;
875     case CookieTreatmentResult::Allow:
876         // Do nothing
877         break;
878     };
879
880     auto userWasPromptedEarlier = hasUserGrantedStorageAccessThroughPrompt(subFrameStatus.second, topFrameDomain);
881     if (userWasPromptedEarlier == StorageAccessPromptWasShown::No) {
882         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "About to ask the user whether they want to grant storage access to %{private}s under %{private}s or not.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
883         completionHandler(StorageAccessStatus::RequiresUserPrompt);
884         return;
885     }
886
887     if (userWasPromptedEarlier == StorageAccessPromptWasShown::Yes)
888         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "Storage access was granted to %{private}s under %{private}s.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
889
890     SQLiteStatement incrementStorageAccess(m_database, "UPDATE ObservedDomains SET timesAccessedAsFirstPartyDueToStorageAccessAPI = timesAccessedAsFirstPartyDueToStorageAccessAPI + 1 WHERE domainID = ?");
891     if (incrementStorageAccess.prepare() != SQLITE_OK
892         || incrementStorageAccess.bindInt(1, subFrameStatus.second) != SQLITE_OK
893         || incrementStorageAccess.step() != SQLITE_DONE) {
894         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::requestStorageAccess failed, error message: %{private}s", this, m_database.lastErrorMsg());
895         ASSERT_NOT_REACHED();
896         return;
897     }
898     
899     grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, userWasPromptedEarlier, [completionHandler = WTFMove(completionHandler)] (StorageAccessWasGranted wasGranted) mutable {
900         completionHandler(wasGranted == StorageAccessWasGranted::Yes ? StorageAccessStatus::HasAccess : StorageAccessStatus::CannotRequestAccess);
901     });
902 }
903
904 void ResourceLoadStatisticsDatabaseStore::requestStorageAccessUnderOpener(DomainInNeedOfStorageAccess&& domainInNeedOfStorageAccess, PageIdentifier openerPageID, OpenerDomain&& openerDomain)
905 {
906     ASSERT(domainInNeedOfStorageAccess != openerDomain);
907     ASSERT(!RunLoop::isMain());
908
909     if (domainInNeedOfStorageAccess == openerDomain)
910         return;
911
912     ensureResourceStatisticsForRegistrableDomain(domainInNeedOfStorageAccess);
913
914     if (cookieTreatmentForOrigin(domainInNeedOfStorageAccess) != CookieTreatmentResult::Allow)
915         return;
916
917     RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ITPDebug, "[Temporary combatibility fix] Storage access was granted for %{private}s under opener page from %{private}s, with user interaction in the opened window.", domainInNeedOfStorageAccess.string().utf8().data(), openerDomain.string().utf8().data());
918     grantStorageAccessInternal(WTFMove(domainInNeedOfStorageAccess), WTFMove(openerDomain), WTF::nullopt, openerPageID, StorageAccessPromptWasShown::No, [](StorageAccessWasGranted) { });
919 }
920
921 void ResourceLoadStatisticsDatabaseStore::grantStorageAccess(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, FrameIdentifier frameID, PageIdentifier pageID, StorageAccessPromptWasShown promptWasShown, CompletionHandler<void(StorageAccessWasGranted)>&& completionHandler)
922 {
923     ASSERT(!RunLoop::isMain());
924
925     if (promptWasShown == StorageAccessPromptWasShown::Yes) {
926         auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
927         ASSERT(subFrameStatus.first == AddedRecord::No);
928         ASSERT(hasHadUserInteraction(subFrameDomain, OperatingDatesWindow::Long));
929         insertDomainRelationshipList(storageAccessUnderTopFrameDomainsQuery, HashSet<RegistrableDomain>({ topFrameDomain }), subFrameStatus.second);
930     }
931
932     grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, promptWasShown, WTFMove(completionHandler));
933 }
934
935 void ResourceLoadStatisticsDatabaseStore::grantStorageAccessInternal(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, Optional<FrameIdentifier> frameID, PageIdentifier pageID, StorageAccessPromptWasShown promptWasShownNowOrEarlier, CompletionHandler<void(StorageAccessWasGranted)>&& completionHandler)
936 {
937     ASSERT(!RunLoop::isMain());
938
939     if (subFrameDomain == topFrameDomain) {
940         completionHandler(StorageAccessWasGranted::Yes);
941         return;
942     }
943
944     if (promptWasShownNowOrEarlier == StorageAccessPromptWasShown::Yes) {
945 #ifndef NDEBUG
946         auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
947         ASSERT(subFrameStatus.first == AddedRecord::No);
948         ASSERT(hasHadUserInteraction(subFrameDomain, OperatingDatesWindow::Long));
949         ASSERT(hasUserGrantedStorageAccessThroughPrompt(subFrameStatus.second, topFrameDomain) == StorageAccessPromptWasShown::Yes);
950 #endif
951         setUserInteraction(subFrameDomain, true, WallTime::now());
952     }
953
954     RunLoop::main().dispatch([subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, store = makeRef(store()), completionHandler = WTFMove(completionHandler)]() mutable {
955         store->callGrantStorageAccessHandler(subFrameDomain, topFrameDomain, frameID, pageID, [completionHandler = WTFMove(completionHandler), store = store.copyRef()](StorageAccessWasGranted wasGranted) mutable {
956             store->statisticsQueue().dispatch([wasGranted, completionHandler = WTFMove(completionHandler)] () mutable {
957                 completionHandler(wasGranted);
958             });
959         });
960     });
961
962 }
963
964 void ResourceLoadStatisticsDatabaseStore::grandfatherDataForDomains(const HashSet<RegistrableDomain>& domains)
965 {
966     ASSERT(!RunLoop::isMain());
967
968     for (auto& registrableDomain : domains)
969         ensureResourceStatisticsForRegistrableDomain(registrableDomain);
970
971     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET grandfathered = 1 WHERE registrableDomain IN (", domainsToString(domains), ")"));
972     if (domainsToUpdateStatement.prepare() != SQLITE_OK
973         || domainsToUpdateStatement.step() != SQLITE_DONE) {
974         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::grandfatherDataForDomains failed, error message: %{private}s", this, m_database.lastErrorMsg());
975         ASSERT_NOT_REACHED();
976     }
977 }
978
979 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::ensurePrevalentResourcesForDebugMode()
980 {
981     ASSERT(!RunLoop::isMain());
982
983     if (!debugModeEnabled())
984         return { };
985
986     Vector<RegistrableDomain> primaryDomainsToBlock;
987     primaryDomainsToBlock.reserveInitialCapacity(2);
988
989     ensureResourceStatisticsForRegistrableDomain(debugStaticPrevalentResource());
990     setPrevalentResource(debugStaticPrevalentResource(), ResourceLoadPrevalence::High);
991     primaryDomainsToBlock.uncheckedAppend(debugStaticPrevalentResource());
992
993     if (!debugManualPrevalentResource().isEmpty()) {
994         ensureResourceStatisticsForRegistrableDomain(debugManualPrevalentResource());
995         setPrevalentResource(debugManualPrevalentResource(), ResourceLoadPrevalence::High);
996         primaryDomainsToBlock.uncheckedAppend(debugManualPrevalentResource());
997         RELEASE_LOG_INFO(ITPDebug, "Did set %{private}s as prevalent resource for the purposes of ITP Debug Mode.", debugManualPrevalentResource().string().utf8().data());
998     }
999
1000     return primaryDomainsToBlock;
1001 }
1002
1003 void ResourceLoadStatisticsDatabaseStore::logFrameNavigation(const RegistrableDomain& targetDomain, const RegistrableDomain& topFrameDomain, const RegistrableDomain& sourceDomain, bool isRedirect, bool isMainFrame)
1004 {
1005     ASSERT(!RunLoop::isMain());
1006
1007     bool areTargetAndTopFrameDomainsSameSite = targetDomain == topFrameDomain;
1008     bool areTargetAndSourceDomainsSameSite = targetDomain == sourceDomain;
1009
1010     bool statisticsWereUpdated = false;
1011     if (!isMainFrame && !(areTargetAndTopFrameDomainsSameSite || areTargetAndSourceDomainsSameSite)) {
1012         auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
1013         updateLastSeen(targetDomain, ResourceLoadStatistics::reduceTimeResolution(WallTime::now()));
1014         insertDomainRelationshipList(subframeUnderTopFrameDomainsQuery, HashSet<RegistrableDomain>({ topFrameDomain }), targetResult.second);
1015         statisticsWereUpdated = true;
1016     }
1017
1018     if (isRedirect && !areTargetAndSourceDomainsSameSite) {
1019         if (isMainFrame) {
1020             auto redirectingDomainResult = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
1021             auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
1022             insertDomainRelationshipList(topFrameUniqueRedirectsToQuery, HashSet<RegistrableDomain>({ targetDomain }), redirectingDomainResult.second);
1023             insertDomainRelationshipList(topFrameUniqueRedirectsFromQuery, HashSet<RegistrableDomain>({ sourceDomain }), targetResult.second);
1024         } else {
1025             auto redirectingDomainResult = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
1026             auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
1027             insertDomainRelationshipList(subresourceUniqueRedirectsToQuery, HashSet<RegistrableDomain>({ targetDomain }), redirectingDomainResult.second);
1028             insertDomainRelationshipList(subresourceUniqueRedirectsFromQuery, HashSet<RegistrableDomain>({ sourceDomain }), targetResult.second);
1029         }
1030         statisticsWereUpdated = true;
1031     }
1032
1033     if (statisticsWereUpdated)
1034         scheduleStatisticsProcessingRequestIfNecessary();
1035 }
1036
1037 void ResourceLoadStatisticsDatabaseStore::logCrossSiteLoadWithLinkDecoration(const NavigatedFromDomain& fromDomain, const NavigatedToDomain& toDomain)
1038 {
1039     ASSERT(!RunLoop::isMain());
1040     ASSERT(fromDomain != toDomain);
1041
1042     auto fromDomainResult = ensureResourceStatisticsForRegistrableDomain(fromDomain);
1043     insertDomainRelationshipList(topFrameLinkDecorationsFromQuery, HashSet<RegistrableDomain>({ toDomain }), fromDomainResult.second);
1044     scheduleStatisticsProcessingRequestIfNecessary();
1045 }
1046
1047 void ResourceLoadStatisticsDatabaseStore::setUserInteraction(const RegistrableDomain& domain, bool hadUserInteraction, WallTime mostRecentInteraction)
1048 {
1049     ASSERT(!RunLoop::isMain());
1050
1051     if (m_mostRecentUserInteractionStatement.bindInt(1, hadUserInteraction) != SQLITE_OK
1052         || m_mostRecentUserInteractionStatement.bindDouble(2, mostRecentInteraction.secondsSinceEpoch().value()) != SQLITE_OK
1053         || m_mostRecentUserInteractionStatement.bindText(3, domain.string()) != SQLITE_OK
1054         || m_mostRecentUserInteractionStatement.step() != SQLITE_DONE) {
1055         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::setUserInteraction, error message: %{private}s", this, m_database.lastErrorMsg());
1056         ASSERT_NOT_REACHED();
1057         return;
1058     }
1059
1060     int resetResult = m_mostRecentUserInteractionStatement.reset();
1061     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1062 }
1063
1064 void ResourceLoadStatisticsDatabaseStore::logUserInteraction(const TopFrameDomain& domain)
1065 {
1066     ASSERT(!RunLoop::isMain());
1067
1068     ensureResourceStatisticsForRegistrableDomain(domain);
1069     setUserInteraction(domain, true, WallTime::now());
1070 }
1071
1072 void ResourceLoadStatisticsDatabaseStore::clearUserInteraction(const RegistrableDomain& domain)
1073 {
1074     ASSERT(!RunLoop::isMain());
1075
1076     auto targetResult = ensureResourceStatisticsForRegistrableDomain(domain);
1077     setUserInteraction(domain, false, { });
1078
1079     SQLiteStatement removeStorageAccess(m_database, "DELETE FROM StorageAccessUnderTopFrameDomains WHERE domainID = ?");
1080     if (removeStorageAccess.prepare() != SQLITE_OK
1081         || removeStorageAccess.bindInt(1, targetResult.second) != SQLITE_OK
1082         || removeStorageAccess.step() != SQLITE_DONE) {
1083         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::logUserInteraction failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
1084         ASSERT_NOT_REACHED();
1085     }
1086 }
1087
1088 bool ResourceLoadStatisticsDatabaseStore::hasHadUserInteraction(const RegistrableDomain& domain, OperatingDatesWindow operatingDatesWindow)
1089 {
1090     ASSERT(!RunLoop::isMain());
1091
1092     if (m_hadUserInteractionStatement.bindText(1, domain.string()) != SQLITE_OK
1093         || m_hadUserInteractionStatement.step() != SQLITE_ROW) {
1094         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_hadUserInteractionStatement failed, error message: %{private}s", this, m_database.lastErrorMsg());
1095
1096         int resetResult = m_hadUserInteractionStatement.reset();
1097         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1098         return false;
1099     }
1100
1101     bool hadUserInteraction = !!m_hadUserInteractionStatement.getColumnInt(0);
1102     if (!hadUserInteraction) {
1103         int resetResult = m_hadUserInteractionStatement.reset();
1104         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1105         return false;
1106     }
1107     
1108     WallTime mostRecentUserInteractionTime = WallTime::fromRawSeconds(m_hadUserInteractionStatement.getColumnDouble(1));
1109
1110     if (hasStatisticsExpired(mostRecentUserInteractionTime, operatingDatesWindow)) {
1111         // Drop privacy sensitive data because we no longer need it.
1112         // Set timestamp to 0 so that statistics merge will know
1113         // it has been reset as opposed to its default -1.
1114         clearUserInteraction(domain);
1115         hadUserInteraction = false;
1116     }
1117     
1118     int resetResult = m_hadUserInteractionStatement.reset();
1119     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1120
1121     return hadUserInteraction;
1122 }
1123
1124 void ResourceLoadStatisticsDatabaseStore::setPrevalentResource(const RegistrableDomain& domain, ResourceLoadPrevalence newPrevalence)
1125 {
1126     ASSERT(!RunLoop::isMain());
1127     if (shouldSkip(domain))
1128         return;
1129
1130     if (m_updatePrevalentResourceStatement.bindInt(1, 1) != SQLITE_OK
1131         || m_updatePrevalentResourceStatement.bindText(2, domain.string()) != SQLITE_OK
1132         || m_updatePrevalentResourceStatement.step() != SQLITE_DONE) {
1133         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_updatePrevalentResourceStatement failed, error message: %{private}s", this, m_database.lastErrorMsg());
1134         ASSERT_NOT_REACHED();
1135         return;
1136     }
1137     
1138     int resetResult = m_updatePrevalentResourceStatement.reset();
1139     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1140
1141     if (newPrevalence == ResourceLoadPrevalence::VeryHigh) {
1142         if (m_updateVeryPrevalentResourceStatement.bindInt(1, 1) != SQLITE_OK
1143             || m_updateVeryPrevalentResourceStatement.bindText(2, domain.string()) != SQLITE_OK
1144             || m_updateVeryPrevalentResourceStatement.step() != SQLITE_DONE) {
1145             RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::m_updateVeryPrevalentResourceStatement failed, error message: %{private}s", this, m_database.lastErrorMsg());
1146             ASSERT_NOT_REACHED();
1147             return;
1148         }
1149
1150         int resetResult = m_updateVeryPrevalentResourceStatement.reset();
1151         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1152     }
1153
1154     StdSet<unsigned> nonPrevalentRedirectionSources;
1155     recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(domainID(domain).value(), nonPrevalentRedirectionSources, 0);
1156     setDomainsAsPrevalent(WTFMove(nonPrevalentRedirectionSources));
1157 }
1158
1159 void ResourceLoadStatisticsDatabaseStore::setDomainsAsPrevalent(StdSet<unsigned>&& domains)
1160 {
1161     ASSERT(!RunLoop::isMain());
1162
1163     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET isPrevalent = 1 WHERE domainID IN (", buildList(WTF::IteratorRange<StdSet<unsigned>::iterator>(domains.begin(), domains.end())), ")"));
1164     if (domainsToUpdateStatement.prepare() != SQLITE_OK
1165         || domainsToUpdateStatement.step() != SQLITE_DONE) {
1166         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::setDomainsAsPrevalent failed, error message: %{private}s", this, m_database.lastErrorMsg());
1167         ASSERT_NOT_REACHED();
1168         return;
1169     }
1170 }
1171
1172 void ResourceLoadStatisticsDatabaseStore::dumpResourceLoadStatistics(CompletionHandler<void(const String&)>&& completionHandler)
1173 {
1174     ASSERT(!RunLoop::isMain());
1175
1176     // FIXME(195088): Implement SQLite-based dumping routines.
1177     ASSERT_NOT_REACHED();
1178     completionHandler({ });
1179 }
1180
1181 bool ResourceLoadStatisticsDatabaseStore::predicateValueForDomain(WebCore::SQLiteStatement& predicateStatement, const RegistrableDomain& domain) const
1182 {
1183     ASSERT(!RunLoop::isMain());
1184     
1185     if (predicateStatement.bindText(1, domain.string()) != SQLITE_OK
1186         || predicateStatement.step() != SQLITE_ROW) {
1187
1188         int resetResult = predicateStatement.reset();
1189         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1190
1191         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::predicateValueForDomain failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
1192         return false;
1193     }
1194
1195     bool result = !!predicateStatement.getColumnInt(0);
1196     int resetResult = predicateStatement.reset();
1197     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1198     return result;
1199 }
1200
1201 bool ResourceLoadStatisticsDatabaseStore::isPrevalentResource(const RegistrableDomain& domain) const
1202 {
1203     ASSERT(!RunLoop::isMain());
1204
1205     if (shouldSkip(domain))
1206         return false;
1207
1208     return predicateValueForDomain(m_isPrevalentResourceStatement, domain);
1209 }
1210
1211 bool ResourceLoadStatisticsDatabaseStore::isVeryPrevalentResource(const RegistrableDomain& domain) const
1212 {
1213     ASSERT(!RunLoop::isMain());
1214
1215     if (shouldSkip(domain))
1216         return false;
1217
1218     return predicateValueForDomain(m_isVeryPrevalentResourceStatement, domain);
1219 }
1220
1221 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsSubresourceUnder(const SubResourceDomain& subresourceDomain, const TopFrameDomain& topFrameDomain) const
1222 {
1223     ASSERT(!RunLoop::isMain());
1224
1225     return relationshipExists(m_subresourceUnderTopFrameDomainExists, domainID(subresourceDomain), topFrameDomain);
1226 }
1227
1228 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsSubFrameUnder(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain) const
1229 {
1230     ASSERT(!RunLoop::isMain());
1231
1232     return relationshipExists(m_subframeUnderTopFrameDomainExists, domainID(subFrameDomain), topFrameDomain);
1233 }
1234
1235 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsRedirectingTo(const RedirectedFromDomain& redirectedFromDomain, const RedirectedToDomain& redirectedToDomain) const
1236 {
1237     ASSERT(!RunLoop::isMain());
1238
1239     return relationshipExists(m_subresourceUniqueRedirectsToExists, domainID(redirectedFromDomain), redirectedToDomain);
1240 }
1241
1242 void ResourceLoadStatisticsDatabaseStore::clearPrevalentResource(const RegistrableDomain& domain)
1243 {
1244     ASSERT(!RunLoop::isMain());
1245
1246     ensureResourceStatisticsForRegistrableDomain(domain);
1247     
1248     if (m_clearPrevalentResourceStatement.bindText(1, domain.string()) != SQLITE_OK
1249         || m_clearPrevalentResourceStatement.step() != SQLITE_DONE) {
1250         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::clearPrevalentResource, error message: %{private}s", this, m_database.lastErrorMsg());
1251         ASSERT_NOT_REACHED();
1252         return;
1253     }
1254     
1255     int resetResult = m_clearPrevalentResourceStatement.reset();
1256     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1257 }
1258
1259 void ResourceLoadStatisticsDatabaseStore::setGrandfathered(const RegistrableDomain& domain, bool value)
1260 {
1261     ASSERT(!RunLoop::isMain());
1262
1263     ensureResourceStatisticsForRegistrableDomain(domain);
1264     
1265     if (m_updateGrandfatheredStatement.bindInt(1, value) != SQLITE_OK
1266         || m_updateGrandfatheredStatement.bindText(2, domain.string()) != SQLITE_OK
1267         || m_updateGrandfatheredStatement.step() != SQLITE_DONE) {
1268         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::setGrandfathered failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
1269         ASSERT_NOT_REACHED();
1270         return;
1271     }
1272     
1273     int resetResult = m_updateGrandfatheredStatement.reset();
1274     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1275 }
1276
1277 bool ResourceLoadStatisticsDatabaseStore::isGrandfathered(const RegistrableDomain& domain) const
1278 {
1279     ASSERT(!RunLoop::isMain());
1280
1281     return predicateValueForDomain(m_isGrandfatheredStatement, domain);
1282 }
1283
1284 void ResourceLoadStatisticsDatabaseStore::setSubframeUnderTopFrameDomain(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain)
1285 {
1286     ASSERT(!RunLoop::isMain());
1287
1288     auto result = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
1289
1290     // For consistency, make sure we also have a statistics entry for the top frame domain.
1291     insertDomainRelationshipList(subframeUnderTopFrameDomainsQuery, HashSet<RegistrableDomain>({ topFrameDomain }), result.second);
1292 }
1293
1294 void ResourceLoadStatisticsDatabaseStore::setSubresourceUnderTopFrameDomain(const SubResourceDomain& subresourceDomain, const TopFrameDomain& topFrameDomain)
1295 {
1296     ASSERT(!RunLoop::isMain());
1297
1298     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1299
1300     // For consistency, make sure we also have a statistics entry for the top frame domain.
1301     insertDomainRelationshipList(subresourceUnderTopFrameDomainsQuery, HashSet<RegistrableDomain>({ topFrameDomain }), result.second);
1302 }
1303
1304 void ResourceLoadStatisticsDatabaseStore::setSubresourceUniqueRedirectTo(const SubResourceDomain& subresourceDomain, const RedirectDomain& redirectDomain)
1305 {
1306     ASSERT(!RunLoop::isMain());
1307
1308     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1309
1310     // For consistency, make sure we also have a statistics entry for the redirect domain.
1311     insertDomainRelationshipList(subresourceUniqueRedirectsToQuery, HashSet<RegistrableDomain>({ redirectDomain }), result.second);
1312 }
1313
1314 void ResourceLoadStatisticsDatabaseStore::setSubresourceUniqueRedirectFrom(const SubResourceDomain& subresourceDomain, const RedirectDomain& redirectDomain)
1315 {
1316     ASSERT(!RunLoop::isMain());
1317
1318     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1319
1320     // For consistency, make sure we also have a statistics entry for the redirect domain.
1321     insertDomainRelationshipList(subresourceUniqueRedirectsFromQuery, HashSet<RegistrableDomain>({ redirectDomain }), result.second);
1322 }
1323
1324 void ResourceLoadStatisticsDatabaseStore::setTopFrameUniqueRedirectTo(const TopFrameDomain& topFrameDomain, const RedirectDomain& redirectDomain)
1325 {
1326     ASSERT(!RunLoop::isMain());
1327
1328     auto result = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1329
1330     // For consistency, make sure we also have a statistics entry for the redirect domain.
1331     insertDomainRelationshipList(topFrameUniqueRedirectsToQuery, HashSet<RegistrableDomain>({ redirectDomain }), result.second);
1332 }
1333
1334 void ResourceLoadStatisticsDatabaseStore::setTopFrameUniqueRedirectFrom(const TopFrameDomain& topFrameDomain, const RedirectDomain& redirectDomain)
1335 {
1336     ASSERT(!RunLoop::isMain());
1337
1338     auto result = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1339
1340     // For consistency, make sure we also have a statistics entry for the redirect domain.
1341     insertDomainRelationshipList(topFrameUniqueRedirectsFromQuery, HashSet<RegistrableDomain>({ redirectDomain }), result.second);
1342 }
1343
1344 std::pair<ResourceLoadStatisticsDatabaseStore::AddedRecord, unsigned> ResourceLoadStatisticsDatabaseStore::ensureResourceStatisticsForRegistrableDomain(const RegistrableDomain& domain)
1345 {
1346     ASSERT(!RunLoop::isMain());
1347
1348     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK) {
1349         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::ensureResourceStatisticsForRegistrableDomain failed, error message: %{private}s", this, m_database.lastErrorMsg());
1350         ASSERT_NOT_REACHED();
1351         return { AddedRecord::No, 0 };
1352     }
1353     
1354     if (m_domainIDFromStringStatement.step() == SQLITE_ROW) {
1355         unsigned domainID = m_domainIDFromStringStatement.getColumnInt(0);
1356
1357         int resetResult = m_domainIDFromStringStatement.reset();
1358         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1359         return { AddedRecord::No, domainID };
1360     }
1361
1362     int resetResult = m_domainIDFromStringStatement.reset();
1363     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1364
1365     ResourceLoadStatistics newObservation(domain);
1366     insertObservedDomain(newObservation);
1367
1368     return { AddedRecord::Yes, domainID(domain).value() };
1369 }
1370
1371 void ResourceLoadStatisticsDatabaseStore::clearDatabaseContents()
1372 {
1373     m_database.clearAllTables();
1374
1375     if (!createSchema()) {
1376         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::clearDatabaseContents failed, error message: %{private}s, database path: %{private}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
1377         ASSERT_NOT_REACHED();
1378         return;
1379     }
1380 }
1381
1382 void ResourceLoadStatisticsDatabaseStore::clear(CompletionHandler<void()>&& completionHandler)
1383 {
1384     ASSERT(!RunLoop::isMain());
1385
1386     clearDatabaseContents();
1387     clearOperatingDates();
1388
1389     auto callbackAggregator = CallbackAggregator::create(WTFMove(completionHandler));
1390
1391     removeAllStorageAccess([callbackAggregator = callbackAggregator.copyRef()] { });
1392
1393     auto registrableDomainsToBlockAndDeleteCookiesFor = ensurePrevalentResourcesForDebugMode();
1394     RegistrableDomainsToBlockCookiesFor domainsToBlock { registrableDomainsToBlockAndDeleteCookiesFor, { } };
1395     updateCookieBlockingForDomains(domainsToBlock, [callbackAggregator = callbackAggregator.copyRef()] { });
1396 }
1397
1398 ResourceLoadStatisticsDatabaseStore::CookieTreatmentResult ResourceLoadStatisticsDatabaseStore::cookieTreatmentForOrigin(const RegistrableDomain& domain) const
1399 {
1400     ASSERT(!RunLoop::isMain());
1401
1402     SQLiteStatement statement(m_database, "SELECT isPrevalent, hadUserInteraction FROM ObservedDomains WHERE registrableDomain = ?");
1403     if (statement.prepare() != SQLITE_OK
1404         || statement.bindText(1, domain.string()) != SQLITE_OK) {
1405         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::cookieTreatmentForOrigin failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
1406         ASSERT_NOT_REACHED();
1407     }
1408     
1409     if (statement.step() != SQLITE_ROW)
1410         return CookieTreatmentResult::Allow;
1411     
1412     bool isPrevalent = !!statement.getColumnInt(0);
1413     if (!isPrevalent)
1414         return CookieTreatmentResult::Allow;
1415
1416     bool hadUserInteraction = statement.getColumnInt(1) ? true : false;
1417     return hadUserInteraction ? CookieTreatmentResult::BlockAndKeep : CookieTreatmentResult::BlockAndPurge;
1418 }
1419     
1420 StorageAccessPromptWasShown ResourceLoadStatisticsDatabaseStore::hasUserGrantedStorageAccessThroughPrompt(unsigned requestingDomainID, const RegistrableDomain& firstPartyDomain) const
1421 {
1422     ASSERT(!RunLoop::isMain());
1423
1424     auto firstPartyPrimaryDomainID = domainID(firstPartyDomain).value();
1425
1426     SQLiteStatement statement(m_database, "SELECT COUNT(*) FROM StorageAccessUnderTopFrameDomains WHERE domainID = ? AND topLevelDomainID = ?");
1427     if (statement.prepare() != SQLITE_OK
1428         || statement.bindInt(1, requestingDomainID) != SQLITE_OK
1429         || statement.bindInt(2, firstPartyPrimaryDomainID) != SQLITE_OK
1430         || statement.step() != SQLITE_ROW)
1431         return StorageAccessPromptWasShown::No;
1432
1433     return !statement.getColumnInt(0) ? StorageAccessPromptWasShown::Yes : StorageAccessPromptWasShown::No;
1434 }
1435
1436 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::domainsToBlockAndDeleteCookiesFor() const
1437 {
1438     ASSERT(!RunLoop::isMain());
1439     
1440     Vector<RegistrableDomain> results;
1441     SQLiteStatement statement(m_database, "SELECT registrableDomain FROM ObservedDomains WHERE isPrevalent = 1 AND hadUserInteraction = 0"_s);
1442     if (statement.prepare() != SQLITE_OK)
1443         return results;
1444     
1445     while (statement.step() == SQLITE_ROW)
1446         results.append(RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(0)));
1447     
1448     return results;
1449 }
1450
1451 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::domainsToBlockButKeepCookiesFor() const
1452 {
1453     ASSERT(!RunLoop::isMain());
1454
1455     Vector<RegistrableDomain> results;
1456     SQLiteStatement statement(m_database, "SELECT registrableDomain FROM ObservedDomains WHERE isPrevalent = 1 AND hadUserInteraction = 1"_s);
1457     if (statement.prepare() != SQLITE_OK)
1458         return results;
1459     
1460     while (statement.step() == SQLITE_ROW)
1461         results.append(RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(0)));
1462
1463     return results;
1464 }
1465
1466 void ResourceLoadStatisticsDatabaseStore::updateCookieBlocking(CompletionHandler<void()>&& completionHandler)
1467 {
1468     ASSERT(!RunLoop::isMain());
1469
1470     auto domainsToBlockAndDeleteCookiesFor = this->domainsToBlockAndDeleteCookiesFor();
1471     auto domainsToBlockButKeepCookiesFor = this->domainsToBlockButKeepCookiesFor();
1472
1473     if (domainsToBlockAndDeleteCookiesFor.isEmpty() && domainsToBlockButKeepCookiesFor.isEmpty()) {
1474         completionHandler();
1475         return;
1476     }
1477
1478     RegistrableDomainsToBlockCookiesFor domainsToBlock { domainsToBlockAndDeleteCookiesFor, domainsToBlockButKeepCookiesFor };
1479
1480     if (debugLoggingEnabled() && !domainsToBlockAndDeleteCookiesFor.isEmpty() && !domainsToBlockButKeepCookiesFor.isEmpty())
1481         debugLogDomainsInBatches("block", domainsToBlock);
1482
1483     RunLoop::main().dispatch([weakThis = makeWeakPtr(*this), store = makeRef(store()), domainsToBlock = crossThreadCopy(domainsToBlock), completionHandler = WTFMove(completionHandler)] () mutable {
1484         store->callUpdatePrevalentDomainsToBlockCookiesForHandler(domainsToBlock, [weakThis = WTFMove(weakThis), store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1485             store->statisticsQueue().dispatch([weakThis = WTFMove(weakThis), completionHandler = WTFMove(completionHandler)]() mutable {
1486                 completionHandler();
1487                 if (!weakThis)
1488                     return;
1489                 RELEASE_LOG_INFO_IF(weakThis->debugLoggingEnabled(), ITPDebug, "Done updating cookie blocking.");
1490             });
1491         });
1492     });
1493 }
1494
1495 Vector<ResourceLoadStatisticsDatabaseStore::PrevalentDomainData> ResourceLoadStatisticsDatabaseStore::prevalentDomains() const
1496 {
1497     ASSERT(!RunLoop::isMain());
1498     
1499     Vector<PrevalentDomainData> results;
1500     SQLiteStatement statement(m_database, "SELECT domainID, registrableDomain, mostRecentUserInteractionTime, hadUserInteraction, grandfathered FROM ObservedDomains WHERE isPrevalent = 1"_s);
1501     if (statement.prepare() != SQLITE_OK)
1502         return results;
1503     
1504     while (statement.step() == SQLITE_ROW) {
1505         results.append({ static_cast<unsigned>(statement.getColumnInt(0))
1506             , RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(1))
1507             , WallTime::fromRawSeconds(statement.getColumnDouble(2))
1508             , statement.getColumnInt(3) ? true : false
1509             , statement.getColumnInt(4) ? true : false
1510         });
1511     }
1512     
1513     return results;
1514 }
1515
1516 Vector<unsigned> ResourceLoadStatisticsDatabaseStore::findExpiredUserInteractions() const
1517 {
1518     ASSERT(!RunLoop::isMain());
1519
1520     Vector<unsigned> results;
1521     Optional<Seconds> expirationDateTime = statisticsEpirationTime();
1522     if (!expirationDateTime)
1523         return results;
1524
1525     if (m_findExpiredUserInteractionStatement.bindDouble(1, expirationDateTime.value().value()) != SQLITE_OK)
1526         return results;
1527
1528     while (m_findExpiredUserInteractionStatement.step() == SQLITE_ROW)
1529         results.append(m_findExpiredUserInteractionStatement.getColumnInt(0));
1530
1531     int resetResult = m_findExpiredUserInteractionStatement.reset();
1532     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1533
1534     return results;
1535 }
1536
1537 void ResourceLoadStatisticsDatabaseStore::clearExpiredUserInteractions()
1538 {
1539     ASSERT(!RunLoop::isMain());
1540
1541     auto expiredRecords = findExpiredUserInteractions();
1542     if (expiredRecords.isEmpty())
1543         return;
1544
1545     auto expiredRecordIDs = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(expiredRecords.begin(), expiredRecords.end()));
1546
1547     SQLiteStatement clearExpiredInteraction(m_database, makeString("UPDATE ObservedDomains SET mostRecentUserInteractionTime = 0, hadUserInteraction = 1 WHERE domainID IN (", expiredRecordIDs, ")"));
1548     if (clearExpiredInteraction.prepare() != SQLITE_OK)
1549         return;
1550
1551     SQLiteStatement removeStorageAccess(m_database, makeString("DELETE FROM StorageAccessUnderTopFrameDomains ", expiredRecordIDs, ")"));
1552     if (removeStorageAccess.prepare() != SQLITE_OK)
1553         return;
1554
1555     if (clearExpiredInteraction.step() != SQLITE_DONE
1556         || removeStorageAccess.step() != SQLITE_DONE) {
1557         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::clearExpiredUserInteractions statement(s) failed to step, error message: %{private}s", this, m_database.lastErrorMsg());
1558         ASSERT_NOT_REACHED();
1559     }
1560 }
1561
1562 void ResourceLoadStatisticsDatabaseStore::clearGrandfathering(Vector<unsigned>&& domainIDsToClear)
1563 {
1564     ASSERT(!RunLoop::isMain());
1565
1566     if (domainIDsToClear.isEmpty())
1567         return;
1568
1569     auto listToClear = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(domainIDsToClear.begin(), domainIDsToClear.end()));
1570
1571     SQLiteStatement clearGrandfatheringStatement(m_database, makeString("UPDATE ObservedDomains SET grandfathered = 0 WHERE domainID IN (", listToClear, ")"));
1572     if (clearGrandfatheringStatement.prepare() != SQLITE_OK)
1573         return;
1574     
1575     if (clearGrandfatheringStatement.step() != SQLITE_DONE) {
1576         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::clearGrandfathering failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
1577         ASSERT_NOT_REACHED();
1578     }
1579 }
1580
1581 bool ResourceLoadStatisticsDatabaseStore::hasHadUnexpiredRecentUserInteraction(const PrevalentDomainData& resourceStatistic, OperatingDatesWindow operatingDatesWindow)
1582 {
1583     if (resourceStatistic.hadUserInteraction && hasStatisticsExpired(resourceStatistic.mostRecentUserInteractionTime, operatingDatesWindow)) {
1584         clearUserInteraction(resourceStatistic.registrableDomain);
1585         return false;
1586     }
1587
1588     return resourceStatistic.hadUserInteraction;
1589 }
1590
1591 bool ResourceLoadStatisticsDatabaseStore::shouldRemoveAllWebsiteDataFor(const PrevalentDomainData& resourceStatistic, bool shouldCheckForGrandfathering)
1592 {
1593     return !hasHadUnexpiredRecentUserInteraction(resourceStatistic, OperatingDatesWindow::Long) && (!shouldCheckForGrandfathering || !resourceStatistic.grandfathered);
1594 }
1595
1596 bool ResourceLoadStatisticsDatabaseStore::shouldRemoveAllButCookiesFor(const PrevalentDomainData& resourceStatistic, bool shouldCheckForGrandfathering) const
1597 {
1598     UNUSED_PARAM(resourceStatistic);
1599     UNUSED_PARAM(shouldCheckForGrandfathering);
1600     return false;
1601 }
1602
1603 Vector<std::pair<RegistrableDomain, WebsiteDataToRemove>> ResourceLoadStatisticsDatabaseStore::registrableDomainsToRemoveWebsiteDataFor()
1604 {
1605     ASSERT(!RunLoop::isMain());
1606
1607     bool shouldCheckForGrandfathering = endOfGrandfatheringTimestamp() > WallTime::now();
1608     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && endOfGrandfatheringTimestamp();
1609
1610     if (shouldClearGrandfathering)
1611         clearEndOfGrandfatheringTimeStamp();
1612
1613     clearExpiredUserInteractions();
1614     
1615     Vector<std::pair<RegistrableDomain, WebsiteDataToRemove>> domainsToRemoveWebsiteDataFor;
1616
1617     Vector<PrevalentDomainData> prevalentDomains = this->prevalentDomains();
1618     Vector<unsigned> domainIDsToClearGrandfathering;
1619     for (auto& statistic : prevalentDomains) {
1620         if (shouldRemoveAllWebsiteDataFor(statistic, shouldCheckForGrandfathering))
1621             domainsToRemoveWebsiteDataFor.append(std::make_pair(statistic.registrableDomain, WebsiteDataToRemove::All));
1622         else if (shouldRemoveAllButCookiesFor(statistic, shouldCheckForGrandfathering))
1623             domainsToRemoveWebsiteDataFor.append(std::make_pair(statistic.registrableDomain, WebsiteDataToRemove::AllButCookies));
1624
1625         if (shouldClearGrandfathering && statistic.grandfathered)
1626             domainIDsToClearGrandfathering.append(statistic.domainID);
1627     }
1628
1629     clearGrandfathering(WTFMove(domainIDsToClearGrandfathering));
1630     
1631     return domainsToRemoveWebsiteDataFor;
1632 }
1633
1634 void ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded()
1635 {
1636     ASSERT(!RunLoop::isMain());
1637
1638     unsigned count = 0;
1639     if (m_observedDomainCount.step() == SQLITE_ROW)
1640         count = m_observedDomainCount.getColumnInt(0);
1641
1642     int resetResult = m_observedDomainCount.reset();
1643     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1644
1645     if (count <= parameters().maxStatisticsEntries)
1646         return;
1647
1648     ASSERT(parameters().pruneEntriesDownTo <= parameters().maxStatisticsEntries);
1649
1650     size_t countLeftToPrune = count - parameters().pruneEntriesDownTo;
1651     ASSERT(countLeftToPrune);
1652
1653     SQLiteStatement recordsToPrune(m_database, "SELECT domainID FROM ObservedDomains ORDER BY hadUserInteraction, isPrevalent, lastSeen LIMIT ?");
1654     if (recordsToPrune.prepare() != SQLITE_OK
1655         || recordsToPrune.bindInt(1, countLeftToPrune) != SQLITE_OK) {
1656         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded failed, error message: %{private}s", this, m_database.lastErrorMsg());
1657         ASSERT_NOT_REACHED();
1658         return;
1659     }
1660
1661     Vector<unsigned> entriesToPrune;
1662     while (recordsToPrune.step() == SQLITE_ROW)
1663         entriesToPrune.append(recordsToPrune.getColumnInt(0));
1664
1665     auto listToPrune = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(entriesToPrune.begin(), entriesToPrune.end()));
1666
1667     SQLiteStatement pruneCommand(m_database, makeString("DELETE from ObservedDomains WHERE domainID IN (", listToPrune, ")"));
1668     if (pruneCommand.prepare() != SQLITE_OK
1669         || pruneCommand.step() != SQLITE_DONE) {
1670         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded failed, error message: %{private}s", this, m_database.lastErrorMsg());
1671         ASSERT_NOT_REACHED();
1672         return;
1673     }
1674 }
1675
1676 void ResourceLoadStatisticsDatabaseStore::updateLastSeen(const RegistrableDomain& domain, WallTime lastSeen)
1677 {
1678     ASSERT(!RunLoop::isMain());
1679
1680     if (m_updateLastSeenStatement.bindDouble(1, lastSeen.secondsSinceEpoch().value()) != SQLITE_OK
1681         || m_updateLastSeenStatement.bindText(2, domain.string()) != SQLITE_OK
1682         || m_updateLastSeenStatement.step() != SQLITE_DONE) {
1683         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::updateLastSeen failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
1684         ASSERT_NOT_REACHED();
1685         return;
1686     }
1687     
1688     int resetResult = m_updateLastSeenStatement.reset();
1689     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1690 }
1691
1692 void ResourceLoadStatisticsDatabaseStore::setLastSeen(const RegistrableDomain& domain, Seconds seconds)
1693 {
1694     ASSERT(!RunLoop::isMain());
1695
1696     ensureResourceStatisticsForRegistrableDomain(domain);
1697     updateLastSeen(domain, WallTime::fromRawSeconds(seconds.seconds()));
1698 }
1699
1700 void ResourceLoadStatisticsDatabaseStore::setPrevalentResource(const RegistrableDomain& domain)
1701 {
1702     ASSERT(!RunLoop::isMain());
1703
1704     if (shouldSkip(domain))
1705         return;
1706
1707     ensureResourceStatisticsForRegistrableDomain(domain);
1708     setPrevalentResource(domain, ResourceLoadPrevalence::High);
1709 }
1710
1711 void ResourceLoadStatisticsDatabaseStore::setVeryPrevalentResource(const RegistrableDomain& domain)
1712 {
1713     ASSERT(!RunLoop::isMain());
1714
1715     if (shouldSkip(domain))
1716         return;
1717     
1718     ensureResourceStatisticsForRegistrableDomain(domain);
1719     setPrevalentResource(domain, ResourceLoadPrevalence::VeryHigh);
1720 }
1721
1722 void ResourceLoadStatisticsDatabaseStore::updateDataRecordsRemoved(const RegistrableDomain& domain, int value)
1723 {
1724     ASSERT(!RunLoop::isMain());
1725
1726     if (m_updateDataRecordsRemovedStatement.bindInt(1, value) != SQLITE_OK
1727         || m_updateDataRecordsRemovedStatement.bindText(2, domain.string()) != SQLITE_OK
1728         || m_updateDataRecordsRemovedStatement.step() != SQLITE_DONE) {
1729         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::updateDataRecordsRemoved failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
1730         ASSERT_NOT_REACHED();
1731         return;
1732     }
1733
1734     int resetResult = m_updateDataRecordsRemovedStatement.reset();
1735     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1736 }
1737
1738 bool ResourceLoadStatisticsDatabaseStore::isCorrectSubStatisticsCount(const RegistrableDomain& subframeDomain, const TopFrameDomain& topFrameDomain)
1739 {
1740     SQLiteStatement subFrameUnderTopFrameCount(m_database, countSubframeUnderTopFrameQuery);
1741     SQLiteStatement subresourceUnderTopFrameCount(m_database, countSubresourceUnderTopFrameQuery);
1742     SQLiteStatement subresourceUniqueRedirectsTo(m_database, countSubresourceUniqueRedirectsToQuery);
1743     
1744     if (subFrameUnderTopFrameCount.prepare() != SQLITE_OK
1745         || subresourceUnderTopFrameCount.prepare() != SQLITE_OK
1746         || subresourceUniqueRedirectsTo.prepare() != SQLITE_OK) {
1747         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::countSubStatisticsTesting failed to prepare, error message: %{private}s", this, m_database.lastErrorMsg());
1748         ASSERT_NOT_REACHED();
1749         return false;
1750     }
1751     
1752     if (subFrameUnderTopFrameCount.bindInt(1, domainID(subframeDomain).value()) != SQLITE_OK
1753         || subFrameUnderTopFrameCount.bindInt(2, domainID(topFrameDomain).value()) != SQLITE_OK
1754         || subresourceUnderTopFrameCount.bindInt(1, domainID(subframeDomain).value()) != SQLITE_OK
1755         || subresourceUnderTopFrameCount.bindInt(2, domainID(topFrameDomain).value()) != SQLITE_OK
1756         || subresourceUniqueRedirectsTo.bindInt(1, domainID(subframeDomain).value()) != SQLITE_OK
1757         || subresourceUniqueRedirectsTo.bindInt(2, domainID(topFrameDomain).value()) != SQLITE_OK) {
1758         RELEASE_LOG_ERROR_IF_ALLOWED(m_sessionID, "%p - ResourceLoadStatisticsDatabaseStore::countSubStatisticsTesting failed to bind, error message: %{private}s", this, m_database.lastErrorMsg());
1759         ASSERT_NOT_REACHED();
1760         return false;
1761     }
1762     
1763     if (subFrameUnderTopFrameCount.step() != SQLITE_ROW
1764         || subresourceUnderTopFrameCount.step() != SQLITE_ROW
1765         || subresourceUniqueRedirectsTo.step() != SQLITE_ROW)
1766         return false;
1767     
1768     return (subFrameUnderTopFrameCount.getColumnInt(0) == 1 && subresourceUnderTopFrameCount.getColumnInt(0) == 1 && subresourceUniqueRedirectsTo.getColumnInt(0) == 1);
1769 }
1770
1771
1772 } // namespace WebKit
1773
1774 #endif