c6b845f44a20bc27aff0385c725b5df95cf22e1e
[WebKit-https.git] / Source / WebKit / NetworkProcess / Classifier / ResourceLoadStatisticsDatabaseStore.cpp
1 /*
2  * Copyright (C) 2019 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ResourceLoadStatisticsDatabaseStore.h"
28
29 #if ENABLE(RESOURCE_LOAD_STATISTICS)
30
31 #include "Logging.h"
32 #include "NetworkSession.h"
33 #include "PluginProcessManager.h"
34 #include "PluginProcessProxy.h"
35 #include "ResourceLoadStatisticsMemoryStore.h"
36 #include "StorageAccessStatus.h"
37 #include "WebProcessProxy.h"
38 #include "WebResourceLoadStatisticsTelemetry.h"
39 #include "WebsiteDataStore.h"
40 #include <WebCore/DocumentStorageAccess.h>
41 #include <WebCore/KeyedCoding.h>
42 #include <WebCore/NetworkStorageSession.h>
43 #include <WebCore/ResourceLoadStatistics.h>
44 #include <WebCore/SQLiteDatabase.h>
45 #include <WebCore/SQLiteStatement.h>
46 #include <wtf/CallbackAggregator.h>
47 #include <wtf/DateMath.h>
48 #include <wtf/HashMap.h>
49 #include <wtf/MathExtras.h>
50 #include <wtf/StdSet.h>
51 #include <wtf/text/StringBuilder.h>
52
53 namespace WebKit {
54 using namespace WebCore;
55
56 #if PLATFORM(COCOA)
57 #define RELEASE_LOG_IF_ALLOWED(sessionID, fmt, ...) RELEASE_LOG_IF(sessionID.isAlwaysOnLoggingAllowed(), Network, "%p - ResourceLoadStatisticsDatabaseStore::" fmt, this, ##__VA_ARGS__)
58 #define RELEASE_LOG_ERROR_IF_ALLOWED(sessionID, fmt, ...) RELEASE_LOG_ERROR_IF(sessionID.isAlwaysOnLoggingAllowed(), Network, "%p - ResourceLoadStatisticsDatabaseStore::" fmt, this, ##__VA_ARGS__)
59 #else
60 #define RELEASE_LOG_IF_ALLOWED(sessionID, fmt, ...)  ((void)0)
61 #define RELEASE_LOG_ERROR_IF_ALLOWED(sessionID, fmt, ...)  ((void)0)
62 #endif
63
64 constexpr auto observedDomainCountQuery = "SELECT COUNT(*) FROM ObservedDomains"_s;
65 constexpr auto insertObservedDomainQuery = "INSERT INTO ObservedDomains (registrableDomain, lastSeen, hadUserInteraction,"
66     "mostRecentUserInteractionTime, grandfathered, isPrevalent, isVeryPrevalent, dataRecordsRemoved, timesAccessedAsFirstPartyDueToUserInteraction,"
67     "timesAccessedAsFirstPartyDueToStorageAccessAPI) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"_s;
68 constexpr auto insertTopLevelDomainQuery = "INSERT INTO TopLevelDomains VALUES (?)"_s;
69 constexpr auto domainIDFromStringQuery = "SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
70 constexpr auto storageAccessUnderTopFrameDomainsQuery = "INSERT INTO StorageAccessUnderTopFrameDomains (domainID, topLevelDomainID) "
71     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
72 constexpr auto topFrameUniqueRedirectsToQuery = "INSERT INTO TopFrameUniqueRedirectsTo (sourceDomainID, toDomainID) SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
73 constexpr auto topFrameUniqueRedirectsToExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameUniqueRedirectsTo WHERE sourceDomainID = ? "
74     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
75 constexpr auto topFrameUniqueRedirectsFromQuery = "INSERT INTO TopFrameUniqueRedirectsFrom (targetDomainID, fromDomainID) "
76     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
77 constexpr auto topFrameUniqueRedirectsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameUniqueRedirectsFrom WHERE targetDomainID = ? "
78     "AND fromDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
79 constexpr auto topFrameLinkDecorationsFromQuery = "INSERT INTO TopFrameLinkDecorationsFrom (fromDomainID, toDomainID) "
80     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
81 constexpr auto topFrameLinkDecorationsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM TopFrameLinkDecorationsFrom WHERE fromDomainID = ? "
82     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
83 constexpr auto subframeUnderTopFrameDomainsQuery = "INSERT INTO SubframeUnderTopFrameDomains (subFrameDomainID, topFrameDomainID) "
84     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
85 constexpr auto subframeUnderTopFrameDomainExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubframeUnderTopFrameDomains WHERE subFrameDomainID = ? "
86     "AND topFrameDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
87 constexpr auto subresourceUnderTopFrameDomainsQuery = "INSERT INTO SubresourceUnderTopFrameDomains (subresourceDomainID, topFrameDomainID) "
88     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain = ?"_s;
89 constexpr auto subresourceUnderTopFrameDomainExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUnderTopFrameDomains "
90     "WHERE subresourceDomainID = ? AND topFrameDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
91 constexpr auto subresourceUniqueRedirectsToQuery = "INSERT INTO SubresourceUniqueRedirectsTo (subresourceDomainID, toDomainID) "
92     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
93 constexpr auto subresourceUniqueRedirectsToExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUniqueRedirectsTo WHERE subresourceDomainID = ? "
94     "AND toDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
95 constexpr auto subresourceUniqueRedirectsFromQuery = "INSERT INTO SubresourceUniqueRedirectsFrom (subresourceDomainID, fromDomainID) "
96     "SELECT ?, domainID FROM ObservedDomains WHERE registrableDomain == ?"_s;
97 constexpr auto subresourceUniqueRedirectsFromExistsQuery = "SELECT EXISTS (SELECT 1 FROM SubresourceUniqueRedirectsFrom WHERE subresourceDomainID = ? "
98     "AND fromDomainID = (SELECT domainID FROM ObservedDomains WHERE registrableDomain = ?))"_s;
99 constexpr auto mostRecentUserInteractionQuery = "UPDATE ObservedDomains SET hadUserInteraction = ?, mostRecentUserInteractionTime = ? "
100     "WHERE registrableDomain = ?"_s;
101 constexpr auto updateLastSeenQuery = "UPDATE ObservedDomains SET lastSeen = ? WHERE registrableDomain = ?"_s;
102 constexpr auto updatePrevalentResourceQuery = "UPDATE ObservedDomains SET isPrevalent = ? WHERE registrableDomain = ?"_s;
103 constexpr auto isPrevalentResourceQuery = "SELECT isPrevalent FROM ObservedDomains WHERE registrableDomain = ?"_s;
104 constexpr auto updateVeryPrevalentResourceQuery = "UPDATE ObservedDomains SET isVeryPrevalent = ? WHERE registrableDomain = ?"_s;
105 constexpr auto isVeryPrevalentResourceQuery = "SELECT isVeryPrevalent FROM ObservedDomains WHERE registrableDomain = ?"_s;
106 constexpr auto clearPrevalentResourceQuery = "UPDATE ObservedDomains SET isPrevalent = 0, isVeryPrevalent = 0 WHERE registrableDomain = ?"_s;
107 constexpr auto hadUserInteractionQuery = "SELECT hadUserInteraction, mostRecentUserInteractionTime FROM ObservedDomains WHERE registrableDomain = ?"_s;
108 constexpr auto updateGrandfatheredQuery = "UPDATE ObservedDomains SET grandfathered = ? WHERE registrableDomain = ?"_s;
109 constexpr auto isGrandfatheredQuery = "SELECT grandfathered FROM ObservedDomains WHERE registrableDomain = ?"_s;
110 constexpr auto findExpiredUserInteractionQuery = "SELECT domainID FROM ObservedDomains WHERE hadUserInteraction = 1 AND mostRecentUserInteractionTime < ?"_s;
111
112 constexpr auto createObservedDomain = "CREATE TABLE ObservedDomains ("
113     "domainID INTEGER PRIMARY KEY, registrableDomain TEXT NOT NULL UNIQUE ON CONFLICT FAIL, lastSeen REAL NOT NULL, "
114     "hadUserInteraction INTEGER NOT NULL, mostRecentUserInteractionTime REAL NOT NULL, grandfathered INTEGER NOT NULL, "
115     "isPrevalent INTEGER NOT NULL, isVeryPrevalent INTEGER NOT NULL, dataRecordsRemoved INTEGER NOT NULL,"
116     "timesAccessedAsFirstPartyDueToUserInteraction INTEGER NOT NULL, timesAccessedAsFirstPartyDueToStorageAccessAPI INTEGER NOT NULL);"_s;
117     
118 constexpr auto createTopLevelDomains = "CREATE TABLE TopLevelDomains ("
119     "topLevelDomainID INTEGER PRIMARY KEY, CONSTRAINT fkDomainID FOREIGN KEY(topLevelDomainID) "
120     "REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
121     
122 constexpr auto createStorageAccessUnderTopFrameDomains = "CREATE TABLE StorageAccessUnderTopFrameDomains ("
123     "domainID INTEGER NOT NULL, topLevelDomainID INTEGER NOT NULL ON CONFLICT FAIL, "
124     "CONSTRAINT fkDomainID FOREIGN KEY(domainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
125     "FOREIGN KEY(topLevelDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
126     
127 constexpr auto createTopFrameUniqueRedirectsTo = "CREATE TABLE TopFrameUniqueRedirectsTo ("
128     "sourceDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
129     "FOREIGN KEY(sourceDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
130     "FOREIGN KEY(toDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
131     
132 constexpr auto createTopFrameUniqueRedirectsFrom = "CREATE TABLE TopFrameUniqueRedirectsFrom ("
133     "targetDomainID INTEGER NOT NULL, fromDomainID INTEGER NOT NULL, "
134     "FOREIGN KEY(targetDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
135     "FOREIGN KEY(fromDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
136
137 constexpr auto createTopFrameLinkDecorationsFrom = "CREATE TABLE TopFrameLinkDecorationsFrom ("
138     "fromDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
139     "FOREIGN KEY(fromDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE, "
140     "FOREIGN KEY(toDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
141
142 constexpr auto createSubframeUnderTopFrameDomains = "CREATE TABLE SubframeUnderTopFrameDomains ("
143     "subFrameDomainID INTEGER NOT NULL, topFrameDomainID INTEGER NOT NULL, "
144     "FOREIGN KEY(subFrameDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
145     "FOREIGN KEY(topFrameDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
146     
147 constexpr auto createSubresourceUnderTopFrameDomains = "CREATE TABLE SubresourceUnderTopFrameDomains ("
148     "subresourceDomainID INTEGER NOT NULL, topFrameDomainID INTEGER NOT NULL, "
149     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
150     "FOREIGN KEY(topFrameDomainID) REFERENCES TopLevelDomains(topLevelDomainID) ON DELETE CASCADE);"_s;
151     
152 constexpr auto createSubresourceUniqueRedirectsTo = "CREATE TABLE SubresourceUniqueRedirectsTo ("
153     "subresourceDomainID INTEGER NOT NULL, toDomainID INTEGER NOT NULL, "
154     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
155     "FOREIGN KEY(toDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
156     
157 constexpr auto createSubresourceUniqueRedirectsFrom = "CREATE TABLE SubresourceUniqueRedirectsFrom ("
158     "subresourceDomainID INTEGER NOT NULL, fromDomainID INTEGER NOT NULL, "
159     "FOREIGN KEY(subresourceDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE, "
160     "FOREIGN KEY(fromDomainID) REFERENCES ObservedDomains(domainID) ON DELETE CASCADE);"_s;
161     
162 ResourceLoadStatisticsDatabaseStore::ResourceLoadStatisticsDatabaseStore(WebResourceLoadStatisticsStore& store, WorkQueue& workQueue, ShouldIncludeLocalhost shouldIncludeLocalhost, const String& storageDirectoryPath)
163     : ResourceLoadStatisticsStore(store, workQueue, shouldIncludeLocalhost)
164     , m_storageDirectoryPath(storageDirectoryPath + "/observations.db")
165     , m_observedDomainCount(m_database, observedDomainCountQuery)
166     , m_insertObservedDomainStatement(m_database, insertObservedDomainQuery)
167     , m_insertTopLevelDomainStatement(m_database, insertTopLevelDomainQuery)
168     , m_domainIDFromStringStatement(m_database, domainIDFromStringQuery)
169     , m_storageAccessUnderTopFrameDomainsStatement(m_database, storageAccessUnderTopFrameDomainsQuery)
170     , m_topFrameUniqueRedirectsTo(m_database, topFrameUniqueRedirectsToQuery)
171     , m_topFrameUniqueRedirectsToExists(m_database, topFrameUniqueRedirectsToExistsQuery)
172     , m_topFrameUniqueRedirectsFrom(m_database, topFrameUniqueRedirectsFromQuery)
173     , m_topFrameUniqueRedirectsFromExists(m_database, topFrameUniqueRedirectsFromExistsQuery)
174     , m_topFrameLinkDecorationsFrom(m_database, topFrameLinkDecorationsFromQuery)
175     , m_topFrameLinkDecorationsFromExists(m_database, topFrameLinkDecorationsFromExistsQuery)
176     , m_subframeUnderTopFrameDomains(m_database, subframeUnderTopFrameDomainsQuery)
177     , m_subframeUnderTopFrameDomainExists(m_database, subframeUnderTopFrameDomainExistsQuery)
178     , m_subresourceUnderTopFrameDomains(m_database, subresourceUnderTopFrameDomainsQuery)
179     , m_subresourceUnderTopFrameDomainExists(m_database, subresourceUnderTopFrameDomainExistsQuery)
180     , m_subresourceUniqueRedirectsTo(m_database, subresourceUniqueRedirectsToQuery)
181     , m_subresourceUniqueRedirectsToExists(m_database, subresourceUniqueRedirectsToExistsQuery)
182     , m_subresourceUniqueRedirectsFrom(m_database, subresourceUniqueRedirectsFromQuery)
183     , m_subresourceUniqueRedirectsFromExists(m_database, subresourceUniqueRedirectsFromExistsQuery)
184     , m_mostRecentUserInteractionStatement(m_database, mostRecentUserInteractionQuery)
185     , m_updateLastSeenStatement(m_database, updateLastSeenQuery)
186     , m_updatePrevalentResourceStatement(m_database, updatePrevalentResourceQuery)
187     , m_isPrevalentResourceStatement(m_database, isPrevalentResourceQuery)
188     , m_updateVeryPrevalentResourceStatement(m_database, updateVeryPrevalentResourceQuery)
189     , m_isVeryPrevalentResourceStatement(m_database, isVeryPrevalentResourceQuery)
190     , m_clearPrevalentResourceStatement(m_database, clearPrevalentResourceQuery)
191     , m_hadUserInteractionStatement(m_database, hadUserInteractionQuery)
192     , m_updateGrandfatheredStatement(m_database, updateGrandfatheredQuery)
193     , m_isGrandfatheredStatement(m_database, isGrandfatheredQuery)
194     , m_findExpiredUserInteractionStatement(m_database, findExpiredUserInteractionQuery)
195 {
196     ASSERT(!RunLoop::isMain());
197
198     if (!m_database.open(m_storageDirectoryPath)) {
199         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::open failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
200         ASSERT_NOT_REACHED();
201         return;
202     }
203     
204     // Since we are using a workerQueue, the sequential dispatch blocks may be called by different threads.
205     m_database.disableThreadingChecks();
206
207     if (!m_database.tableExists("ObservedDomains"_s)) {
208         if (!createSchema()) {
209             RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::createSchema failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
210             ASSERT_NOT_REACHED();
211             return;
212         }
213     }
214
215     if (!prepareStatements()) {
216         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::prepareStatements failed, error message: %{public}s, database path: %{public}s", this, m_database.lastErrorMsg(), m_storageDirectoryPath.utf8().data());
217         ASSERT_NOT_REACHED();
218         return;
219     }
220
221     workQueue.dispatchAfter(5_s, [weakThis = makeWeakPtr(*this)] {
222         if (weakThis)
223             weakThis->calculateAndSubmitTelemetry();
224     });
225 }
226
227 bool ResourceLoadStatisticsDatabaseStore::isEmpty() const
228 {
229     ASSERT(!RunLoop::isMain());
230     
231     bool result = false;
232     if (m_observedDomainCount.step() == SQLITE_ROW)
233         result = !m_observedDomainCount.getColumnInt(0);
234     
235     int resetResult = m_observedDomainCount.reset();
236     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
237     
238     return result;
239 }
240
241 bool ResourceLoadStatisticsDatabaseStore::createSchema()
242 {
243     ASSERT(!RunLoop::isMain());
244
245     if (!m_database.executeCommand(createObservedDomain)) {
246         LOG_ERROR("Could not create ObservedDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
247         return false;
248     }
249
250     if (!m_database.executeCommand(createTopLevelDomains)) {
251         LOG_ERROR("Could not create TopLevelDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
252         return false;
253     }
254
255     if (!m_database.executeCommand(createStorageAccessUnderTopFrameDomains)) {
256         LOG_ERROR("Could not create StorageAccessUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
257         return false;
258     }
259
260     if (!m_database.executeCommand(createTopFrameUniqueRedirectsTo)) {
261         LOG_ERROR("Could not create TopFrameUniqueRedirectsTo table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
262         return false;
263     }
264
265     if (!m_database.executeCommand(createTopFrameUniqueRedirectsFrom)) {
266         LOG_ERROR("Could not create TopFrameUniqueRedirectsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
267         return false;
268     }
269
270     if (!m_database.executeCommand(createTopFrameLinkDecorationsFrom)) {
271         LOG_ERROR("Could not create TopFrameLinkDecorationsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
272         return false;
273     }
274     
275     if (!m_database.executeCommand(createSubframeUnderTopFrameDomains)) {
276         LOG_ERROR("Could not create SubframeUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
277         return false;
278     }
279
280     if (!m_database.executeCommand(createSubresourceUnderTopFrameDomains)) {
281         LOG_ERROR("Could not create SubresourceUnderTopFrameDomains table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
282         return false;
283     }
284
285     if (!m_database.executeCommand(createSubresourceUniqueRedirectsTo)) {
286         LOG_ERROR("Could not create SubresourceUniqueRedirectsTo table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
287         return false;
288     }
289
290     if (!m_database.executeCommand(createSubresourceUniqueRedirectsFrom)) {
291         LOG_ERROR("Could not create SubresourceUniqueRedirectsFrom table in database (%i) - %s", m_database.lastError(), m_database.lastErrorMsg());
292         return false;
293     }
294
295     return true;
296 }
297
298 bool ResourceLoadStatisticsDatabaseStore::prepareStatements()
299 {
300     ASSERT(!RunLoop::isMain());
301         
302     if (m_observedDomainCount.prepare() != SQLITE_OK
303         || m_insertObservedDomainStatement.prepare() != SQLITE_OK
304         || m_insertTopLevelDomainStatement.prepare() != SQLITE_OK
305         || m_domainIDFromStringStatement.prepare() != SQLITE_OK
306         || m_storageAccessUnderTopFrameDomainsStatement.prepare() != SQLITE_OK
307         || m_topFrameUniqueRedirectsTo.prepare() != SQLITE_OK
308         || m_topFrameUniqueRedirectsToExists.prepare() != SQLITE_OK
309         || m_topFrameUniqueRedirectsFrom.prepare() != SQLITE_OK
310         || m_topFrameUniqueRedirectsFromExists.prepare() != SQLITE_OK
311         || m_subframeUnderTopFrameDomains.prepare() != SQLITE_OK
312         || m_subframeUnderTopFrameDomainExists.prepare() != SQLITE_OK
313         || m_subresourceUnderTopFrameDomains.prepare() != SQLITE_OK
314         || m_subresourceUnderTopFrameDomainExists.prepare() != SQLITE_OK
315         || m_subresourceUniqueRedirectsTo.prepare() != SQLITE_OK
316         || m_subresourceUniqueRedirectsToExists.prepare() != SQLITE_OK
317         || m_subresourceUniqueRedirectsFrom.prepare() != SQLITE_OK
318         || m_subresourceUniqueRedirectsFromExists.prepare() != SQLITE_OK
319         || m_updateLastSeenStatement.prepare() != SQLITE_OK
320         || m_mostRecentUserInteractionStatement.prepare() != SQLITE_OK
321         || m_updatePrevalentResourceStatement.prepare() != SQLITE_OK
322         || m_isPrevalentResourceStatement.prepare() != SQLITE_OK
323         || m_updateVeryPrevalentResourceStatement.prepare() != SQLITE_OK
324         || m_isVeryPrevalentResourceStatement.prepare() != SQLITE_OK
325         || m_clearPrevalentResourceStatement.prepare() != SQLITE_OK
326         || m_hadUserInteractionStatement.prepare() != SQLITE_OK
327         || m_updateGrandfatheredStatement.prepare() != SQLITE_OK
328         || m_isGrandfatheredStatement.prepare() != SQLITE_OK
329         || m_findExpiredUserInteractionStatement.prepare() != SQLITE_OK
330         ) {
331         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::prepareStatements failed to prepare, error message: %{public}s", this, m_database.lastErrorMsg());
332         ASSERT_NOT_REACHED();
333         return false;
334     }
335
336     return true;
337 }
338
339 bool ResourceLoadStatisticsDatabaseStore::insertObservedDomain(const ResourceLoadStatistics& loadStatistics)
340 {
341     ASSERT(!RunLoop::isMain());
342
343 #ifndef NDEBUG
344     ASSERT(confirmDomainDoesNotExist(loadStatistics.registrableDomain));
345 #endif
346
347     if (m_insertObservedDomainStatement.bindText(1, loadStatistics.registrableDomain.string()) != SQLITE_OK
348         || m_insertObservedDomainStatement.bindDouble(2, loadStatistics.lastSeen.secondsSinceEpoch().value()) != SQLITE_OK
349         || m_insertObservedDomainStatement.bindInt(3, loadStatistics.hadUserInteraction) != SQLITE_OK
350         || m_insertObservedDomainStatement.bindDouble(4, loadStatistics.mostRecentUserInteractionTime.secondsSinceEpoch().value()) != SQLITE_OK
351         || m_insertObservedDomainStatement.bindInt(5, loadStatistics.grandfathered) != SQLITE_OK
352         || m_insertObservedDomainStatement.bindInt(6, loadStatistics.isPrevalentResource) != SQLITE_OK
353         || m_insertObservedDomainStatement.bindInt(7, loadStatistics.isVeryPrevalentResource) != SQLITE_OK
354         || m_insertObservedDomainStatement.bindInt(8, loadStatistics.dataRecordsRemoved) != SQLITE_OK
355         || m_insertObservedDomainStatement.bindInt(9, loadStatistics.timesAccessedAsFirstPartyDueToUserInteraction) != SQLITE_OK
356         || m_insertObservedDomainStatement.bindInt(10, loadStatistics.timesAccessedAsFirstPartyDueToStorageAccessAPI) != SQLITE_OK) {
357         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::insertObservedDomain failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
358         ASSERT_NOT_REACHED();
359         return false;
360     }
361
362     if (m_insertObservedDomainStatement.step() != SQLITE_DONE) {
363         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::insertObservedDomain failed to commit, error message: %{public}s", this, m_database.lastErrorMsg());
364         ASSERT_NOT_REACHED();
365         return false;
366     }
367
368     int resetResult = m_insertObservedDomainStatement.reset();
369     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
370
371     return true;
372 }
373
374 bool ResourceLoadStatisticsDatabaseStore::relationshipExists(WebCore::SQLiteStatement& statement, unsigned firstDomainID, const RegistrableDomain& secondDomain) const
375 {
376     ASSERT(!RunLoop::isMain());
377
378     if (statement.bindInt(1, firstDomainID) != SQLITE_OK
379         || statement.bindText(2, secondDomain.string()) != SQLITE_OK
380         || statement.step() != SQLITE_ROW) {
381         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::m_insertDomainRelationshipStatement failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
382         ASSERT_NOT_REACHED();
383         return false;
384     }
385     
386     bool relationShipExists = !!statement.getColumnInt(0);
387
388     int resetResult = statement.reset();
389     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
390     
391     return relationShipExists;
392 }
393
394 bool ResourceLoadStatisticsDatabaseStore::insertDomainRelationship(WebCore::SQLiteStatement& statement, unsigned domainID, const RegistrableDomain& topFrame)
395 {
396     ASSERT(!RunLoop::isMain());
397
398     if (statement.bindInt(1, domainID) != SQLITE_OK
399         || statement.bindText(2, topFrame.string()) != SQLITE_OK
400         || statement.step() != SQLITE_DONE) {
401         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::m_insertDomainRelationshipStatement failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
402         ASSERT_NOT_REACHED();
403         return false;
404     }
405
406     int resetResult = statement.reset();
407     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
408     
409     return true;
410 }
411
412 #ifndef NDEBUG
413 bool ResourceLoadStatisticsDatabaseStore::confirmDomainDoesNotExist(const RegistrableDomain& domain) const
414 {
415     ASSERT(!RunLoop::isMain());
416
417     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK) {
418         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::confirmDomainDoesNotExist failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
419         ASSERT_NOT_REACHED();
420         return false;
421     }
422         
423     if (m_domainIDFromStringStatement.step() == SQLITE_ROW) {
424         int resetResult = m_domainIDFromStringStatement.reset();
425         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
426         return false;
427     }
428
429     int resetResult = m_domainIDFromStringStatement.reset();
430     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
431     
432     return true;
433 }
434 #endif
435
436 unsigned ResourceLoadStatisticsDatabaseStore::domainID(const RegistrableDomain& domain) const
437 {
438     ASSERT(!RunLoop::isMain());
439
440     unsigned domainID = 0;
441
442     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK
443         || m_domainIDFromStringStatement.step() != SQLITE_ROW) {
444         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::domainIDFromString failed, error message: %{public}s", this, m_database.lastErrorMsg());
445         ASSERT_NOT_REACHED();
446         return domainID;
447     }
448
449     domainID = m_domainIDFromStringStatement.getColumnInt(0);
450
451     int resetResult = m_domainIDFromStringStatement.reset();
452     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
453     
454     return domainID;
455 }
456
457 void ResourceLoadStatisticsDatabaseStore::insertDomainRelationships(const ResourceLoadStatistics& loadStatistics)
458 {
459     ASSERT(!RunLoop::isMain());
460
461     auto registrableDomainID = domainID(loadStatistics.registrableDomain);
462     for (auto& topFrameDomain : loadStatistics.storageAccessUnderTopFrameDomains)
463         insertDomainRelationship(m_storageAccessUnderTopFrameDomainsStatement, registrableDomainID, topFrameDomain);
464
465     for (auto& toDomain : loadStatistics.topFrameUniqueRedirectsTo)
466         insertDomainRelationship(m_topFrameUniqueRedirectsTo, registrableDomainID, toDomain);
467     
468     for (auto& fromDomain : loadStatistics.topFrameUniqueRedirectsFrom)
469         insertDomainRelationship(m_topFrameUniqueRedirectsFrom, registrableDomainID, fromDomain);
470     
471     for (auto& topFrameDomain : loadStatistics.subframeUnderTopFrameDomains)
472         insertDomainRelationship(m_subframeUnderTopFrameDomains, registrableDomainID, topFrameDomain);
473     
474     for (auto& topFrameDomain : loadStatistics.subresourceUnderTopFrameDomains)
475         insertDomainRelationship(m_subresourceUnderTopFrameDomains, registrableDomainID, topFrameDomain);
476     
477     for (auto& toDomain : loadStatistics.subresourceUniqueRedirectsTo)
478         insertDomainRelationship(m_subresourceUniqueRedirectsTo, registrableDomainID, toDomain);
479     
480     for (auto& fromDomain : loadStatistics.subresourceUniqueRedirectsFrom)
481         insertDomainRelationship(m_subresourceUniqueRedirectsFrom, registrableDomainID, fromDomain);
482 }
483
484 void ResourceLoadStatisticsDatabaseStore::populateFromMemoryStore(const ResourceLoadStatisticsMemoryStore& memoryStore)
485 {
486     ASSERT(!RunLoop::isMain());
487
488     if (!isEmpty())
489         return;
490
491     auto& statisticsMap = memoryStore.data();
492     for (const auto& statistic : statisticsMap)
493         insertObservedDomain(statistic.value);
494
495     // Make a separate pass for inter-domain relationships so we
496     // can refer to the ObservedDomain table entries
497     for (auto& statistic : statisticsMap)
498         insertDomainRelationships(statistic.value);
499
500     m_database.runVacuumCommand();
501 }
502
503 void ResourceLoadStatisticsDatabaseStore::calculateAndSubmitTelemetry() const
504 {
505     ASSERT(!RunLoop::isMain());
506
507     // FIXME(195088): Implement for Database version.
508 }
509
510 static String domainsToString(const HashSet<RegistrableDomain>& domains)
511 {
512     StringBuilder builder;
513     for (auto domainName : domains) {
514         if (!builder.isEmpty())
515             builder.appendLiteral(", ");
516         builder.append('"');
517         builder.append(domainName.string());
518         builder.append('"');
519     }
520
521     return builder.toString();
522 }
523
524 void ResourceLoadStatisticsDatabaseStore::incrementRecordsDeletedCountForDomains(HashSet<RegistrableDomain>&& domains)
525 {
526     ASSERT(!RunLoop::isMain());
527
528     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET dataRecordsRemoved = dataRecordsRemoved + 1 WHERE registrableDomain IN (", domainsToString(domains), ")"));
529     if (domainsToUpdateStatement.prepare() != SQLITE_OK
530         || domainsToUpdateStatement.step() != SQLITE_DONE) {
531         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::incrementStatisticsForDomains failed, error message: %{public}s", this, m_database.lastErrorMsg());
532         ASSERT_NOT_REACHED();
533     }
534 }
535
536 unsigned ResourceLoadStatisticsDatabaseStore::recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(unsigned primaryDomainID, StdSet<unsigned>& nonPrevalentRedirectionSources, unsigned numberOfRecursiveCalls)
537 {
538     ASSERT(!RunLoop::isMain());
539
540     if (numberOfRecursiveCalls >= maxNumberOfRecursiveCallsInRedirectTraceBack) {
541         RELEASE_LOG(ResourceLoadStatistics, "Hit %u recursive calls in redirect backtrace. Returning early.", maxNumberOfRecursiveCallsInRedirectTraceBack);
542         return numberOfRecursiveCalls;
543     }
544     
545     ++numberOfRecursiveCalls;
546
547     StdSet<unsigned> newlyIdentifiedDomains;
548     SQLiteStatement findSubresources(m_database, makeString("SELECT SubresourceUniqueRedirectsFrom.fromDomainID from SubresourceUniqueRedirectsFrom INNER JOIN ObservedDomains ON ObservedDomains.domainID = SubresourceUniqueRedirectsFrom.fromDomainID WHERE subresourceDomainID = ", String::number(primaryDomainID), "AND ObservedDomains.isPrevalent = 0"));
549     if (findSubresources.prepare() == SQLITE_OK) {
550         while (findSubresources.step() == SQLITE_ROW) {
551             auto newDomainID = findSubresources.getColumnInt(0);
552             auto insertResult = nonPrevalentRedirectionSources.insert(newDomainID);
553             if (insertResult.second)
554                 newlyIdentifiedDomains.insert(newDomainID);
555         }
556     }
557
558     SQLiteStatement findTopFrames(m_database, makeString("SELECT TopFrameUniqueRedirectsFrom.fromDomainID from TopFrameUniqueRedirectsFrom INNER JOIN ObservedDomains ON ObservedDomains.domainID = TopFrameUniqueRedirectsFrom.fromDomainID WHERE targetDomainID = ", String::number(primaryDomainID), "AND ObservedDomains.isPrevalent = 0"));
559     if (findTopFrames.prepare() == SQLITE_OK) {
560         while (findTopFrames.step() == SQLITE_ROW) {
561             auto newDomainID = findTopFrames.getColumnInt(0);
562             auto insertResult = nonPrevalentRedirectionSources.insert(newDomainID);
563             if (insertResult.second)
564                 newlyIdentifiedDomains.insert(newDomainID);
565         }
566     }
567
568     if (newlyIdentifiedDomains.empty())
569         return numberOfRecursiveCalls;
570
571     for (auto domainID : newlyIdentifiedDomains)
572         numberOfRecursiveCalls = recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(domainID, nonPrevalentRedirectionSources, numberOfRecursiveCalls);
573
574     return numberOfRecursiveCalls;
575 }
576
577 template <typename IteratorType>
578 static String buildList(const WTF::IteratorRange<IteratorType>& values)
579 {
580     StringBuilder builder;
581     for (auto domainID : values) {
582         if (!builder.isEmpty())
583             builder.appendLiteral(", ");
584         builder.appendNumber(domainID);
585     }
586     return builder.toString();
587 }
588
589 void ResourceLoadStatisticsDatabaseStore::markAsPrevalentIfHasRedirectedToPrevalent()
590 {
591     ASSERT(!RunLoop::isMain());
592
593     StdSet<unsigned> prevalentDueToRedirect;
594     SQLiteStatement subresourceRedirectStatement(m_database, "SELECT DISTINCT SubresourceUniqueRedirectsTo.subresourceDomainID FROM SubresourceUniqueRedirectsTo JOIN ObservedDomains ON ObservedDomains.domainID = SubresourceUniqueRedirectsTo.toDomainID AND ObservedDomains.isPrevalent = 1"_s);
595     if (subresourceRedirectStatement.prepare() == SQLITE_OK) {
596         while (subresourceRedirectStatement.step() == SQLITE_ROW)
597             prevalentDueToRedirect.insert(subresourceRedirectStatement.getColumnInt(0));
598     }
599
600     SQLiteStatement topFrameRedirectStatement(m_database, "SELECT DISTINCT TopFrameUniqueRedirectsTo.sourceDomainID FROM TopFrameUniqueRedirectsTo JOIN ObservedDomains ON ObservedDomains.domainID = TopFrameUniqueRedirectsTo.toDomainID AND ObservedDomains.isPrevalent = 1"_s);
601     if (topFrameRedirectStatement.prepare() == SQLITE_OK) {
602         while (topFrameRedirectStatement.step() == SQLITE_ROW)
603             prevalentDueToRedirect.insert(topFrameRedirectStatement.getColumnInt(0));
604     }
605
606     SQLiteStatement markPrevalentStatement(m_database, makeString("UPDATE ObservedDomains SET isPrevalent = 1 WHERE domainID IN (", buildList(WTF::IteratorRange<StdSet<unsigned>::iterator>(prevalentDueToRedirect.begin(), prevalentDueToRedirect.end())), ")"));
607     if (markPrevalentStatement.prepare() != SQLITE_OK
608         || markPrevalentStatement.step() != SQLITE_DONE) {
609         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::markAsPrevalentIfHasRedirectedToPrevalent failed to execute, error message: %{public}s", this, m_database.lastErrorMsg());
610         ASSERT_NOT_REACHED();
611     }
612 }
613
614 HashMap<unsigned, ResourceLoadStatisticsDatabaseStore::NotVeryPrevalentResources> ResourceLoadStatisticsDatabaseStore::findNotVeryPrevalentResources()
615 {
616     ASSERT(!RunLoop::isMain());
617
618     HashMap<unsigned, NotVeryPrevalentResources> results;
619     SQLiteStatement notVeryPrevalentResourcesStatement(m_database, "SELECT domainID, registrableDomain, isPrevalent FROM ObservedDomains WHERE isVeryPrevalent = 0"_s);
620     if (notVeryPrevalentResourcesStatement.prepare() == SQLITE_OK) {
621         while (notVeryPrevalentResourcesStatement.step() == SQLITE_ROW) {
622             unsigned key = static_cast<unsigned>(notVeryPrevalentResourcesStatement.getColumnInt(0));
623             NotVeryPrevalentResources value({ RegistrableDomain::uncheckedCreateFromRegistrableDomainString(notVeryPrevalentResourcesStatement.getColumnText(1))
624                 , notVeryPrevalentResourcesStatement.getColumnInt(2) ? ResourceLoadPrevalence::High : ResourceLoadPrevalence::Low
625                 , 0, 0, 0, 0 });
626             results.add(key, value);
627         }
628     }
629
630     StringBuilder builder;
631     for (auto value : results.keys()) {
632         if (!builder.isEmpty())
633             builder.appendLiteral(", ");
634         builder.appendNumber(value);
635     }
636
637     auto domainIDsOfInterest = builder.toString();
638
639     SQLiteStatement subresourceUnderTopFrameDomainsStatement(m_database, makeString("SELECT subresourceDomainID, COUNT(topFrameDomainID) FROM SubresourceUnderTopFrameDomains WHERE subresourceDomainID IN (", domainIDsOfInterest, ") GROUP BY subresourceDomainID"));
640     if (subresourceUnderTopFrameDomainsStatement.prepare() == SQLITE_OK) {
641         while (subresourceUnderTopFrameDomainsStatement.step() == SQLITE_ROW) {
642             unsigned domainID = static_cast<unsigned>(subresourceUnderTopFrameDomainsStatement.getColumnInt(0));
643             auto result = results.find(domainID);
644             if (result != results.end())
645                 result->value.subresourceUnderTopFrameDomainsCount = static_cast<unsigned>(subresourceUnderTopFrameDomainsStatement.getColumnInt(1));
646         }
647     }
648
649     SQLiteStatement subresourceUniqueRedirectsToCountStatement(m_database, makeString("SELECT subresourceDomainID, COUNT(toDomainID) FROM SubresourceUniqueRedirectsTo WHERE subresourceDomainID IN (", domainIDsOfInterest, ") GROUP BY subresourceDomainID"));
650     if (subresourceUniqueRedirectsToCountStatement.prepare() == SQLITE_OK) {
651         while (subresourceUniqueRedirectsToCountStatement.step() == SQLITE_ROW) {
652             unsigned domainID = static_cast<unsigned>(subresourceUniqueRedirectsToCountStatement.getColumnInt(0));
653             auto result = results.find(domainID);
654             if (result != results.end())
655                 result->value.subresourceUniqueRedirectsToCount = static_cast<unsigned>(subresourceUniqueRedirectsToCountStatement.getColumnInt(1));
656         }
657     }
658
659     SQLiteStatement subframeUnderTopFrameDomainsCountStatement(m_database, makeString("SELECT subresourceDomainID, COUNT(topFrameDomainID) FROM SubresourceUnderTopFrameDomains WHERE subresourceDomainID IN (", domainIDsOfInterest, ") GROUP BY subresourceDomainID"));
660     if (subframeUnderTopFrameDomainsCountStatement.prepare() == SQLITE_OK) {
661         while (subframeUnderTopFrameDomainsCountStatement.step() == SQLITE_ROW) {
662             unsigned domainID = static_cast<unsigned>(subframeUnderTopFrameDomainsCountStatement.getColumnInt(0));
663             auto result = results.find(domainID);
664             if (result != results.end())
665                 result->value.subframeUnderTopFrameDomainsCount = static_cast<unsigned>(subframeUnderTopFrameDomainsCountStatement.getColumnInt(1));
666         }
667     }
668
669     SQLiteStatement topFrameUniqueRedirectsToCountStatement(m_database, makeString("SELECT sourceDomainID, COUNT(toDomainID) FROM TopFrameUniqueRedirectsTo WHERE sourceDomainID IN (", domainIDsOfInterest, ") GROUP BY sourceDomainID"));
670     if (topFrameUniqueRedirectsToCountStatement.prepare() == SQLITE_OK) {
671         while (topFrameUniqueRedirectsToCountStatement.step() == SQLITE_ROW) {
672             unsigned domainID = static_cast<unsigned>(topFrameUniqueRedirectsToCountStatement.getColumnInt(0));
673             auto result = results.find(domainID);
674             if (result != results.end())
675                 result->value.topFrameUniqueRedirectsToCount = static_cast<unsigned>(topFrameUniqueRedirectsToCountStatement.getColumnInt(1));
676         }
677     }
678
679     return results;
680 }
681
682 void ResourceLoadStatisticsDatabaseStore::reclassifyResources()
683 {
684     ASSERT(!RunLoop::isMain());
685
686     auto notVeryPrevalentResources = findNotVeryPrevalentResources();
687
688     for (auto& resourceStatistic : notVeryPrevalentResources.values()) {
689         if (shouldSkip(resourceStatistic.registerableDomain))
690             continue;
691
692         auto newPrevalence = classifier().calculateResourcePrevalence(resourceStatistic.subresourceUnderTopFrameDomainsCount, resourceStatistic.subresourceUniqueRedirectsToCount, resourceStatistic.subframeUnderTopFrameDomainsCount, resourceStatistic.topFrameUniqueRedirectsToCount, resourceStatistic.prevalence);
693         if (newPrevalence != resourceStatistic.prevalence)
694             setPrevalentResource(resourceStatistic.registerableDomain, newPrevalence);
695     }
696 }
697
698 void ResourceLoadStatisticsDatabaseStore::classifyPrevalentResources()
699 {
700     ASSERT(!RunLoop::isMain());
701     ensurePrevalentResourcesForDebugMode();
702     markAsPrevalentIfHasRedirectedToPrevalent();
703     reclassifyResources();
704 }
705
706 void ResourceLoadStatisticsDatabaseStore::syncStorageIfNeeded()
707 {
708     ASSERT(!RunLoop::isMain());
709     m_database.runVacuumCommand();
710 }
711
712 void ResourceLoadStatisticsDatabaseStore::syncStorageImmediately()
713 {
714     ASSERT(!RunLoop::isMain());
715     m_database.runVacuumCommand();
716 }
717
718 void ResourceLoadStatisticsDatabaseStore::hasStorageAccess(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain, Optional<FrameID> frameID, PageID pageID, CompletionHandler<void(bool)>&& completionHandler)
719 {
720     ASSERT(!RunLoop::isMain());
721
722     ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
723
724     switch (cookieTreatmentForOrigin(subFrameDomain)) {
725     case CookieTreatmentResult::BlockAndPurge:
726         completionHandler(false);
727         return;
728     case CookieTreatmentResult::BlockAndKeep:
729         completionHandler(true);
730         return;
731     case CookieTreatmentResult::Allow:
732         // Do nothing
733         break;
734     };
735
736     RunLoop::main().dispatch([store = makeRef(store()), subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, completionHandler = WTFMove(completionHandler)]() mutable {
737         store->callHasStorageAccessForFrameHandler(subFrameDomain, topFrameDomain, frameID.value(), pageID, [store = store.copyRef(), completionHandler = WTFMove(completionHandler)](bool result) mutable {
738             store->statisticsQueue().dispatch([completionHandler = WTFMove(completionHandler), result] () mutable {
739                 completionHandler(result);
740             });
741         });
742     });
743 }
744
745 void ResourceLoadStatisticsDatabaseStore::requestStorageAccess(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, FrameID frameID, PageID pageID, CompletionHandler<void(StorageAccessStatus)>&& completionHandler)
746 {
747     ASSERT(!RunLoop::isMain());
748
749     auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
750
751     switch (cookieTreatmentForOrigin(subFrameDomain)) {
752     case CookieTreatmentResult::BlockAndPurge: {
753 #if !RELEASE_LOG_DISABLED
754         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ResourceLoadStatisticsDebug, "Cannot grant storage access to %{public}s since its cookies are blocked in third-party contexts and it has not received user interaction as first-party.", subFrameDomain.string().utf8().data());
755 #endif
756         completionHandler(StorageAccessStatus::CannotRequestAccess);
757         }
758         return;
759     case CookieTreatmentResult::BlockAndKeep: {
760 #if !RELEASE_LOG_DISABLED
761         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ResourceLoadStatisticsDebug, "No need to grant storage access to %{public}s since its cookies are not blocked in third-party contexts.", subFrameDomain.string().utf8().data());
762 #endif
763         completionHandler(StorageAccessStatus::HasAccess);
764         }
765         return;
766     case CookieTreatmentResult::Allow:
767         // Do nothing
768         break;
769     };
770
771     auto userWasPromptedEarlier = hasUserGrantedStorageAccessThroughPrompt(subFrameStatus.second, topFrameDomain);
772     if (userWasPromptedEarlier == StorageAccessPromptWasShown::No) {
773 #if !RELEASE_LOG_DISABLED
774         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ResourceLoadStatisticsDebug, "About to ask the user whether they want to grant storage access to %{public}s under %{public}s or not.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
775 #endif
776         completionHandler(StorageAccessStatus::RequiresUserPrompt);
777         return;
778     }
779
780 #if !RELEASE_LOG_DISABLED
781     if (userWasPromptedEarlier == StorageAccessPromptWasShown::Yes)
782         RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ResourceLoadStatisticsDebug, "Storage access was granted to %{public}s under %{public}s.", subFrameDomain.string().utf8().data(), topFrameDomain.string().utf8().data());
783 #endif
784
785     SQLiteStatement incrementStorageAccess(m_database, makeString("UPDATE ObservedDomains SET timesAccessedAsFirstPartyDueToStorageAccessAPI = timesAccessedAsFirstPartyDueToStorageAccessAPI + 1 WHERE domainID = ", String::number(subFrameStatus.second)));
786     if (incrementStorageAccess.prepare() != SQLITE_OK
787         || incrementStorageAccess.step() != SQLITE_DONE) {
788         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::requestStorageAccess failed, error message: %{public}s", this, m_database.lastErrorMsg());
789         ASSERT_NOT_REACHED();
790     }
791     
792     grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, userWasPromptedEarlier, [completionHandler = WTFMove(completionHandler)] (StorageAccessWasGranted wasGranted) mutable {
793         completionHandler(wasGranted == StorageAccessWasGranted::Yes ? StorageAccessStatus::HasAccess : StorageAccessStatus::CannotRequestAccess);
794     });
795 }
796
797 void ResourceLoadStatisticsDatabaseStore::requestStorageAccessUnderOpener(DomainInNeedOfStorageAccess&& domainInNeedOfStorageAccess, OpenerPageID openerPageID, OpenerDomain&& openerDomain)
798 {
799     ASSERT(domainInNeedOfStorageAccess != openerDomain);
800     ASSERT(!RunLoop::isMain());
801
802     if (domainInNeedOfStorageAccess == openerDomain)
803         return;
804
805     ensureResourceStatisticsForRegistrableDomain(domainInNeedOfStorageAccess);
806
807     if (cookieTreatmentForOrigin(domainInNeedOfStorageAccess) != CookieTreatmentResult::Allow)
808         return;
809
810 #if !RELEASE_LOG_DISABLED
811     RELEASE_LOG_INFO_IF(debugLoggingEnabled(), ResourceLoadStatisticsDebug, "[Temporary combatibility fix] Storage access was granted for %{public}s under opener page from %{public}s, with user interaction in the opened window.", domainInNeedOfStorageAccess.string().utf8().data(), openerDomain.string().utf8().data());
812 #endif
813     grantStorageAccessInternal(WTFMove(domainInNeedOfStorageAccess), WTFMove(openerDomain), WTF::nullopt, openerPageID, StorageAccessPromptWasShown::No, [](StorageAccessWasGranted) { });
814 }
815
816 void ResourceLoadStatisticsDatabaseStore::grantStorageAccess(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, uint64_t frameID, uint64_t pageID, StorageAccessPromptWasShown promptWasShown, CompletionHandler<void(StorageAccessWasGranted)>&& completionHandler)
817 {
818     ASSERT(!RunLoop::isMain());
819
820     if (promptWasShown == StorageAccessPromptWasShown::Yes) {
821         auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
822         ASSERT(subFrameStatus.first == AddedRecord::No);
823         ASSERT(hasHadUserInteraction(subFrameDomain, OperatingDatesWindow::Long));
824         insertDomainRelationship(m_storageAccessUnderTopFrameDomainsStatement, subFrameStatus.second, topFrameDomain);
825     }
826
827     grantStorageAccessInternal(WTFMove(subFrameDomain), WTFMove(topFrameDomain), frameID, pageID, promptWasShown, WTFMove(completionHandler));
828 }
829
830 void ResourceLoadStatisticsDatabaseStore::grantStorageAccessInternal(SubFrameDomain&& subFrameDomain, TopFrameDomain&& topFrameDomain, Optional<FrameID> frameID, PageID pageID, StorageAccessPromptWasShown promptWasShownNowOrEarlier, CompletionHandler<void(StorageAccessWasGranted)>&& completionHandler)
831 {
832     ASSERT(!RunLoop::isMain());
833
834     if (subFrameDomain == topFrameDomain) {
835         completionHandler(StorageAccessWasGranted::Yes);
836         return;
837     }
838
839     if (promptWasShownNowOrEarlier == StorageAccessPromptWasShown::Yes) {
840 #ifndef NDEBUG
841         auto subFrameStatus = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
842         ASSERT(subFrameStatus.first == AddedRecord::No);
843         ASSERT(hasHadUserInteraction(subFrameDomain, OperatingDatesWindow::Long));
844         ASSERT(hasUserGrantedStorageAccessThroughPrompt(subFrameStatus.second, topFrameDomain) == StorageAccessPromptWasShown::Yes);
845 #endif
846         setUserInteraction(subFrameDomain, true, WallTime::now());
847     }
848
849     RunLoop::main().dispatch([subFrameDomain = subFrameDomain.isolatedCopy(), topFrameDomain = topFrameDomain.isolatedCopy(), frameID, pageID, store = makeRef(store()), completionHandler = WTFMove(completionHandler)]() mutable {
850         store->callGrantStorageAccessHandler(subFrameDomain, topFrameDomain, frameID, pageID, [completionHandler = WTFMove(completionHandler), store = store.copyRef()](StorageAccessWasGranted wasGranted) mutable {
851             store->statisticsQueue().dispatch([wasGranted, completionHandler = WTFMove(completionHandler)] () mutable {
852                 completionHandler(wasGranted);
853             });
854         });
855     });
856
857 }
858
859 void ResourceLoadStatisticsDatabaseStore::grandfatherDataForDomains(const HashSet<RegistrableDomain>& domains)
860 {
861     ASSERT(!RunLoop::isMain());
862
863     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET grandfathered = 1 WHERE registrableDomain IN (", domainsToString(domains), ")"));
864     if (domainsToUpdateStatement.prepare() != SQLITE_OK
865         || domainsToUpdateStatement.step() != SQLITE_DONE) {
866         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::grandfatherDataForDomains failed, error message: %{public}s", this, m_database.lastErrorMsg());
867         ASSERT_NOT_REACHED();
868     }
869 }
870
871 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::ensurePrevalentResourcesForDebugMode()
872 {
873     ASSERT(!RunLoop::isMain());
874
875     if (!debugModeEnabled())
876         return { };
877
878     Vector<RegistrableDomain> primaryDomainsToBlock;
879     primaryDomainsToBlock.reserveInitialCapacity(2);
880
881     ensureResourceStatisticsForRegistrableDomain(debugStaticPrevalentResource());
882     setPrevalentResource(debugStaticPrevalentResource(), ResourceLoadPrevalence::High);
883     primaryDomainsToBlock.uncheckedAppend(debugStaticPrevalentResource());
884
885     if (!debugManualPrevalentResource().isEmpty()) {
886         ensureResourceStatisticsForRegistrableDomain(debugManualPrevalentResource());
887         setPrevalentResource(debugManualPrevalentResource(), ResourceLoadPrevalence::High);
888         primaryDomainsToBlock.uncheckedAppend(debugManualPrevalentResource());
889 #if !RELEASE_LOG_DISABLED
890         RELEASE_LOG_INFO(ResourceLoadStatisticsDebug, "Did set %{public}s as prevalent resource for the purposes of ITP Debug Mode.", debugManualPrevalentResource().string().utf8().data());
891 #endif
892     }
893
894     return primaryDomainsToBlock;
895 }
896
897 void ResourceLoadStatisticsDatabaseStore::logFrameNavigation(const RegistrableDomain& targetDomain, const RegistrableDomain& topFrameDomain, const RegistrableDomain& sourceDomain, bool isRedirect, bool isMainFrame)
898 {
899     ASSERT(!RunLoop::isMain());
900
901     bool areTargetAndTopFrameDomainsSameSite = targetDomain == topFrameDomain;
902     bool areTargetAndSourceDomainsSameSite = targetDomain == sourceDomain;
903
904     bool statisticsWereUpdated = false;
905     if (!isMainFrame && !(areTargetAndTopFrameDomainsSameSite || areTargetAndSourceDomainsSameSite)) {
906         auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
907         updateLastSeen(targetDomain, ResourceLoadStatistics::reduceTimeResolution(WallTime::now()));
908
909         if (!relationshipExists(m_subframeUnderTopFrameDomainExists, targetResult.second, topFrameDomain)) {
910             insertDomainRelationship(m_subframeUnderTopFrameDomains, targetResult.second, topFrameDomain);
911             statisticsWereUpdated = true;
912         }
913     }
914
915     if (isRedirect && !areTargetAndSourceDomainsSameSite) {
916         if (isMainFrame) {
917             auto redirectingDomainResult = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
918             if (!relationshipExists(m_topFrameUniqueRedirectsToExists, redirectingDomainResult.second, targetDomain)) {
919                 insertDomainRelationship(m_topFrameUniqueRedirectsTo, redirectingDomainResult.second, targetDomain);
920                 statisticsWereUpdated = true;
921             }
922
923             auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
924             if (!relationshipExists(m_topFrameUniqueRedirectsFromExists, targetResult.second, sourceDomain)) {
925                 insertDomainRelationship(m_topFrameUniqueRedirectsFrom, targetResult.second, sourceDomain);
926                 statisticsWereUpdated = true;
927             }
928         } else {
929             auto redirectingDomainResult = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
930             if (!relationshipExists(m_subresourceUniqueRedirectsToExists, redirectingDomainResult.second, targetDomain)) {
931                 insertDomainRelationship(m_subresourceUniqueRedirectsTo, redirectingDomainResult.second, targetDomain);
932                 statisticsWereUpdated = true;
933             }
934
935             auto targetResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
936             if (!relationshipExists(m_subresourceUniqueRedirectsFromExists, targetResult.second, sourceDomain)) {
937                 insertDomainRelationship(m_subresourceUniqueRedirectsFrom, targetResult.second, sourceDomain);
938                 statisticsWereUpdated = true;
939             }
940         }
941     }
942
943     if (statisticsWereUpdated)
944         scheduleStatisticsProcessingRequestIfNecessary();
945 }
946
947 void ResourceLoadStatisticsDatabaseStore::logSubresourceLoading(const SubResourceDomain& targetDomain, const TopFrameDomain& topFrameDomain, WallTime lastSeen)
948 {
949     ASSERT(!RunLoop::isMain());
950
951     auto result = ensureResourceStatisticsForRegistrableDomain(targetDomain);
952     updateLastSeen(targetDomain, lastSeen);
953
954     auto targetDomainID = result.second;
955     if (!relationshipExists(m_subresourceUnderTopFrameDomainExists, targetDomainID, topFrameDomain)) {
956         insertDomainRelationship(m_subresourceUnderTopFrameDomains, targetDomainID, topFrameDomain);
957         scheduleStatisticsProcessingRequestIfNecessary();
958     }
959 }
960
961 void ResourceLoadStatisticsDatabaseStore::logSubresourceRedirect(const RedirectedFromDomain& sourceDomain, const RedirectedToDomain& targetDomain)
962 {
963     ASSERT(!RunLoop::isMain());
964
965     auto sourceDomainResult = ensureResourceStatisticsForRegistrableDomain(sourceDomain);
966     auto targetDomainResult = ensureResourceStatisticsForRegistrableDomain(targetDomain);
967
968     bool isNewRedirectToEntry = false;
969     if (!relationshipExists(m_subresourceUniqueRedirectsToExists, sourceDomainResult.second, targetDomain)) {
970         insertDomainRelationship(m_subresourceUniqueRedirectsTo, sourceDomainResult.second, targetDomain);
971         isNewRedirectToEntry = true;
972     }
973
974     bool isNewRedirectFromEntry = false;
975     if (!relationshipExists(m_subresourceUniqueRedirectsFromExists, targetDomainResult.second, sourceDomain)) {
976         insertDomainRelationship(m_subresourceUniqueRedirectsFrom, targetDomainResult.second, sourceDomain);
977         isNewRedirectFromEntry = true;
978     }
979
980     if (isNewRedirectToEntry || isNewRedirectFromEntry)
981         scheduleStatisticsProcessingRequestIfNecessary();
982 }
983
984 void ResourceLoadStatisticsDatabaseStore::logCrossSiteLoadWithLinkDecoration(const NavigatedFromDomain& fromDomain, const NavigatedToDomain& toDomain)
985 {
986     ASSERT(!RunLoop::isMain());
987     ASSERT(fromDomain != toDomain);
988
989     auto fromDomainResult = ensureResourceStatisticsForRegistrableDomain(fromDomain);
990
991     if (!relationshipExists(m_topFrameLinkDecorationsFromExists, fromDomainResult.second, toDomain)) {
992         insertDomainRelationship(m_topFrameLinkDecorationsFrom, fromDomainResult.second, toDomain);
993         scheduleStatisticsProcessingRequestIfNecessary();
994     }
995 }
996
997 void ResourceLoadStatisticsDatabaseStore::setUserInteraction(const RegistrableDomain& domain, bool hadUserInteraction, WallTime mostRecentInteraction)
998 {
999     ASSERT(!RunLoop::isMain());
1000
1001     if (m_mostRecentUserInteractionStatement.bindInt(1, hadUserInteraction)
1002         || m_mostRecentUserInteractionStatement.bindDouble(2, mostRecentInteraction.secondsSinceEpoch().value()) != SQLITE_OK
1003         || m_mostRecentUserInteractionStatement.bindText(3, domain.string()) != SQLITE_OK
1004         || m_mostRecentUserInteractionStatement.step() != SQLITE_DONE) {
1005         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::setUserInteraction, error message: %{public}s", this, m_database.lastErrorMsg());
1006         ASSERT_NOT_REACHED();
1007         return;
1008     }
1009
1010     int resetResult = m_mostRecentUserInteractionStatement.reset();
1011     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1012 }
1013
1014 void ResourceLoadStatisticsDatabaseStore::logUserInteraction(const TopFrameDomain& domain)
1015 {
1016     ASSERT(!RunLoop::isMain());
1017
1018     ensureResourceStatisticsForRegistrableDomain(domain);
1019     setUserInteraction(domain, true, WallTime::now());
1020 }
1021
1022 void ResourceLoadStatisticsDatabaseStore::clearUserInteraction(const RegistrableDomain& domain)
1023 {
1024     ASSERT(!RunLoop::isMain());
1025
1026     auto targetResult = ensureResourceStatisticsForRegistrableDomain(domain);
1027     setUserInteraction(domain, false, { });
1028
1029     SQLiteStatement removeStorageAccess(m_database, makeString("DELETE FROM StorageAccessUnderTopFrameDomains WHERE domainID = ", String::number(targetResult.second)));
1030     if (removeStorageAccess.prepare() != SQLITE_OK
1031         || removeStorageAccess.step() != SQLITE_DONE) {
1032         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::logUserInteraction failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1033         ASSERT_NOT_REACHED();
1034     }
1035 }
1036
1037 bool ResourceLoadStatisticsDatabaseStore::hasHadUserInteraction(const RegistrableDomain& domain, OperatingDatesWindow operatingDatesWindow)
1038 {
1039     ASSERT(!RunLoop::isMain());
1040
1041     if (m_hadUserInteractionStatement.bindText(1, domain.string()) != SQLITE_OK
1042         || m_hadUserInteractionStatement.step() != SQLITE_DONE) {
1043         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::m_updatePrevalentResourceStatement failed, error message: %{public}s", this, m_database.lastErrorMsg());
1044         ASSERT_NOT_REACHED();
1045         return false;
1046     }
1047
1048     bool hadUserInteraction = !!m_hadUserInteractionStatement.getColumnInt(0);
1049     if (!hadUserInteraction)
1050         return false;
1051     
1052     WallTime mostRecentUserInteractionTime = WallTime::fromRawSeconds(m_hadUserInteractionStatement.getColumnDouble(1));
1053
1054     if (hasStatisticsExpired(mostRecentUserInteractionTime, operatingDatesWindow)) {
1055         // Drop privacy sensitive data because we no longer need it.
1056         // Set timestamp to 0 so that statistics merge will know
1057         // it has been reset as opposed to its default -1.
1058         clearUserInteraction(domain);
1059         hadUserInteraction = false;
1060     }
1061     
1062     return hadUserInteraction;
1063 }
1064
1065 void ResourceLoadStatisticsDatabaseStore::setPrevalentResource(const RegistrableDomain& domain, ResourceLoadPrevalence newPrevalence)
1066 {
1067     ASSERT(!RunLoop::isMain());
1068     if (shouldSkip(domain))
1069         return;
1070
1071     if (m_updatePrevalentResourceStatement.bindInt(1, 1) != SQLITE_OK
1072         || m_updatePrevalentResourceStatement.bindText(2, domain.string()) != SQLITE_OK
1073         || m_updatePrevalentResourceStatement.step() != SQLITE_DONE) {
1074         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::m_updatePrevalentResourceStatement failed, error message: %{public}s", this, m_database.lastErrorMsg());
1075         ASSERT_NOT_REACHED();
1076         return;
1077     }
1078     
1079     int resetResult = m_updatePrevalentResourceStatement.reset();
1080     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1081
1082     if (newPrevalence == ResourceLoadPrevalence::VeryHigh) {
1083         if (m_updateVeryPrevalentResourceStatement.bindInt(1, 1) != SQLITE_OK
1084             || m_updateVeryPrevalentResourceStatement.bindText(2, domain.string()) != SQLITE_OK
1085             || m_updateVeryPrevalentResourceStatement.step() != SQLITE_DONE) {
1086             RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::m_updateVeryPrevalentResourceStatement failed, error message: %{public}s", this, m_database.lastErrorMsg());
1087             ASSERT_NOT_REACHED();
1088             return;
1089         }
1090
1091         int resetResult = m_updateVeryPrevalentResourceStatement.reset();
1092         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1093     }
1094
1095     StdSet<unsigned> nonPrevalentRedirectionSources;
1096     recursivelyFindNonPrevalentDomainsThatRedirectedToThisDomain(domainID(domain), nonPrevalentRedirectionSources, 0);
1097     setDomainsAsPrevalent(WTFMove(nonPrevalentRedirectionSources));
1098 }
1099
1100 void ResourceLoadStatisticsDatabaseStore::setDomainsAsPrevalent(StdSet<unsigned>&& domains)
1101 {
1102     ASSERT(!RunLoop::isMain());
1103
1104     SQLiteStatement domainsToUpdateStatement(m_database, makeString("UPDATE ObservedDomains SET isPrevalent = 1 WHERE domainID IN (", buildList(WTF::IteratorRange<StdSet<unsigned>::iterator>(domains.begin(), domains.end())), ")"));
1105     if (domainsToUpdateStatement.prepare() != SQLITE_OK
1106         || domainsToUpdateStatement.step() != SQLITE_DONE) {
1107         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::setDomainsAsPrevalent failed, error message: %{public}s", this, m_database.lastErrorMsg());
1108         ASSERT_NOT_REACHED();
1109         return;
1110     }
1111 }
1112
1113 String ResourceLoadStatisticsDatabaseStore::dumpResourceLoadStatistics() const
1114 {
1115     ASSERT(!RunLoop::isMain());
1116
1117     // FIXME(195088): Implement SQLite-based dumping routines.
1118     ASSERT_NOT_REACHED();
1119     return { };
1120 }
1121
1122 bool ResourceLoadStatisticsDatabaseStore::predicateValueForDomain(WebCore::SQLiteStatement& predicateStatement, const RegistrableDomain& domain) const
1123 {
1124     ASSERT(!RunLoop::isMain());
1125     
1126     if (predicateStatement.bindText(1, domain.string()) != SQLITE_OK
1127         || predicateStatement.step() != SQLITE_DONE) {
1128         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::predicateValueForDomain failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1129         ASSERT_NOT_REACHED();
1130         return false;
1131     }
1132
1133     bool result = !!predicateStatement.getColumnInt(0);
1134     int resetResult = predicateStatement.reset();
1135     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1136     return result;
1137 }
1138
1139 bool ResourceLoadStatisticsDatabaseStore::isPrevalentResource(const RegistrableDomain& domain) const
1140 {
1141     ASSERT(!RunLoop::isMain());
1142
1143     if (shouldSkip(domain))
1144         return false;
1145
1146     return predicateValueForDomain(m_isPrevalentResourceStatement, domain);
1147 }
1148
1149 bool ResourceLoadStatisticsDatabaseStore::isVeryPrevalentResource(const RegistrableDomain& domain) const
1150 {
1151     ASSERT(!RunLoop::isMain());
1152
1153     if (shouldSkip(domain))
1154         return false;
1155
1156     return predicateValueForDomain(m_isVeryPrevalentResourceStatement, domain);
1157 }
1158
1159 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsSubresourceUnder(const SubResourceDomain& subresourceDomain, const TopFrameDomain& topFrameDomain) const
1160 {
1161     ASSERT(!RunLoop::isMain());
1162
1163     return relationshipExists(m_subresourceUnderTopFrameDomainExists, domainID(subresourceDomain), topFrameDomain);
1164 }
1165
1166 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsSubFrameUnder(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain) const
1167 {
1168     ASSERT(!RunLoop::isMain());
1169
1170     return relationshipExists(m_subframeUnderTopFrameDomainExists, domainID(subFrameDomain), topFrameDomain);
1171 }
1172
1173 bool ResourceLoadStatisticsDatabaseStore::isRegisteredAsRedirectingTo(const RedirectedFromDomain& redirectedFromDomain, const RedirectedToDomain& redirectedToDomain) const
1174 {
1175     ASSERT(!RunLoop::isMain());
1176
1177     return relationshipExists(m_subresourceUniqueRedirectsToExists, domainID(redirectedFromDomain), redirectedToDomain);
1178 }
1179
1180 void ResourceLoadStatisticsDatabaseStore::clearPrevalentResource(const RegistrableDomain& domain)
1181 {
1182     ASSERT(!RunLoop::isMain());
1183
1184     ensureResourceStatisticsForRegistrableDomain(domain);
1185     
1186     if (m_clearPrevalentResourceStatement.bindText(1, domain.string()) != SQLITE_OK
1187         || m_clearPrevalentResourceStatement.step() != SQLITE_DONE) {
1188         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::clearPrevalentResource, error message: %{public}s", this, m_database.lastErrorMsg());
1189         ASSERT_NOT_REACHED();
1190         return;
1191     }
1192     
1193     int resetResult = m_clearPrevalentResourceStatement.reset();
1194     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1195 }
1196
1197 void ResourceLoadStatisticsDatabaseStore::setGrandfathered(const RegistrableDomain& domain, bool value)
1198 {
1199     ASSERT(!RunLoop::isMain());
1200
1201     ensureResourceStatisticsForRegistrableDomain(domain);
1202     
1203     if (m_updateGrandfatheredStatement.bindInt(1, value) != SQLITE_OK
1204         || m_updateGrandfatheredStatement.bindText(1, domain.string()) != SQLITE_OK
1205         || m_updateGrandfatheredStatement.step() != SQLITE_DONE) {
1206         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::setGrandfathered failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1207         ASSERT_NOT_REACHED();
1208         return;
1209     }
1210     
1211     int resetResult = m_updateGrandfatheredStatement.reset();
1212     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1213 }
1214
1215 bool ResourceLoadStatisticsDatabaseStore::isGrandfathered(const RegistrableDomain& domain) const
1216 {
1217     ASSERT(!RunLoop::isMain());
1218
1219     return predicateValueForDomain(m_isGrandfatheredStatement, domain);
1220 }
1221
1222 void ResourceLoadStatisticsDatabaseStore::setSubframeUnderTopFrameDomain(const SubFrameDomain& subFrameDomain, const TopFrameDomain& topFrameDomain)
1223 {
1224     ASSERT(!RunLoop::isMain());
1225
1226     auto result = ensureResourceStatisticsForRegistrableDomain(subFrameDomain);
1227
1228     // For consistency, make sure we also have a statistics entry for the top frame domain.
1229     ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1230
1231     insertDomainRelationship(m_subframeUnderTopFrameDomains, result.second, topFrameDomain);
1232 }
1233
1234 void ResourceLoadStatisticsDatabaseStore::setSubresourceUnderTopFrameDomain(const SubResourceDomain& subresourceDomain, const TopFrameDomain& topFrameDomain)
1235 {
1236     ASSERT(!RunLoop::isMain());
1237
1238     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1239
1240     // For consistency, make sure we also have a statistics entry for the top frame domain.
1241     ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1242
1243     insertDomainRelationship(m_subresourceUnderTopFrameDomains, result.second, topFrameDomain);
1244 }
1245
1246 void ResourceLoadStatisticsDatabaseStore::setSubresourceUniqueRedirectTo(const SubResourceDomain& subresourceDomain, const RedirectDomain& redirectDomain)
1247 {
1248     ASSERT(!RunLoop::isMain());
1249
1250     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1251
1252     // For consistency, make sure we also have a statistics entry for the redirect domain.
1253     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1254
1255     insertDomainRelationship(m_subresourceUniqueRedirectsTo, result.second, redirectDomain);
1256 }
1257
1258 void ResourceLoadStatisticsDatabaseStore::setSubresourceUniqueRedirectFrom(const SubResourceDomain& subresourceDomain, const RedirectDomain& redirectDomain)
1259 {
1260     ASSERT(!RunLoop::isMain());
1261
1262     auto result = ensureResourceStatisticsForRegistrableDomain(subresourceDomain);
1263
1264     // For consistency, make sure we also have a statistics entry for the redirect domain.
1265     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1266
1267     insertDomainRelationship(m_subresourceUniqueRedirectsFrom, result.second, redirectDomain);
1268 }
1269
1270 void ResourceLoadStatisticsDatabaseStore::setTopFrameUniqueRedirectTo(const TopFrameDomain& topFrameDomain, const RedirectDomain& redirectDomain)
1271 {
1272     ASSERT(!RunLoop::isMain());
1273
1274     auto result = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1275
1276     // For consistency, make sure we also have a statistics entry for the redirect domain.
1277     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1278
1279     insertDomainRelationship(m_topFrameUniqueRedirectsTo, result.second, redirectDomain);
1280 }
1281
1282 void ResourceLoadStatisticsDatabaseStore::setTopFrameUniqueRedirectFrom(const TopFrameDomain& topFrameDomain, const RedirectDomain& redirectDomain)
1283 {
1284     ASSERT(!RunLoop::isMain());
1285
1286     auto result = ensureResourceStatisticsForRegistrableDomain(topFrameDomain);
1287
1288     // For consistency, make sure we also have a statistics entry for the redirect domain.
1289     ensureResourceStatisticsForRegistrableDomain(redirectDomain);
1290
1291     insertDomainRelationship(m_topFrameUniqueRedirectsFrom, result.second, redirectDomain);
1292 }
1293
1294 std::pair<ResourceLoadStatisticsDatabaseStore::AddedRecord, unsigned> ResourceLoadStatisticsDatabaseStore::ensureResourceStatisticsForRegistrableDomain(const RegistrableDomain& domain)
1295 {
1296     ASSERT(!RunLoop::isMain());
1297
1298     if (m_domainIDFromStringStatement.bindText(1, domain.string()) != SQLITE_OK) {
1299         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::createSchema failed, error message: %{public}s", this, m_database.lastErrorMsg());
1300         ASSERT_NOT_REACHED();
1301         return { AddedRecord::No, 0 };
1302     }
1303     
1304     if (m_domainIDFromStringStatement.step() == SQLITE_ROW) {
1305         unsigned domainID = m_domainIDFromStringStatement.getColumnInt(0);
1306
1307         int resetResult = m_domainIDFromStringStatement.reset();
1308         ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1309         return { AddedRecord::No, domainID };
1310     }
1311
1312     int resetResult = m_domainIDFromStringStatement.reset();
1313     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1314
1315     ResourceLoadStatistics newObservation(domain);
1316     insertObservedDomain(newObservation);
1317
1318     return { AddedRecord::Yes, domainID(domain) };
1319 }
1320
1321 void ResourceLoadStatisticsDatabaseStore::clear(CompletionHandler<void()>&& completionHandler)
1322 {
1323     ASSERT(!RunLoop::isMain());
1324
1325     clearOperatingDates();
1326
1327     auto callbackAggregator = CallbackAggregator::create(WTFMove(completionHandler));
1328
1329     removeAllStorageAccess([callbackAggregator = callbackAggregator.copyRef()] { });
1330
1331     auto primaryDomainsToBlock = ensurePrevalentResourcesForDebugMode();
1332     updateCookieBlockingForDomains(primaryDomainsToBlock, [callbackAggregator = callbackAggregator.copyRef()] { });
1333 }
1334
1335 ResourceLoadStatisticsDatabaseStore::CookieTreatmentResult ResourceLoadStatisticsDatabaseStore::cookieTreatmentForOrigin(const RegistrableDomain& domain) const
1336 {
1337     ASSERT(!RunLoop::isMain());
1338
1339     SQLiteStatement statement(m_database, makeString("SELECT isPrevalent, hadUserInteraction FROM ObservedDomains WHERE registrableDomain = ", domain.string()));
1340     if (statement.prepare() != SQLITE_OK
1341         || statement.step() != SQLITE_ROW) {
1342         return CookieTreatmentResult::Allow;
1343     }
1344     
1345     bool isPrevalent = !!statement.getColumnInt(0);
1346     if (!isPrevalent)
1347         return CookieTreatmentResult::Allow;
1348
1349     bool hadUserInteraction = statement.getColumnInt(1) ? true : false;
1350     return hadUserInteraction ? CookieTreatmentResult::BlockAndKeep : CookieTreatmentResult::BlockAndPurge;
1351 }
1352     
1353 StorageAccessPromptWasShown ResourceLoadStatisticsDatabaseStore::hasUserGrantedStorageAccessThroughPrompt(unsigned requestingDomainID, const RegistrableDomain& firstPartyDomain) const
1354 {
1355     ASSERT(!RunLoop::isMain());
1356
1357     auto firstPartyPrimaryDomainID = domainID(firstPartyDomain);
1358
1359     SQLiteStatement statement(m_database, makeString("SELECT COUNT(*) FROM StorageAccessUnderTopFrameDomains WHERE domainID = ", String::number(requestingDomainID), " AND topLevelDomainID = ", String::number(firstPartyPrimaryDomainID)));
1360     if (statement.prepare() != SQLITE_OK
1361         || statement.step() != SQLITE_ROW)
1362         return StorageAccessPromptWasShown::No;
1363
1364     return !statement.getColumnInt(0) ? StorageAccessPromptWasShown::Yes : StorageAccessPromptWasShown::No;
1365 }
1366
1367 Vector<RegistrableDomain> ResourceLoadStatisticsDatabaseStore::domainsToBlock() const
1368 {
1369     ASSERT(!RunLoop::isMain());
1370
1371     Vector<RegistrableDomain> results;
1372     SQLiteStatement statement(m_database, "SELECT registrableDomain FROM ObservedDomains WHERE isPrevalent = 1"_s);
1373     if (statement.prepare() != SQLITE_OK)
1374         return results;
1375     
1376     while (statement.step() == SQLITE_ROW)
1377         results.append(RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(0)));
1378
1379     return results;
1380 }
1381
1382 void ResourceLoadStatisticsDatabaseStore::updateCookieBlocking(CompletionHandler<void()>&& completionHandler)
1383 {
1384     ASSERT(!RunLoop::isMain());
1385
1386     auto domainsToBlock = this->domainsToBlock();
1387
1388     if (domainsToBlock.isEmpty()) {
1389         completionHandler();
1390         return;
1391     }
1392
1393     if (debugLoggingEnabled() && !domainsToBlock.isEmpty())
1394         debugLogDomainsInBatches("block", domainsToBlock);
1395
1396     RunLoop::main().dispatch([weakThis = makeWeakPtr(*this), store = makeRef(store()), domainsToBlock = crossThreadCopy(domainsToBlock), completionHandler = WTFMove(completionHandler)] () mutable {
1397         store->callUpdatePrevalentDomainsToBlockCookiesForHandler(domainsToBlock, [weakThis = WTFMove(weakThis), store = store.copyRef(), completionHandler = WTFMove(completionHandler)]() mutable {
1398             store->statisticsQueue().dispatch([weakThis = WTFMove(weakThis), completionHandler = WTFMove(completionHandler)]() mutable {
1399                 completionHandler();
1400                 if (!weakThis)
1401                     return;
1402 #if !RELEASE_LOG_DISABLED
1403                 RELEASE_LOG_INFO_IF(weakThis->debugLoggingEnabled(), ResourceLoadStatisticsDebug, "Done updating cookie blocking.");
1404 #endif
1405             });
1406         });
1407     });
1408 }
1409
1410 Vector<ResourceLoadStatisticsDatabaseStore::PrevalentDomainData> ResourceLoadStatisticsDatabaseStore::prevalentDomains() const
1411 {
1412     ASSERT(!RunLoop::isMain());
1413     
1414     Vector<PrevalentDomainData> results;
1415     SQLiteStatement statement(m_database, "SELECT domainID, registrableDomain, mostRecentUserInteractionTime, hadUserInteraction, grandfathered FROM ObservedDomains WHERE isPrevalent = 1"_s);
1416     if (statement.prepare() != SQLITE_OK)
1417         return results;
1418     
1419     while (statement.step() == SQLITE_ROW) {
1420         results.append({ static_cast<unsigned>(statement.getColumnInt(0))
1421             , RegistrableDomain::uncheckedCreateFromRegistrableDomainString(statement.getColumnText(1))
1422             , WallTime::fromRawSeconds(statement.getColumnDouble(2))
1423             , statement.getColumnInt(3) ? true : false
1424             , statement.getColumnInt(4) ? true : false
1425         });
1426     }
1427     
1428     return results;
1429 }
1430
1431 Vector<unsigned> ResourceLoadStatisticsDatabaseStore::findExpiredUserInteractions() const
1432 {
1433     ASSERT(!RunLoop::isMain());
1434
1435     Vector<unsigned> results;
1436     Optional<Seconds> expirationDateTime = statisticsEpirationTime();
1437     if (!expirationDateTime)
1438         return results;
1439
1440     if (m_findExpiredUserInteractionStatement.bindDouble(1, expirationDateTime.value().value()) != SQLITE_OK)
1441         return results;
1442
1443     while (m_findExpiredUserInteractionStatement.step() == SQLITE_ROW)
1444         results.append(m_findExpiredUserInteractionStatement.getColumnInt(0));
1445
1446     int resetResult = m_findExpiredUserInteractionStatement.reset();
1447     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1448
1449     return results;
1450 }
1451
1452 void ResourceLoadStatisticsDatabaseStore::clearExpiredUserInteractions()
1453 {
1454     ASSERT(!RunLoop::isMain());
1455
1456     auto expiredRecords = findExpiredUserInteractions();
1457     if (expiredRecords.isEmpty())
1458         return;
1459
1460     auto expiredRecordIDs = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(expiredRecords.begin(), expiredRecords.end()));
1461
1462     SQLiteStatement clearExpiredInteraction(m_database, makeString("UPDATE ObservedDomains SET mostRecentUserInteractionTime = 0, hadUserInteraction = 1 WHERE domainID IN (", expiredRecordIDs, ")"));
1463     if (clearExpiredInteraction.prepare() != SQLITE_OK)
1464         return;
1465
1466     SQLiteStatement removeStorageAccess(m_database, makeString("DELETE FROM StorageAccessUnderTopFrameDomains ", expiredRecordIDs, ")"));
1467     if (removeStorageAccess.prepare() != SQLITE_OK)
1468         return;
1469
1470     if (clearExpiredInteraction.step() != SQLITE_DONE
1471         || removeStorageAccess.step() != SQLITE_DONE) {
1472         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::logUserInteraction failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1473         ASSERT_NOT_REACHED();
1474     }
1475 }
1476
1477 void ResourceLoadStatisticsDatabaseStore::clearGrandfathering(Vector<unsigned>&& domainIDsToClear)
1478 {
1479     ASSERT(!RunLoop::isMain());
1480
1481     if (domainIDsToClear.isEmpty())
1482         return;
1483
1484     auto listToClear = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(domainIDsToClear.begin(), domainIDsToClear.end()));
1485
1486     SQLiteStatement clearGrandfatheringStatement(m_database, makeString("UPDATE ObservedDomains SET grandfathered = 0 WHERE domainID IN (", listToClear, ")"));
1487     if (clearGrandfatheringStatement.prepare() != SQLITE_OK)
1488         return;
1489     
1490     if (clearGrandfatheringStatement.step() != SQLITE_DONE) {
1491         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::clearGrandfathering failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1492         ASSERT_NOT_REACHED();
1493     }
1494 }
1495
1496 bool ResourceLoadStatisticsDatabaseStore::shouldRemoveAllWebsiteDataFor(const PrevalentDomainData& resourceStatistic, bool shouldCheckForGrandfathering) const
1497 {
1498     return !resourceStatistic.hadUserInteraction && (!shouldCheckForGrandfathering || !resourceStatistic.grandfathered);
1499 }
1500
1501 bool ResourceLoadStatisticsDatabaseStore::shouldRemoveAllButCookiesFor(const PrevalentDomainData& resourceStatistic, bool shouldCheckForGrandfathering) const
1502 {
1503     UNUSED_PARAM(resourceStatistic);
1504     UNUSED_PARAM(shouldCheckForGrandfathering);
1505     return false;
1506 }
1507
1508 HashMap<RegistrableDomain, WebsiteDataToRemove> ResourceLoadStatisticsDatabaseStore::registrableDomainsToRemoveWebsiteDataFor()
1509 {
1510     ASSERT(!RunLoop::isMain());
1511
1512     bool shouldCheckForGrandfathering = endOfGrandfatheringTimestamp() > WallTime::now();
1513     bool shouldClearGrandfathering = !shouldCheckForGrandfathering && endOfGrandfatheringTimestamp();
1514
1515     if (shouldClearGrandfathering)
1516         clearEndOfGrandfatheringTimeStamp();
1517
1518     clearExpiredUserInteractions();
1519     
1520     HashMap<RegistrableDomain, WebsiteDataToRemove> domainsToRemoveWebsiteDataFor;
1521
1522     Vector<PrevalentDomainData> prevalentDomains = this->prevalentDomains();
1523     Vector<unsigned> domainIDsToClearGrandfathering;
1524     for (auto& statistic : prevalentDomains) {
1525         if (shouldRemoveAllWebsiteDataFor(statistic, shouldCheckForGrandfathering))
1526             domainsToRemoveWebsiteDataFor.add(statistic.registerableDomain, WebsiteDataToRemove::All);
1527         else if (shouldRemoveAllButCookiesFor(statistic, shouldCheckForGrandfathering))
1528             domainsToRemoveWebsiteDataFor.add(statistic.registerableDomain, WebsiteDataToRemove::AllButCookies);
1529
1530         if (shouldClearGrandfathering && statistic.grandfathered)
1531             domainIDsToClearGrandfathering.append(statistic.domainID);
1532     }
1533
1534     clearGrandfathering(WTFMove(domainIDsToClearGrandfathering));
1535     
1536     return domainsToRemoveWebsiteDataFor;
1537 }
1538
1539 void ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded()
1540 {
1541     ASSERT(!RunLoop::isMain());
1542
1543     unsigned count = 0;
1544     if (m_observedDomainCount.step() == SQLITE_ROW)
1545         count = m_observedDomainCount.getColumnInt(0);
1546
1547     int resetResult = m_observedDomainCount.reset();
1548     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1549
1550     if (count <= parameters().maxStatisticsEntries)
1551         return;
1552
1553     ASSERT(parameters().pruneEntriesDownTo <= parameters().maxStatisticsEntries);
1554
1555     size_t countLeftToPrune = count - parameters().pruneEntriesDownTo;
1556     ASSERT(countLeftToPrune);
1557
1558     SQLiteStatement recordsToPrune(m_database, makeString("SELECT domainID FROM ObservedDomains ORDER BY hadUserInteraction, isPrevalent, lastSeen DESC LIMIT ", String::number(countLeftToPrune)));
1559     if (recordsToPrune.prepare() != SQLITE_OK) {
1560         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded failed, error message: %{public}s", this, m_database.lastErrorMsg());
1561         ASSERT_NOT_REACHED();
1562         return;
1563     }
1564
1565     Vector<unsigned> entriesToPrune;
1566     while (recordsToPrune.step() == SQLITE_ROW)
1567         entriesToPrune.append(recordsToPrune.getColumnInt(0));
1568
1569     auto listToPrune = buildList(WTF::IteratorRange<Vector<unsigned>::iterator>(entriesToPrune.begin(), entriesToPrune.end()));
1570
1571     SQLiteStatement pruneCommand(m_database, makeString("DELETE from ObservedDomains WHERE domainID IN (", listToPrune, ")"));
1572     if (pruneCommand.prepare() != SQLITE_OK
1573         || pruneCommand.step() != SQLITE_DONE) {
1574         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::pruneStatisticsIfNeeded failed, error message: %{public}s", this, m_database.lastErrorMsg());
1575         ASSERT_NOT_REACHED();
1576         return;
1577     }
1578 }
1579
1580 void ResourceLoadStatisticsDatabaseStore::updateLastSeen(const RegistrableDomain& domain, WallTime lastSeen)
1581 {
1582     ASSERT(!RunLoop::isMain());
1583
1584     if (m_updateLastSeenStatement.bindDouble(1, lastSeen.secondsSinceEpoch().value()) != SQLITE_OK
1585         || m_updateLastSeenStatement.bindText(2, domain.string()) != SQLITE_OK
1586         || m_updateLastSeenStatement.step() != SQLITE_DONE) {
1587         RELEASE_LOG_ERROR(Network, "%p - ResourceLoadStatisticsDatabaseStore::updateLastSeen failed to bind, error message: %{public}s", this, m_database.lastErrorMsg());
1588         ASSERT_NOT_REACHED();
1589         return;
1590     }
1591     
1592     int resetResult = m_updateLastSeenStatement.reset();
1593     ASSERT_UNUSED(resetResult, resetResult == SQLITE_OK);
1594 }
1595
1596 void ResourceLoadStatisticsDatabaseStore::setLastSeen(const RegistrableDomain& domain, Seconds seconds)
1597 {
1598     ASSERT(!RunLoop::isMain());
1599
1600     ensureResourceStatisticsForRegistrableDomain(domain);
1601     updateLastSeen(domain, WallTime::fromRawSeconds(seconds.seconds()));
1602 }
1603
1604 void ResourceLoadStatisticsDatabaseStore::setPrevalentResource(const RegistrableDomain& domain)
1605 {
1606     ASSERT(!RunLoop::isMain());
1607
1608     if (shouldSkip(domain))
1609         return;
1610
1611     ensureResourceStatisticsForRegistrableDomain(domain);
1612     setPrevalentResource(domain, ResourceLoadPrevalence::High);
1613 }
1614
1615 void ResourceLoadStatisticsDatabaseStore::setVeryPrevalentResource(const RegistrableDomain& domain)
1616 {
1617     ASSERT(!RunLoop::isMain());
1618
1619     if (shouldSkip(domain))
1620         return;
1621     
1622     ensureResourceStatisticsForRegistrableDomain(domain);
1623     setPrevalentResource(domain, ResourceLoadPrevalence::VeryHigh);
1624 }
1625
1626 } // namespace WebKit
1627
1628 #endif