ResourceLoadStatistics grandfathering happens much too often.
[WebKit-https.git] / Source / WebKit / UIProcess / Storage / ResourceLoadStatisticsPersistentStorage.cpp
1 /*
2  * Copyright (C) 2017 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
14  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
15  * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
17  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
18  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
19  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
20  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
21  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
22  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
23  * THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 #include "config.h"
27 #include "ResourceLoadStatisticsPersistentStorage.h"
28
29 #include "Logging.h"
30 #include "WebResourceLoadStatisticsStore.h"
31 #include <WebCore/FileMonitor.h>
32 #include <WebCore/FileSystem.h>
33 #include <WebCore/KeyedCoding.h>
34 #include <WebCore/SharedBuffer.h>
35 #include <wtf/RunLoop.h>
36 #include <wtf/WorkQueue.h>
37 #include <wtf/threads/BinarySemaphore.h>
38
39 namespace WebKit {
40
41 constexpr Seconds minimumWriteInterval { 5_min };
42
43 using namespace WebCore;
44
45 static bool hasFileChangedSince(const String& path, WallTime since)
46 {
47     ASSERT(!RunLoop::isMain());
48     time_t modificationTime;
49     if (!getFileModificationTime(path, modificationTime))
50         return true;
51
52     return WallTime::fromRawSeconds(modificationTime) > since;
53 }
54
55 static std::unique_ptr<KeyedDecoder> createDecoderForFile(const String& path)
56 {
57     ASSERT(!RunLoop::isMain());
58     auto handle = openAndLockFile(path, OpenForRead);
59     if (handle == invalidPlatformFileHandle)
60         return nullptr;
61
62     long long fileSize = 0;
63     if (!getFileSize(handle, fileSize)) {
64         unlockAndCloseFile(handle);
65         return nullptr;
66     }
67
68     size_t bytesToRead;
69     if (!WTF::convertSafely(fileSize, bytesToRead)) {
70         unlockAndCloseFile(handle);
71         return nullptr;
72     }
73
74     Vector<char> buffer(bytesToRead);
75     size_t totalBytesRead = readFromFile(handle, buffer.data(), buffer.size());
76
77     unlockAndCloseFile(handle);
78
79     if (totalBytesRead != bytesToRead)
80         return nullptr;
81
82     return KeyedDecoder::decoder(reinterpret_cast<const uint8_t*>(buffer.data()), buffer.size());
83 }
84
85 ResourceLoadStatisticsPersistentStorage::ResourceLoadStatisticsPersistentStorage(WebResourceLoadStatisticsStore& store, const String& storageDirectoryPath)
86     : m_memoryStore(store)
87     , m_storageDirectoryPath(storageDirectoryPath)
88     , m_asyncWriteTimer(RunLoop::main(), this, &ResourceLoadStatisticsPersistentStorage::asyncWriteTimerFired)
89 {
90 }
91
92 void ResourceLoadStatisticsPersistentStorage::initialize()
93 {
94     ASSERT(!RunLoop::isMain());
95     populateMemoryStoreFromDisk();
96     startMonitoringDisk();
97 }
98
99 ResourceLoadStatisticsPersistentStorage::~ResourceLoadStatisticsPersistentStorage()
100 {
101     finishAllPendingWorkSynchronously();
102     ASSERT(!m_hasPendingWrite);
103 }
104
105 String ResourceLoadStatisticsPersistentStorage::storageDirectoryPath() const
106 {
107     return m_storageDirectoryPath.isolatedCopy();
108 }
109
110 String ResourceLoadStatisticsPersistentStorage::resourceLogFilePath() const
111 {
112     String storagePath = storageDirectoryPath();
113     if (storagePath.isEmpty())
114         return emptyString();
115
116     return pathByAppendingComponent(storagePath, "full_browsing_session_resourceLog.plist");
117 }
118
119 void ResourceLoadStatisticsPersistentStorage::startMonitoringDisk()
120 {
121     ASSERT(!RunLoop::isMain());
122     if (m_fileMonitor)
123         return;
124
125     String resourceLogPath = resourceLogFilePath();
126     if (resourceLogPath.isEmpty())
127         return;
128
129     m_fileMonitor = std::make_unique<FileMonitor>(resourceLogPath, m_memoryStore.statisticsQueue(), [this] (FileMonitor::FileChangeType type) {
130         ASSERT(!RunLoop::isMain());
131         switch (type) {
132         case FileMonitor::FileChangeType::Modification:
133             refreshMemoryStoreFromDisk();
134             break;
135         case FileMonitor::FileChangeType::Removal:
136             m_memoryStore.clearInMemory();
137             m_fileMonitor = nullptr;
138             monitorDirectoryForNewStatistics();
139             break;
140         }
141     });
142 }
143
144 void ResourceLoadStatisticsPersistentStorage::monitorDirectoryForNewStatistics()
145 {
146     String storagePath = storageDirectoryPath();
147     ASSERT(!storagePath.isEmpty());
148
149     if (!fileExists(storagePath)) {
150         if (!makeAllDirectories(storagePath)) {
151             RELEASE_LOG_ERROR(ResourceLoadStatistics, "ResourceLoadStatisticsPersistentStorage: Failed to create directory path %s", storagePath.utf8().data());
152             return;
153         }
154     }
155
156     m_fileMonitor = std::make_unique<FileMonitor>(storagePath, m_memoryStore.statisticsQueue(), [this] (FileMonitor::FileChangeType type) {
157         ASSERT(!RunLoop::isMain());
158         if (type == FileMonitor::FileChangeType::Removal) {
159             // Directory was removed!
160             m_fileMonitor = nullptr;
161             return;
162         }
163
164         String resourceLogPath = resourceLogFilePath();
165         ASSERT(!resourceLogPath.isEmpty());
166
167         if (!fileExists(resourceLogPath))
168             return;
169
170         m_fileMonitor = nullptr;
171
172         refreshMemoryStoreFromDisk();
173         startMonitoringDisk();
174     });
175 }
176
177 void ResourceLoadStatisticsPersistentStorage::stopMonitoringDisk()
178 {
179     ASSERT(!RunLoop::isMain());
180     m_fileMonitor = nullptr;
181 }
182
183 // This is called when the file changes on disk.
184 void ResourceLoadStatisticsPersistentStorage::refreshMemoryStoreFromDisk()
185 {
186     ASSERT(!RunLoop::isMain());
187
188     String filePath = resourceLogFilePath();
189     if (filePath.isEmpty())
190         return;
191
192     // We sometimes see file changed events from before our load completed (we start
193     // reading at the first change event, but we might receive a series of events related
194     // to the same file operation). Catch this case to avoid reading overly often.
195     if (!hasFileChangedSince(filePath, m_lastStatisticsFileSyncTime))
196         return;
197
198     WallTime readTime = WallTime::now();
199
200     auto decoder = createDecoderForFile(filePath);
201     if (!decoder)
202         return;
203
204     m_memoryStore.mergeWithDataFromDecoder(*decoder);
205     m_lastStatisticsFileSyncTime = readTime;
206 }
207
208 void ResourceLoadStatisticsPersistentStorage::populateMemoryStoreFromDisk()
209 {
210     ASSERT(!RunLoop::isMain());
211
212     String filePath = resourceLogFilePath();
213     if (filePath.isEmpty() || !fileExists(filePath)) {
214         m_memoryStore.grandfatherExistingWebsiteData();
215         monitorDirectoryForNewStatistics();
216         return;
217     }
218
219     if (!hasFileChangedSince(filePath, m_lastStatisticsFileSyncTime)) {
220         // No need to grandfather in this case.
221         return;
222     }
223
224     WallTime readTime = WallTime::now();
225
226     auto decoder = createDecoderForFile(filePath);
227     if (!decoder) {
228         m_memoryStore.grandfatherExistingWebsiteData();
229         return;
230     }
231
232     ASSERT_WITH_MESSAGE(m_memoryStore.isEmpty(), "This is the initial import so the store should be empty");
233     m_memoryStore.mergeWithDataFromDecoder(*decoder);
234
235     m_lastStatisticsFileSyncTime = readTime;
236
237     m_memoryStore.logTestingEvent(ASCIILiteral("PopulatedWithoutGrandfathering"));
238 }
239
240 void ResourceLoadStatisticsPersistentStorage::asyncWriteTimerFired()
241 {
242     ASSERT(RunLoop::isMain());
243     m_memoryStore.statisticsQueue().dispatch([this] () mutable {
244         writeMemoryStoreToDisk();
245     });
246 }
247
248 void ResourceLoadStatisticsPersistentStorage::writeMemoryStoreToDisk()
249 {
250     ASSERT(!RunLoop::isMain());
251
252     m_hasPendingWrite = false;
253     stopMonitoringDisk();
254
255     auto encoder = m_memoryStore.createEncoderFromData();
256     auto rawData = encoder->finishEncoding();
257     if (!rawData)
258         return;
259
260     auto storagePath = storageDirectoryPath();
261     if (!storagePath.isEmpty()) {
262         makeAllDirectories(storagePath);
263         excludeFromBackup();
264     }
265
266     auto handle = openAndLockFile(resourceLogFilePath(), OpenForWrite);
267     if (handle == invalidPlatformFileHandle)
268         return;
269
270     int64_t writtenBytes = writeToFile(handle, rawData->data(), rawData->size());
271     unlockAndCloseFile(handle);
272
273     if (writtenBytes != static_cast<int64_t>(rawData->size()))
274         RELEASE_LOG_ERROR(ResourceLoadStatistics, "ResourceLoadStatisticsPersistentStorage: We only wrote %d out of %zu bytes to disk", static_cast<unsigned>(writtenBytes), rawData->size());
275
276     m_lastStatisticsFileSyncTime = WallTime::now();
277     m_lastStatisticsWriteTime = MonotonicTime::now();
278
279     startMonitoringDisk();
280 }
281
282 void ResourceLoadStatisticsPersistentStorage::scheduleOrWriteMemoryStore(ForceImmediateWrite forceImmediateWrite)
283 {
284     ASSERT(!RunLoop::isMain());
285
286     auto timeSinceLastWrite = MonotonicTime::now() - m_lastStatisticsWriteTime;
287     if (forceImmediateWrite != ForceImmediateWrite::Yes && timeSinceLastWrite < minimumWriteInterval) {
288         if (!m_hasPendingWrite) {
289             m_hasPendingWrite = true;
290             Seconds delay = minimumWriteInterval - timeSinceLastWrite + 1_s;
291             RunLoop::main().dispatch([this, protectedThis = makeRef(*this), delay] {
292                 m_asyncWriteTimer.startOneShot(delay);
293             });
294         }
295         return;
296     }
297
298     writeMemoryStoreToDisk();
299 }
300
301 void ResourceLoadStatisticsPersistentStorage::clear()
302 {
303     ASSERT(!RunLoop::isMain());
304     String filePath = resourceLogFilePath();
305     if (filePath.isEmpty())
306         return;
307
308     stopMonitoringDisk();
309
310     if (!deleteFile(filePath))
311         RELEASE_LOG_ERROR(ResourceLoadStatistics, "ResourceLoadStatisticsPersistentStorage: Unable to delete statistics file: %s", filePath.utf8().data());
312 }
313
314 void ResourceLoadStatisticsPersistentStorage::finishAllPendingWorkSynchronously()
315 {
316     m_asyncWriteTimer.stop();
317
318     BinarySemaphore semaphore;
319     // Make sure any pending work in our queue is finished before we terminate.
320     m_memoryStore.statisticsQueue().dispatch([&semaphore, this] {
321         // Write final file state to disk.
322         if (m_hasPendingWrite)
323             writeMemoryStoreToDisk();
324         semaphore.signal();
325     });
326     semaphore.wait(WallTime::infinity());
327 }
328
329 void ResourceLoadStatisticsPersistentStorage::ref()
330 {
331     m_memoryStore.ref();
332 }
333
334 void ResourceLoadStatisticsPersistentStorage::deref()
335 {
336     m_memoryStore.deref();
337 }
338
339 #if !PLATFORM(IOS)
340 void ResourceLoadStatisticsPersistentStorage::excludeFromBackup() const
341 {
342 }
343 #endif
344
345 } // namespace WebKit