IndexedDB: WAL file keeps growing
authorsihui_liu@apple.com <sihui_liu@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 6 Nov 2018 20:56:54 +0000 (20:56 +0000)
committersihui_liu@apple.com <sihui_liu@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 6 Nov 2018 20:56:54 +0000 (20:56 +0000)
https://bugs.webkit.org/show_bug.cgi?id=191294
<rdar://problem/41333493>

Reviewed by Chris Dumez.

Source/WebCore:

When we quit apps, the database connection may not be shut down properly, and WAL file will be retained on disk.
On the next open of database connection, new logs will be appended to the original WAL file, which keeps
increasing size of the WAL file. We should do a manual checkpoint when we open a IndexedDB to make sure previous
log is written to database and truncate WAL file.

Test: IndexedDB.IndexedDBTempFileSize

* platform/sql/SQLiteDatabase.cpp:
(WebCore::SQLiteDatabase::open):

Tools:

* TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj:
* TestWebKitAPI/Tests/WebKitCocoa/IndexedDBTempFileSize-1.html: Added.
* TestWebKitAPI/Tests/WebKitCocoa/IndexedDBTempFileSize-2.html: Added.
* TestWebKitAPI/Tests/WebKitCocoa/IndexedDBTempFileSize.mm: Added.
(-[IndexedDBFileSizeMessageHandler userContentController:didReceiveScriptMessage:]):
(TEST):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@237882 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WebCore/ChangeLog
Source/WebCore/platform/sql/SQLiteDatabase.cpp
Tools/ChangeLog
Tools/TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj
Tools/TestWebKitAPI/Tests/WebKitCocoa/IndexedDBTempFileSize-1.html [new file with mode: 0644]
Tools/TestWebKitAPI/Tests/WebKitCocoa/IndexedDBTempFileSize-2.html [new file with mode: 0644]
Tools/TestWebKitAPI/Tests/WebKitCocoa/IndexedDBTempFileSize.mm [new file with mode: 0644]

index 0283e07..91a18e9 100644 (file)
@@ -1,3 +1,21 @@
+2018-11-06  Sihui Liu  <sihui_liu@apple.com>
+
+        IndexedDB: WAL file keeps growing
+        https://bugs.webkit.org/show_bug.cgi?id=191294
+        <rdar://problem/41333493>
+
+        Reviewed by Chris Dumez.
+
+        When we quit apps, the database connection may not be shut down properly, and WAL file will be retained on disk.
+        On the next open of database connection, new logs will be appended to the original WAL file, which keeps 
+        increasing size of the WAL file. We should do a manual checkpoint when we open a IndexedDB to make sure previous
+        log is written to database and truncate WAL file.
+
+        Test: IndexedDB.IndexedDBTempFileSize
+
+        * platform/sql/SQLiteDatabase.cpp:
+        (WebCore::SQLiteDatabase::open):
+
 2018-11-06  Ali Juma  <ajuma@chromium.org>
 
         IntersectionObserver doesn't keep target's JS wrapper alive
index 0d436be..3c499a2 100644 (file)
@@ -122,6 +122,13 @@ bool SQLiteDatabase::open(const String& filename, bool forWebSQLDatabase)
     } else
         LOG_ERROR("SQLite database failed to set journal_mode to WAL, error: %s", lastErrorMsg());
 
+    SQLiteStatement checkpointStatement(*this, "PRAGMA wal_checkpoint(TRUNCATE)"_s);
+    if (checkpointStatement.prepareAndStep() == SQLITE_ROW) {
+        if (checkpointStatement.getColumnInt(0))
+            LOG(SQLDatabase, "SQLite database checkpoint is blocked");
+    } else
+        LOG_ERROR("SQLite database failed to checkpoint: %s", lastErrorMsg());
+
     return isOpen();
 }
 
index 1cca495..d2500ff 100644 (file)
@@ -1,3 +1,18 @@
+2018-11-06  Sihui Liu  <sihui_liu@apple.com>
+
+        IndexedDB: WAL file keeps growing
+        https://bugs.webkit.org/show_bug.cgi?id=191294
+        <rdar://problem/41333493>
+
+        Reviewed by Chris Dumez.
+
+        * TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj:
+        * TestWebKitAPI/Tests/WebKitCocoa/IndexedDBTempFileSize-1.html: Added.
+        * TestWebKitAPI/Tests/WebKitCocoa/IndexedDBTempFileSize-2.html: Added.
+        * TestWebKitAPI/Tests/WebKitCocoa/IndexedDBTempFileSize.mm: Added.
+        (-[IndexedDBFileSizeMessageHandler userContentController:didReceiveScriptMessage:]):
+        (TEST):
+
 2018-11-06  Alex Christensen  <achristensen@webkit.org>
 
         Fix tests after r237863
index 1b62665..a2e5925 100644 (file)
                C9C60E661E53A9DC006DA181 /* autoplay-check-in-iframe.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = C9C60E641E53A9BA006DA181 /* autoplay-check-in-iframe.html */; };
                C9E6DD351EA97D0800DD78AA /* FirstResponderSuppression.mm in Sources */ = {isa = PBXBuildFile; fileRef = C9E6DD311EA972D800DD78AA /* FirstResponderSuppression.mm */; };
                CA38459620AE17A900990D3B /* LocalStorageDatabaseTracker.mm in Sources */ = {isa = PBXBuildFile; fileRef = CA38459520AE012E00990D3B /* LocalStorageDatabaseTracker.mm */; };
+               CA5B94D22190C0F40059FE38 /* IndexedDBTempFileSize-1.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = CA5B94D02190C0E00059FE38 /* IndexedDBTempFileSize-1.html */; };
+               CA5B94D32190C0F40059FE38 /* IndexedDBTempFileSize-2.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = CA5B94D12190C0E00059FE38 /* IndexedDBTempFileSize-2.html */; };
+               CA5B94D72191005B0059FE38 /* IndexedDBTempFileSize.mm in Sources */ = {isa = PBXBuildFile; fileRef = CA5B94D62191005B0059FE38 /* IndexedDBTempFileSize.mm */; };
                CD0BD0A61F79924D001AB2CF /* ContextMenuImgWithVideo.mm in Sources */ = {isa = PBXBuildFile; fileRef = CD0BD0A51F799220001AB2CF /* ContextMenuImgWithVideo.mm */; };
                CD0BD0A81F79982D001AB2CF /* ContextMenuImgWithVideo.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = CD0BD0A71F7997C2001AB2CF /* ContextMenuImgWithVideo.html */; };
                CD227E44211A4D5D00D285AF /* PreferredAudioBufferSize.mm in Sources */ = {isa = PBXBuildFile; fileRef = CD227E43211A4D5D00D285AF /* PreferredAudioBufferSize.mm */; };
                                571F7FD01F2961FB00946648 /* IndexedDBStructuredCloneBackwardCompatibility.sqlite3-wal in Copy Resources */,
                                57599E2A1F071AA000A3FB8C /* IndexedDBStructuredCloneBackwardCompatibilityRead.html in Copy Resources */,
                                57599E2B1F071AA000A3FB8C /* IndexedDBStructuredCloneBackwardCompatibilityWrite.html in Copy Resources */,
+                               CA5B94D22190C0F40059FE38 /* IndexedDBTempFileSize-1.html in Copy Resources */,
+                               CA5B94D32190C0F40059FE38 /* IndexedDBTempFileSize-2.html in Copy Resources */,
                                5110FCF91E01CD8A006F8D0B /* IndexUpgrade.blob in Copy Resources */,
                                5110FCF61E01CD83006F8D0B /* IndexUpgrade.sqlite3 in Copy Resources */,
                                2EFF06CD1D8A429A0004BB30 /* input-field-in-scrollable-document.html in Copy Resources */,
                C9C60E641E53A9BA006DA181 /* autoplay-check-in-iframe.html */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.html; path = "autoplay-check-in-iframe.html"; sourceTree = "<group>"; };
                C9E6DD311EA972D800DD78AA /* FirstResponderSuppression.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FirstResponderSuppression.mm; sourceTree = "<group>"; };
                CA38459520AE012E00990D3B /* LocalStorageDatabaseTracker.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = LocalStorageDatabaseTracker.mm; sourceTree = "<group>"; };
+               CA5B94D02190C0E00059FE38 /* IndexedDBTempFileSize-1.html */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.html; path = "IndexedDBTempFileSize-1.html"; sourceTree = "<group>"; };
+               CA5B94D12190C0E00059FE38 /* IndexedDBTempFileSize-2.html */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.html; path = "IndexedDBTempFileSize-2.html"; sourceTree = "<group>"; };
+               CA5B94D62191005B0059FE38 /* IndexedDBTempFileSize.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = IndexedDBTempFileSize.mm; sourceTree = "<group>"; };
                CD0BD0A51F799220001AB2CF /* ContextMenuImgWithVideo.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = ContextMenuImgWithVideo.mm; sourceTree = "<group>"; };
                CD0BD0A71F7997C2001AB2CF /* ContextMenuImgWithVideo.html */ = {isa = PBXFileReference; lastKnownFileType = text.html; path = ContextMenuImgWithVideo.html; sourceTree = "<group>"; };
                CD225C071C45A69200140761 /* ParsedContentRange.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ParsedContentRange.cpp; sourceTree = "<group>"; };
                                51BCEE491C84F4AF0042C82E /* IndexedDBMultiProcess.mm */,
                                51B1EE8D1C80F5880064FB98 /* IndexedDBPersistence.mm */,
                                57599E201F07191700A3FB8C /* IndexedDBStructuredCloneBackwardCompatibility.mm */,
+                               CA5B94D62191005B0059FE38 /* IndexedDBTempFileSize.mm */,
                                0E404A8A2166DDF8008271BA /* InjectedBundleNodeHandleIsSelectElement.mm */,
                                79C5D430209D768300F1E7CA /* InjectedBundleNodeHandleIsTextField.mm */,
                                2DB0232E1E4E871800707123 /* InteractionDeadlockAfterCrash.mm */,
                                571F7FCF1F2961E100946648 /* IndexedDBStructuredCloneBackwardCompatibility.sqlite3-wal */,
                                57599E251F07192C00A3FB8C /* IndexedDBStructuredCloneBackwardCompatibilityRead.html */,
                                57599E231F07192C00A3FB8C /* IndexedDBStructuredCloneBackwardCompatibilityWrite.html */,
+                               CA5B94D02190C0E00059FE38 /* IndexedDBTempFileSize-1.html */,
+                               CA5B94D12190C0E00059FE38 /* IndexedDBTempFileSize-2.html */,
                                5110FCF21E01CD77006F8D0B /* IndexUpgrade.blob */,
                                5110FCF31E01CD77006F8D0B /* IndexUpgrade.sqlite3 */,
                                2EFF06CC1D8A42910004BB30 /* input-field-in-scrollable-document.html */,
                                7C83E0BE1D0A651300FEBCF3 /* IndexedDBMultiProcess.mm in Sources */,
                                7C83E0BF1D0A652200FEBCF3 /* IndexedDBPersistence.mm in Sources */,
                                57599E211F07191900A3FB8C /* IndexedDBStructuredCloneBackwardCompatibility.mm in Sources */,
+                               CA5B94D72191005B0059FE38 /* IndexedDBTempFileSize.mm in Sources */,
                                7A95BDE11E9BEC5F00865498 /* InjectedBundleAppleEvent.cpp in Sources */,
                                7CCE7EFB1A411AE600447C4C /* InjectedBundleBasic.cpp in Sources */,
                                83148B06202AC6A400BADE99 /* InjectedBundleDisableOverrideBuiltinsBehavior.cpp in Sources */,
diff --git a/Tools/TestWebKitAPI/Tests/WebKitCocoa/IndexedDBTempFileSize-1.html b/Tools/TestWebKitAPI/Tests/WebKitCocoa/IndexedDBTempFileSize-1.html
new file mode 100644 (file)
index 0000000..2e12365
--- /dev/null
@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<script>
+
+var openRequest = window.indexedDB.open("IndexedDBTempFileSize");
+openRequest.onupgradeneeded = function(event) {
+    window.webkit.messageHandlers.testHandler.postMessage('UpgradeNeeded');
+
+    event.target.result.createObjectStore("TestObjectStore");
+} 
+
+openRequest.onsuccess = function(event) {
+    window.webkit.messageHandlers.testHandler.postMessage('Success');
+    var transaction = event.target.result.transaction(["TestObjectStore"], "readwrite");
+    var objectStore = transaction.objectStore("TestObjectStore");
+    for (let i = 0; i < 100; i++) {
+        objectStore.add(i, i);
+    }
+}
+
+openRequest.onerror = function(event) {
+    window.webkit.messageHandlers.testHandler.postMessage('Error');
+}
+
+</script>
\ No newline at end of file
diff --git a/Tools/TestWebKitAPI/Tests/WebKitCocoa/IndexedDBTempFileSize-2.html b/Tools/TestWebKitAPI/Tests/WebKitCocoa/IndexedDBTempFileSize-2.html
new file mode 100644 (file)
index 0000000..63698ea
--- /dev/null
@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<script>
+
+var openRequest = window.indexedDB.open("IndexedDBTempFileSize");
+
+openRequest.onsuccess = function(event) {
+    window.webkit.messageHandlers.testHandler.postMessage('Success');
+}
+
+openRequest.onerror = function(event) {
+    window.webkit.messageHandlers.testHandler.postMessage('Error');
+}
+
+</script>
\ No newline at end of file
diff --git a/Tools/TestWebKitAPI/Tests/WebKitCocoa/IndexedDBTempFileSize.mm b/Tools/TestWebKitAPI/Tests/WebKitCocoa/IndexedDBTempFileSize.mm
new file mode 100644 (file)
index 0000000..87128c2
--- /dev/null
@@ -0,0 +1,120 @@
+/*
+ * Copyright (C) 2018 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#import "config.h"
+
+#import "PlatformUtilities.h"
+#import "Test.h"
+#import <WebKit/WKProcessPoolPrivate.h>
+#import <WebKit/WKUserContentControllerPrivate.h>
+#import <WebKit/WKWebViewConfigurationPrivate.h>
+#import <WebKit/WKWebViewPrivate.h>
+#import <WebKit/WKWebsiteDataStorePrivate.h>
+#import <WebKit/WebKit.h>
+#import <WebKit/_WKProcessPoolConfiguration.h>
+#import <WebKit/_WKUserStyleSheet.h>
+#import <WebKit/_WKWebsiteDataStoreConfiguration.h>
+#import <wtf/RetainPtr.h>
+
+#if WK_API_ENABLED
+
+static bool readyToContinue;
+static bool receivedScriptMessage;
+static RetainPtr<WKScriptMessage> lastScriptMessage;
+
+@interface IndexedDBFileSizeMessageHandler : NSObject <WKScriptMessageHandler>
+@end
+
+@implementation IndexedDBFileSizeMessageHandler
+
+- (void)userContentController:(WKUserContentController *)userContentController didReceiveScriptMessage:(WKScriptMessage *)message
+{
+    receivedScriptMessage = true;
+    lastScriptMessage = message;
+}
+
+@end
+
+TEST(IndexedDB, IndexedDBTempFileSize)
+{
+    auto handler = adoptNS([[IndexedDBFileSizeMessageHandler alloc] init]);
+    auto configuration = adoptNS([[WKWebViewConfiguration alloc] init]);
+    [[configuration userContentController] addScriptMessageHandler:handler.get() name:@"testHandler"];
+
+    RetainPtr<NSURL> idbPath = [NSURL fileURLWithPath:[@"~/Library/WebKit/TestWebKitAPI/CustomWebsiteData/IndexedDB/" stringByExpandingTildeInPath] isDirectory:YES];
+    RetainPtr<NSURL> walFilePath = [NSURL fileURLWithPath:[@"~/Library/WebKit/TestWebKitAPI/CustomWebsiteData/IndexedDB/file__0/IndexedDBTempFileSize/IndexedDB.sqlite3-wal" stringByExpandingTildeInPath] isDirectory:NO];
+
+    auto websiteDataStoreConfiguration = adoptNS([[_WKWebsiteDataStoreConfiguration alloc] init]);
+    websiteDataStoreConfiguration.get()._indexedDBDatabaseDirectory = idbPath.get();
+
+    configuration.get().websiteDataStore = [[[WKWebsiteDataStore alloc] _initWithConfiguration:websiteDataStoreConfiguration.get()] autorelease];
+    auto types = adoptNS([[NSSet alloc] initWithObjects:WKWebsiteDataTypeIndexedDBDatabases, nil]);
+
+    [configuration.get().websiteDataStore removeDataOfTypes:types.get() modifiedSince:[NSDate distantPast] completionHandler:^() {
+        readyToContinue = true;
+    }];
+    readyToContinue = false;
+    TestWebKitAPI::Util::run(&readyToContinue);
+
+    // Do some IndexedDB operations to generate WAL file.
+    auto webView = adoptNS([[WKWebView alloc] initWithFrame:NSMakeRect(0, 0, 800, 600) configuration:configuration.get()]);
+    NSURLRequest *request = [NSURLRequest requestWithURL:[[NSBundle mainBundle] URLForResource:@"IndexedDBTempFileSize-1" withExtension:@"html" subdirectory:@"TestWebKitAPI.resources"]];
+    [webView loadRequest:request];
+
+    receivedScriptMessage = false;
+    TestWebKitAPI::Util::run(&receivedScriptMessage);
+    RetainPtr<NSString> string1 = (NSString *)[lastScriptMessage body];
+
+    receivedScriptMessage = false;
+    TestWebKitAPI::Util::run(&receivedScriptMessage);
+    RetainPtr<NSString> string2 = (NSString *)[lastScriptMessage body];
+
+    // Terminate network process to keep WAL on disk.
+    webView = nil;
+    [configuration.get().processPool _terminateNetworkProcess];
+
+    EXPECT_TRUE([[NSFileManager defaultManager] fileExistsAtPath:walFilePath.get().path]);
+    RetainPtr<NSDictionary> fileAttributes = [[NSFileManager defaultManager] attributesOfItemAtPath:walFilePath.get().path error:nil];
+    NSNumber *fileSizeBefore = [fileAttributes objectForKey:NSFileSize];
+
+    // Open the same database again.
+    webView = adoptNS([[WKWebView alloc] initWithFrame:NSMakeRect(0, 0, 800, 600) configuration:configuration.get()]);
+    request = [NSURLRequest requestWithURL:[[NSBundle mainBundle] URLForResource:@"IndexedDBTempFileSize-2" withExtension:@"html" subdirectory:@"TestWebKitAPI.resources"]];
+    [webView loadRequest:request];
+
+    receivedScriptMessage = false;
+    TestWebKitAPI::Util::run(&receivedScriptMessage);
+    RetainPtr<NSString> string3 = (NSString *)[lastScriptMessage body];
+
+    fileAttributes = [[NSFileManager defaultManager] attributesOfItemAtPath:walFilePath.get().path error:nil];
+    NSNumber *fileSizeAfter = [fileAttributes objectForKey:NSFileSize];
+    EXPECT_GT([fileSizeBefore longLongValue], [fileSizeAfter longLongValue]);
+
+    EXPECT_WK_STREQ(@"UpgradeNeeded", string1.get());
+    EXPECT_WK_STREQ(@"Success", string2.get());
+    EXPECT_WK_STREQ(@"Success", string3.get()); 
+}
+
+#endif