Reviewed by Kevin.
[WebKit-https.git] / WebKit / Misc.subproj / WebFileDatabase.m
1 /*      
2     WebFileDatabase.m
3     Copyright (C) 2003 Apple Computer, Inc. All rights reserved.    
4 */
5
6 #import <WebKit/WebFileDatabase.h>
7 #import <WebKit/WebKitLogging.h>
8 #import <WebKit/WebLRUFileList.h>
9 #import <WebKit/WebNSFileManagerExtras.h>
10
11 #import <fcntl.h>
12 #import <fts.h>
13 #import <pthread.h>
14 #import <string.h>
15 #import <sys/stat.h>
16 #import <sys/types.h>
17 #import <sys/mman.h>
18
19 #if ERROR_DISABLED
20 #define BEGIN_EXCEPTION_HANDLER
21 #define END_EXCEPTION_HANDLER
22 #else
23 #define BEGIN_EXCEPTION_HANDLER NS_DURING
24 #define END_EXCEPTION_HANDLER \
25     NS_HANDLER \
26         ERROR("Uncaught exception: %@ [%@] [%@]", [localException class], [localException reason], [localException userInfo]); \
27     NS_ENDHANDLER
28 #endif
29
30 static pthread_once_t databaseInitControl = PTHREAD_ONCE_INIT;
31 static NSNumber *WebFileDirectoryPOSIXPermissions;
32 static NSNumber *WebFilePOSIXPermissions;
33 static NSRunLoop *syncRunLoop;
34
35 #define UniqueFilePathSize (34)
36 static void UniqueFilePathForKey(id key, char *buffer);
37
38 #define MinThreadPriority (10)
39 static int SetThreadPriority(int priority);
40
41 typedef enum
42 {
43     WebFileDatabaseSetObjectOp,
44     WebFileDatabaseRemoveObjectOp,
45 } WebFileDatabaseOpcode;
46
47 enum
48 {
49     MAX_UNSIGNED_LENGTH = 20, // long enough to hold the string representation of a 64-bit unsigned number
50     SYNC_IDLE_THRESHOLD = 10,
51 };
52
53 // interface WebFileDatabaseOp -------------------------------------------------------------
54
55 @interface WebFileDatabaseOp : NSObject
56 {
57     WebFileDatabaseOpcode opcode;
58     id key;
59     id object; 
60 }
61
62 +(id)opWithCode:(WebFileDatabaseOpcode)opcode key:(id)key object:(id)object;
63 -(id)initWithCode:(WebFileDatabaseOpcode)opcode key:(id)key object:(id)object;
64
65 -(WebFileDatabaseOpcode)opcode;
66 -(id)key;
67 -(id)object;
68 -(void)perform:(WebFileDatabase *)target;
69
70 @end
71
72
73 // implementation WebFileDatabaseOp -------------------------------------------------------------
74
75 @implementation WebFileDatabaseOp
76
77 +(id)opWithCode:(WebFileDatabaseOpcode)theOpcode key:(id)theKey object:(id)theObject
78 {
79     return [[[WebFileDatabaseOp alloc] initWithCode:theOpcode key:theKey object:theObject] autorelease];
80 }
81
82 -(id)initWithCode:(WebFileDatabaseOpcode)theOpcode key:(id)theKey object:(id)theObject
83 {
84     ASSERT(theKey);
85
86     if ((self = [super init])) {
87         
88         opcode = theOpcode;
89         key = [theKey retain];
90         object = [theObject retain];
91         
92         return self;
93     }
94   
95     return nil;
96 }
97
98 -(WebFileDatabaseOpcode)opcode
99 {
100     return opcode;
101 }
102
103 -(id)key
104 {
105     return key;
106 }
107
108 -(id)object
109 {
110     return object;
111 }
112
113 -(void)perform:(WebFileDatabase *)target
114 {
115     ASSERT(target);
116
117     switch (opcode) {
118         case WebFileDatabaseSetObjectOp:
119             [target performSetObject:object forKey:key];
120             break;
121         case WebFileDatabaseRemoveObjectOp:
122             [target performRemoveObjectForKey:key];
123             break;
124         default:
125             ASSERT_NOT_REACHED();
126             break;
127     }
128 }
129
130 -(void)dealloc
131 {
132     [key release];
133     [object release];
134     
135     [super dealloc];
136 }
137
138 @end
139
140
141 // interface WebFileDatabasePrivate -----------------------------------------------------------
142
143 @interface WebFileDatabase (WebFileDatabasePrivate)
144
145 -(void)_createLRUList:(id)arg;
146 -(void)_truncateToSizeLimit:(unsigned)size;
147
148 @end
149
150 // implementation WebFileDatabasePrivate ------------------------------------------------------
151
152 @implementation WebFileDatabase (WebFileDatabasePrivate)
153
154 static int SetThreadPriority(int priority) 
155 {
156     struct sched_param sp;
157
158     memset(&sp, 0, sizeof(struct sched_param));
159     sp.sched_priority=priority;
160     if (pthread_setschedparam(pthread_self(), SCHED_OTHER, &sp) == -1) {
161         ERROR("Failed to change priority.");
162         return -1;
163     }
164     return 0;
165 }
166
167 static void UniqueFilePathForKey(id key, char *buffer)
168 {
169     const char *s;
170     UInt32 hash1;
171     UInt32 hash2;
172     CFIndex len;
173     CFIndex cnt;
174
175     s = [[[[key description] lowercaseString] stringByStandardizingPath] lossyCString];
176     len = strlen(s);
177
178     // compute first hash    
179     hash1 = len;
180     for (cnt = 0; cnt < len; cnt++) {
181         hash1 += (hash1 << 8) + s[cnt];
182     }
183     hash1 += (hash1 << (len & 31));
184
185     // compute second hash    
186     hash2 = len;
187     for (cnt = 0; cnt < len; cnt++) {
188         hash2 = (37 * hash2) ^ s[cnt];
189     }
190
191     snprintf(buffer, UniqueFilePathSize, "%.2lu/%.2lu/%.10lu-%.10lu.cache", ((hash1 & 0xff) >> 4), ((hash2 & 0xff) >> 4), hash1, hash2);
192 }
193
194 -(void)_createLRUList:(id)arg
195 {
196     SetThreadPriority(MinThreadPriority + 1); // make this a little higher priority than the syncRunLoop thread
197
198     NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
199
200     BEGIN_EXCEPTION_HANDLER
201     
202     WebLRUFileList *fileList = WebLRUFileListCreate();
203     WebLRUFileListRebuildFileDataUsingRootDirectory(fileList, [path fileSystemRepresentation]);
204     lru = fileList;
205
206     END_EXCEPTION_HANDLER
207
208     LOG(FileDatabaseActivity, "lru list created");
209
210 #if BUILDING_ON_PANTHER
211     [pool release];
212 #else
213     [pool drain];
214 #endif
215 }
216
217 -(void)_truncateToSizeLimit:(unsigned)size
218 {
219     NSFileManager *defaultManager;
220     
221     if (!lru || size > [self usage]) {
222         return;
223     }
224
225     if (size == 0) {
226         [self removeAllObjects];
227     }
228     else {
229         defaultManager = [NSFileManager defaultManager];
230         [mutex lock];
231         while ([self usage] > size) {
232             char uniqueKey[UniqueFilePathSize];
233             if (!WebLRUFileListGetPathOfOldestFile(lru, uniqueKey, UniqueFilePathSize)) {
234                 break;
235             }
236             NSString *filePath = [[NSString alloc] initWithFormat:@"%@/%s", path, uniqueKey];
237             [defaultManager _webkit_removeFileOnlyAtPath:filePath];
238             [filePath release];
239             WebLRUFileListRemoveOldestFileFromList(lru);
240         }
241         [mutex unlock];
242     }
243 }
244
245 @end
246
247
248 // implementation WebFileDatabase -------------------------------------------------------------
249
250 @implementation WebFileDatabase
251
252 // creation functions ---------------------------------------------------------------------------
253 #pragma mark creation functions
254
255 +(void)_syncLoop:(id)arg
256 {
257     SetThreadPriority(MinThreadPriority);
258
259     NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
260     NSPort *placeholderPort;
261
262     BEGIN_EXCEPTION_HANDLER
263
264     syncRunLoop = [NSRunLoop currentRunLoop];
265
266     while (YES) {
267         BEGIN_EXCEPTION_HANDLER
268         // we specifically use an NSRunLoop here to get autorelease pool support
269         placeholderPort = [NSPort port];
270         [syncRunLoop addPort:placeholderPort forMode:NSDefaultRunLoopMode];
271         [syncRunLoop run];
272         [syncRunLoop removePort:placeholderPort forMode:NSDefaultRunLoopMode];
273         END_EXCEPTION_HANDLER
274     }
275
276     END_EXCEPTION_HANDLER
277
278 #if BUILDING_ON_PANTHER
279     [pool release];
280 #else
281     [pool drain];
282 #endif
283 }
284
285 static void databaseInit()
286 {
287     // set file perms to owner read/write/execute only
288     WebFileDirectoryPOSIXPermissions = [[NSNumber numberWithInt:(WEB_UREAD | WEB_UWRITE | WEB_UEXEC)] retain];
289
290     // set file perms to owner read/write only
291     WebFilePOSIXPermissions = [[NSNumber numberWithInt:(WEB_UREAD | WEB_UWRITE)] retain];
292
293     [NSThread detachNewThreadSelector:@selector(_syncLoop:) toTarget:[WebFileDatabase class] withObject:nil];
294 }
295
296 -(id)initWithPath:(NSString *)thePath
297 {
298     pthread_once(&databaseInitControl, databaseInit);
299
300     [super initWithPath:thePath];
301     
302     if (self == nil || thePath == nil) {
303         [self release];
304         return nil;
305     }
306
307     ops = [[NSMutableArray alloc] init];
308     setCache = [[NSMutableDictionary alloc] init];
309     removeCache = [[NSMutableSet alloc] init];
310     timer = nil;
311     mutex = [[NSRecursiveLock alloc] init];
312     
313     return self;
314 }
315
316 // WebFileDatabase objects are never released, so we need no dealloc implementation.
317
318 -(void)setTimer
319 {
320     if (timer == nil) {
321         NSDate *fireDate = [[NSDate alloc] initWithTimeIntervalSinceNow:SYNC_IDLE_THRESHOLD];
322         timer = [[NSTimer alloc] initWithFireDate:fireDate interval:SYNC_IDLE_THRESHOLD target:self selector:@selector(lazySync:) userInfo:nil repeats:YES];
323         [fireDate release];
324         [syncRunLoop addTimer:timer forMode:NSDefaultRunLoopMode];
325     }
326 }
327
328 // database functions ---------------------------------------------------------------------------
329 #pragma mark database functions
330
331 -(void)setObject:(id)object forKey:(id)key
332 {
333     WebFileDatabaseOp *op;
334
335     ASSERT(object);
336     ASSERT(key);
337
338     touch = CFAbsoluteTimeGetCurrent();
339
340     LOG(FileDatabaseActivity, "%p - %@", object, key);
341     
342     [mutex lock];
343     
344     [setCache setObject:object forKey:key];
345     op = [[WebFileDatabaseOp alloc] initWithCode:WebFileDatabaseSetObjectOp key:key object:object];
346     [ops addObject:op];
347     [op release];
348     [self setTimer];
349     
350     [mutex unlock];
351 }
352
353 -(void)removeObjectForKey:(id)key
354 {
355     WebFileDatabaseOp *op;
356
357     ASSERT(key);
358
359     touch = CFAbsoluteTimeGetCurrent();
360     
361     [mutex lock];
362     
363     [removeCache addObject:key];
364     op = [[WebFileDatabaseOp alloc] initWithCode:WebFileDatabaseRemoveObjectOp key:key object:nil];
365     [ops addObject:op];
366     [op release];
367     [self setTimer];
368     
369     [mutex unlock];
370 }
371
372 -(void)removeAllObjects
373 {
374     touch = CFAbsoluteTimeGetCurrent();
375
376     [mutex lock];
377     [setCache removeAllObjects];
378     [removeCache removeAllObjects];
379     [ops removeAllObjects];
380     [self close];
381     [[NSFileManager defaultManager] _webkit_backgroundRemoveFileAtPath:path];
382     [self open];
383     [mutex unlock];
384
385     LOG(FileDatabaseActivity, "removeAllObjects");
386 }
387
388 -(id)objectForKey:(id)key
389 {
390     volatile id result;
391     
392     ASSERT(key);
393
394     touch = CFAbsoluteTimeGetCurrent();
395
396     // check caches
397     [mutex lock];
398     if ([removeCache containsObject:key]) {
399         [mutex unlock];
400         return nil;
401     }
402     if ((result = [setCache objectForKey:key])) {
403         [mutex unlock];
404         return result;
405     }
406     [mutex unlock];
407
408     // go to disk
409     char uniqueKey[UniqueFilePathSize];
410     UniqueFilePathForKey(key, uniqueKey);
411     NSString *filePath = [[NSString alloc] initWithFormat:@"%@/%s", path, uniqueKey];
412     NSData *data = [[NSData alloc] initWithContentsOfFile:filePath];
413     NSUnarchiver * volatile unarchiver = nil;
414
415     NS_DURING
416         if (data) {
417             unarchiver = [[NSUnarchiver alloc] initForReadingWithData:data];
418             if (unarchiver) {
419                 id fileKey = [unarchiver decodeObject];
420                 if ([fileKey isEqual:key]) {
421                     id object = [unarchiver decodeObject];
422                     if (object) {
423                         // Decoded objects go away when the unarchiver does, so we need to
424                         // retain this so we can return it to our caller.
425                         result = [[object retain] autorelease];
426                         if (lru) {
427                             // if we can't update the list yet, that's too bad
428                             // but not critically bad
429                             WebLRUFileListTouchFileWithPath(lru, uniqueKey);
430                         }
431                         LOG(FileDatabaseActivity, "read disk cache file - %@", key);
432                     }
433                 }
434             }
435         }
436     NS_HANDLER
437         LOG(FileDatabaseActivity, "cannot unarchive cache file - %@", key);
438         result = nil;
439     NS_ENDHANDLER
440
441     [unarchiver release];
442     [data release];
443     [filePath release];
444
445     LOG(Timing, "getting value for %@ took %f", key, (CFAbsoluteTimeGetCurrent() - touch));
446     
447     return result;
448 }
449
450 -(void)performSetObject:(id)object forKey:(id)key
451 {
452     NSString *filePath;
453     NSMutableData *data;
454     NSDictionary *attributes;
455     NSDictionary *directoryAttributes;
456     NSArchiver *archiver;
457     NSFileManager *defaultManager;
458     char uniqueKey[UniqueFilePathSize];
459     BOOL result;
460
461     ASSERT(object);
462     ASSERT(key);
463
464     UniqueFilePathForKey(key, uniqueKey);
465
466     LOG(FileDatabaseActivity, "%@ - %s", key, uniqueKey);
467
468     data = [NSMutableData data];
469     archiver = [[NSArchiver alloc] initForWritingWithMutableData:data];
470     [archiver encodeObject:key];
471     [archiver encodeObject:object];
472     
473     attributes = [NSDictionary dictionaryWithObjectsAndKeys:
474         NSUserName(), NSFileOwnerAccountName,
475         WebFilePOSIXPermissions, NSFilePosixPermissions,
476         NULL
477     ];
478
479     directoryAttributes = [NSDictionary dictionaryWithObjectsAndKeys:
480         NSUserName(), NSFileOwnerAccountName,
481         WebFileDirectoryPOSIXPermissions, NSFilePosixPermissions,
482         NULL
483     ];
484
485     defaultManager = [NSFileManager defaultManager];
486
487     filePath = [[NSString alloc] initWithFormat:@"%@/%s", path, uniqueKey];
488     attributes = [defaultManager fileAttributesAtPath:filePath traverseLink:YES];
489
490     // update usage and truncate before writing file
491     // this has the effect of _always_ keeping disk usage under sizeLimit by clearing away space in anticipation of the write.
492     WebLRUFileListSetFileData(lru, uniqueKey, [data length], CFAbsoluteTimeGetCurrent());
493     [self _truncateToSizeLimit:[self sizeLimit]];
494
495     result = [defaultManager _webkit_createFileAtPathWithIntermediateDirectories:filePath contents:data attributes:attributes directoryAttributes:directoryAttributes];
496
497     if (!result) {
498         WebLRUFileListRemoveFileWithPath(lru, uniqueKey);
499     }
500
501     [archiver release];
502     [filePath release];    
503 }
504
505 -(void)performRemoveObjectForKey:(id)key
506 {
507     NSString *filePath;
508     char uniqueKey[UniqueFilePathSize];
509     
510     ASSERT(key);
511
512     LOG(FileDatabaseActivity, "%@", key);
513
514     UniqueFilePathForKey(key, uniqueKey);
515     filePath = [[NSString alloc] initWithFormat:@"%@/%s", path, uniqueKey];
516     [[NSFileManager defaultManager] _webkit_removeFileOnlyAtPath:filePath];
517     WebLRUFileListRemoveFileWithPath(lru, uniqueKey);
518     [filePath release];
519 }
520
521 // database management functions ---------------------------------------------------------------------------
522 #pragma mark database management functions
523
524 -(void)open
525 {
526     NSFileManager *manager;
527     NSDictionary *attributes;
528     BOOL isDir;
529     
530     if (!isOpen) {
531         manager = [NSFileManager defaultManager];
532         if ([manager fileExistsAtPath:path isDirectory:&isDir]) {
533             if (isDir) {
534                 isOpen = YES;
535             }
536         }
537         else {
538             attributes = [NSDictionary dictionaryWithObjectsAndKeys:
539                 [NSDate date], @"NSFileModificationDate",
540                 NSUserName(), @"NSFileOwnerAccountName",
541                 WebFileDirectoryPOSIXPermissions, @"NSFilePosixPermissions",
542                 NULL
543             ];
544             
545             isOpen = [manager _webkit_createDirectoryAtPathWithIntermediateDirectories:path attributes:attributes];
546         }
547
548         // remove any leftover turds
549         [manager _webkit_backgroundRemoveLeftoverFiles:path];
550         
551         if (isOpen) {
552             [NSThread detachNewThreadSelector:@selector(_createLRUList:) toTarget:self withObject:nil];
553         }
554     }
555 }
556
557 -(void)close
558 {
559     if (isOpen) {
560         isOpen = NO;
561         if (lru) {
562             WebLRUFileListRelease(lru);
563             lru = NULL;
564         }
565     }
566 }
567
568 -(void)lazySync:(NSTimer *)theTimer
569 {
570     if (!lru) {
571         // wait for lru to finish getting created        
572         return;
573     }
574
575 #ifndef NDEBUG
576     CFTimeInterval mark = CFAbsoluteTimeGetCurrent();
577 #endif
578
579     LOG(FileDatabaseActivity, ">>> BEFORE lazySync\n%@", WebLRUFileListDescription(lru));
580
581     WebFileDatabaseOp *op;
582
583     ASSERT(theTimer);
584
585     while (touch + SYNC_IDLE_THRESHOLD < CFAbsoluteTimeGetCurrent() && [ops count] > 0) {
586         [mutex lock];
587
588         if (timer) {
589             [timer invalidate];
590             [timer autorelease];
591             timer = nil;
592         }
593         
594         op = [ops lastObject];
595         if (op) {
596             [op retain];
597             [ops removeLastObject];
598             [op perform:self];
599             [setCache removeObjectForKey:[op key]];
600             [removeCache removeObject:[op key]];
601             [op release];
602         }
603
604         [mutex unlock];
605     }
606
607     // come back later to finish the work...
608     if ([ops count] > 0) {
609         [mutex lock];
610         [self setTimer];
611         [mutex unlock];
612     }
613
614 #ifndef NDEBUG
615     if (lru)
616         LOG(FileDatabaseActivity, "<<< AFTER lazySync\n%@", WebLRUFileListDescription(lru));
617
618     CFTimeInterval now = CFAbsoluteTimeGetCurrent();
619     LOG(FileDatabaseActivity, "lazySync ran in %.3f secs.", now - mark);
620 #endif
621 }
622
623 -(void)sync
624 {
625     NSArray *array;
626
627     if (!lru) {
628         // wait for lru to finish getting created        
629         return;
630     }
631
632     touch = CFAbsoluteTimeGetCurrent();
633
634     LOG(FileDatabaseActivity, ">>> BEFORE sync\n%@", WebLRUFileListDescription(lru));
635     
636     [mutex lock];
637     array = [ops copy];
638     [ops removeAllObjects];
639     [timer invalidate];
640     [timer autorelease];
641     timer = nil;
642     [setCache removeAllObjects];
643     [removeCache removeAllObjects];
644     [mutex unlock];
645
646     [array makeObjectsPerformSelector:@selector(perform:) withObject:self];
647     [array release];
648
649     LOG(FileDatabaseActivity, "<<< AFTER sync\n%@", WebLRUFileListDescription(lru));
650 }
651
652 -(unsigned)count
653 {
654     if (lru)
655         return WebLRUFileListCountItems(lru);
656     
657     return 0;
658 }
659
660 -(unsigned)usage
661 {
662     if (lru)
663         return WebLRUFileListGetTotalSize(lru);
664     
665     return 0;
666 }
667
668 -(void)setSizeLimit:(unsigned)limit
669 {
670     sizeLimit = limit;
671     if (limit < [self usage]) {
672         [self _truncateToSizeLimit:limit];
673     }
674 }
675
676 @end