9288563838d9ac64ae12eebd78b221b3c26c30d9
[WebKit-https.git] / WebKit / Misc.subproj / WebFileDatabase.m
1 /*      
2     WebFileDatabase.m
3     Copyright (C) 2003 Apple Computer, Inc. All rights reserved.    
4 */
5
6 #import <WebKit/WebFileDatabase.h>
7 #import <WebKit/WebKitLogging.h>
8 #import <WebKit/WebLRUFileList.h>
9
10 #import <Foundation/NSFileManager_NSURLExtras.h>
11
12 #import <fcntl.h>
13 #import <fts.h>
14 #import <pthread.h>
15 #import <string.h>
16 #import <sys/stat.h>
17 #import <sys/types.h>
18 #import <sys/mman.h>
19
20 #if ERROR_DISABLED
21 #define BEGIN_EXCEPTION_HANDLER
22 #define END_EXCEPTION_HANDLER
23 #else
24 #define BEGIN_EXCEPTION_HANDLER NS_DURING
25 #define END_EXCEPTION_HANDLER \
26     NS_HANDLER \
27         ERROR("Uncaught exception: %@ [%@] [%@]", [localException class], [localException reason], [localException userInfo]); \
28     NS_ENDHANDLER
29 #endif
30
31 static pthread_once_t databaseInitControl = PTHREAD_ONCE_INIT;
32 static NSNumber *WebFileDirectoryPOSIXPermissions;
33 static NSNumber *WebFilePOSIXPermissions;
34 static NSRunLoop *syncRunLoop;
35
36 #define UniqueFilePathSize (34)
37 static void UniqueFilePathForKey(id key, char *buffer);
38
39 #define MinThreadPriority (10)
40 static int SetThreadPriority(int priority);
41
42 typedef enum
43 {
44     WebFileDatabaseSetObjectOp,
45     WebFileDatabaseRemoveObjectOp,
46 } WebFileDatabaseOpcode;
47
48 enum
49 {
50     MAX_UNSIGNED_LENGTH = 20, // long enough to hold the string representation of a 64-bit unsigned number
51     SYNC_IDLE_THRESHOLD = 10,
52 };
53
54 // interface WebFileDatabaseOp -------------------------------------------------------------
55
56 @interface WebFileDatabaseOp : NSObject
57 {
58     WebFileDatabaseOpcode opcode;
59     id key;
60     id object; 
61 }
62
63 +(id)opWithCode:(WebFileDatabaseOpcode)opcode key:(id)key object:(id)object;
64 -(id)initWithCode:(WebFileDatabaseOpcode)opcode key:(id)key object:(id)object;
65
66 -(WebFileDatabaseOpcode)opcode;
67 -(id)key;
68 -(id)object;
69 -(void)perform:(WebFileDatabase *)target;
70
71 @end
72
73
74 // implementation WebFileDatabaseOp -------------------------------------------------------------
75
76 @implementation WebFileDatabaseOp
77
78 +(id)opWithCode:(WebFileDatabaseOpcode)theOpcode key:(id)theKey object:(id)theObject
79 {
80     return [[[WebFileDatabaseOp alloc] initWithCode:theOpcode key:theKey object:theObject] autorelease];
81 }
82
83 -(id)initWithCode:(WebFileDatabaseOpcode)theOpcode key:(id)theKey object:(id)theObject
84 {
85     ASSERT(theKey);
86
87     if ((self = [super init])) {
88         
89         opcode = theOpcode;
90         key = [theKey retain];
91         object = [theObject retain];
92         
93         return self;
94     }
95   
96     return nil;
97 }
98
99 -(WebFileDatabaseOpcode)opcode
100 {
101     return opcode;
102 }
103
104 -(id)key
105 {
106     return key;
107 }
108
109 -(id)object
110 {
111     return object;
112 }
113
114 -(void)perform:(WebFileDatabase *)target
115 {
116     ASSERT(target);
117
118     switch (opcode) {
119         case WebFileDatabaseSetObjectOp:
120             [target performSetObject:object forKey:key];
121             break;
122         case WebFileDatabaseRemoveObjectOp:
123             [target performRemoveObjectForKey:key];
124             break;
125         default:
126             ASSERT_NOT_REACHED();
127             break;
128     }
129 }
130
131 -(void)dealloc
132 {
133     [key release];
134     [object release];
135     
136     [super dealloc];
137 }
138
139 @end
140
141
142 // interface WebFileDatabasePrivate -----------------------------------------------------------
143
144 @interface WebFileDatabase (WebFileDatabasePrivate)
145
146 -(void)_createLRUList:(id)arg;
147 -(void)_truncateToSizeLimit:(unsigned)size;
148
149 @end
150
151 // implementation WebFileDatabasePrivate ------------------------------------------------------
152
153 @implementation WebFileDatabase (WebFileDatabasePrivate)
154
155 static int SetThreadPriority(int priority) 
156 {
157     struct sched_param sp;
158
159     memset(&sp, 0, sizeof(struct sched_param));
160     sp.sched_priority=priority;
161     if (pthread_setschedparam(pthread_self(), SCHED_OTHER, &sp) == -1) {
162         ERROR("Failed to change priority.");
163         return -1;
164     }
165     return 0;
166 }
167
168 static void UniqueFilePathForKey(id key, char *buffer)
169 {
170     const char *s;
171     UInt32 hash1;
172     UInt32 hash2;
173     CFIndex len;
174     CFIndex cnt;
175
176     s = [[[[key description] lowercaseString] stringByStandardizingPath] lossyCString];
177     len = strlen(s);
178
179     // compute first hash    
180     hash1 = len;
181     for (cnt = 0; cnt < len; cnt++) {
182         hash1 += (hash1 << 8) + s[cnt];
183     }
184     hash1 += (hash1 << (len & 31));
185
186     // compute second hash    
187     hash2 = len;
188     for (cnt = 0; cnt < len; cnt++) {
189         hash2 = (37 * hash2) ^ s[cnt];
190     }
191
192     snprintf(buffer, UniqueFilePathSize, "%.2lu/%.2lu/%.10lu-%.10lu.cache", ((hash1 & 0xff) >> 4), ((hash2 & 0xff) >> 4), hash1, hash2);
193 }
194
195 -(void)_createLRUList:(id)arg
196 {
197     SetThreadPriority(MinThreadPriority + 1); // make this a little higher priority than the syncRunLoop thread
198
199     NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
200
201     BEGIN_EXCEPTION_HANDLER
202     
203     WebLRUFileList *fileList = WebLRUFileListCreate();
204     WebLRUFileListRebuildFileDataUsingRootDirectory(fileList, [path fileSystemRepresentation]);
205     lru = fileList;
206
207     END_EXCEPTION_HANDLER
208
209     LOG(FileDatabaseActivity, "lru list created");
210
211 #if BUILDING_ON_PANTHER
212     [pool release];
213 #else
214     [pool drain];
215 #endif
216 }
217
218 -(void)_truncateToSizeLimit:(unsigned)size
219 {
220     NSFileManager *defaultManager;
221     
222     if (!lru || size > [self usage]) {
223         return;
224     }
225
226     if (size == 0) {
227         [self removeAllObjects];
228     }
229     else {
230         defaultManager = [NSFileManager defaultManager];
231         [mutex lock];
232         while ([self usage] > size) {
233             char uniqueKey[UniqueFilePathSize];
234             if (!WebLRUFileListGetPathOfOldestFile(lru, uniqueKey, UniqueFilePathSize)) {
235                 break;
236             }
237             NSString *filePath = [[NSString alloc] initWithFormat:@"%@/%s", path, uniqueKey];
238             [defaultManager _web_removeFileOnlyAtPath:filePath];
239             [filePath release];
240             WebLRUFileListRemoveOldestFileFromList(lru);
241         }
242         [mutex unlock];
243     }
244 }
245
246 @end
247
248
249 // implementation WebFileDatabase -------------------------------------------------------------
250
251 @implementation WebFileDatabase
252
253 // creation functions ---------------------------------------------------------------------------
254 #pragma mark creation functions
255
256 +(void)_syncLoop:(id)arg
257 {
258     SetThreadPriority(MinThreadPriority);
259
260     NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
261     NSPort *placeholderPort;
262
263     BEGIN_EXCEPTION_HANDLER
264
265     syncRunLoop = [NSRunLoop currentRunLoop];
266
267     while (YES) {
268         BEGIN_EXCEPTION_HANDLER
269         // we specifically use an NSRunLoop here to get autorelease pool support
270         placeholderPort = [NSPort port];
271         [syncRunLoop addPort:placeholderPort forMode:NSDefaultRunLoopMode];
272         [syncRunLoop run];
273         [syncRunLoop removePort:placeholderPort forMode:NSDefaultRunLoopMode];
274         END_EXCEPTION_HANDLER
275     }
276
277     END_EXCEPTION_HANDLER
278
279 #if BUILDING_ON_PANTHER
280     [pool release];
281 #else
282     [pool drain];
283 #endif
284 }
285
286 static void databaseInit()
287 {
288     // set file perms to owner read/write/execute only
289     WebFileDirectoryPOSIXPermissions = [[NSNumber numberWithInt:(WEB_UREAD | WEB_UWRITE | WEB_UEXEC)] retain];
290
291     // set file perms to owner read/write only
292     WebFilePOSIXPermissions = [[NSNumber numberWithInt:(WEB_UREAD | WEB_UWRITE)] retain];
293
294     [NSThread detachNewThreadSelector:@selector(_syncLoop:) toTarget:[WebFileDatabase class] withObject:nil];
295 }
296
297 -(id)initWithPath:(NSString *)thePath
298 {
299     pthread_once(&databaseInitControl, databaseInit);
300
301     [super initWithPath:thePath];
302     
303     if (self == nil || thePath == nil) {
304         [self release];
305         return nil;
306     }
307
308     ops = [[NSMutableArray alloc] init];
309     setCache = [[NSMutableDictionary alloc] init];
310     removeCache = [[NSMutableSet alloc] init];
311     timer = nil;
312     mutex = [[NSRecursiveLock alloc] init];
313     
314     return self;
315 }
316
317 // WebFileDatabase objects are never released, so we need no dealloc implementation.
318
319 -(void)setTimer
320 {
321     if (timer == nil) {
322         NSDate *fireDate = [[NSDate alloc] initWithTimeIntervalSinceNow:SYNC_IDLE_THRESHOLD];
323         timer = [[NSTimer alloc] initWithFireDate:fireDate interval:SYNC_IDLE_THRESHOLD target:self selector:@selector(lazySync:) userInfo:nil repeats:YES];
324         [fireDate release];
325         [syncRunLoop addTimer:timer forMode:NSDefaultRunLoopMode];
326     }
327 }
328
329 // database functions ---------------------------------------------------------------------------
330 #pragma mark database functions
331
332 -(void)setObject:(id)object forKey:(id)key
333 {
334     WebFileDatabaseOp *op;
335
336     ASSERT(object);
337     ASSERT(key);
338
339     touch = CFAbsoluteTimeGetCurrent();
340
341     LOG(FileDatabaseActivity, "%p - %@", object, key);
342     
343     [mutex lock];
344     
345     [setCache setObject:object forKey:key];
346     op = [[WebFileDatabaseOp alloc] initWithCode:WebFileDatabaseSetObjectOp key:key object:object];
347     [ops addObject:op];
348     [op release];
349     [self setTimer];
350     
351     [mutex unlock];
352 }
353
354 -(void)removeObjectForKey:(id)key
355 {
356     WebFileDatabaseOp *op;
357
358     ASSERT(key);
359
360     touch = CFAbsoluteTimeGetCurrent();
361     
362     [mutex lock];
363     
364     [removeCache addObject:key];
365     op = [[WebFileDatabaseOp alloc] initWithCode:WebFileDatabaseRemoveObjectOp key:key object:nil];
366     [ops addObject:op];
367     [op release];
368     [self setTimer];
369     
370     [mutex unlock];
371 }
372
373 -(void)removeAllObjects
374 {
375     touch = CFAbsoluteTimeGetCurrent();
376
377     [mutex lock];
378     [setCache removeAllObjects];
379     [removeCache removeAllObjects];
380     [ops removeAllObjects];
381     [self close];
382     [[NSFileManager defaultManager] _web_backgroundRemoveFileAtPath:path];
383     [self open];
384     [mutex unlock];
385
386     LOG(FileDatabaseActivity, "removeAllObjects");
387 }
388
389 -(id)objectForKey:(id)key
390 {
391     volatile id result;
392     
393     ASSERT(key);
394
395     touch = CFAbsoluteTimeGetCurrent();
396
397     // check caches
398     [mutex lock];
399     if ([removeCache containsObject:key]) {
400         [mutex unlock];
401         return nil;
402     }
403     if ((result = [setCache objectForKey:key])) {
404         [mutex unlock];
405         return result;
406     }
407     [mutex unlock];
408
409     // go to disk
410     char uniqueKey[UniqueFilePathSize];
411     UniqueFilePathForKey(key, uniqueKey);
412     NSString *filePath = [[NSString alloc] initWithFormat:@"%@/%s", path, uniqueKey];
413     NSData *data = [[NSData alloc] initWithContentsOfFile:filePath];
414     NSUnarchiver * volatile unarchiver = nil;
415
416     NS_DURING
417         if (data) {
418             unarchiver = [[NSUnarchiver alloc] initForReadingWithData:data];
419             if (unarchiver) {
420                 id fileKey = [unarchiver decodeObject];
421                 if ([fileKey isEqual:key]) {
422                     id object = [unarchiver decodeObject];
423                     if (object) {
424                         // Decoded objects go away when the unarchiver does, so we need to
425                         // retain this so we can return it to our caller.
426                         result = [[object retain] autorelease];
427                         if (lru) {
428                             // if we can't update the list yet, that's too bad
429                             // but not critically bad
430                             WebLRUFileListTouchFileWithPath(lru, uniqueKey);
431                         }
432                         LOG(FileDatabaseActivity, "read disk cache file - %@", key);
433                     }
434                 }
435             }
436         }
437     NS_HANDLER
438         LOG(FileDatabaseActivity, "cannot unarchive cache file - %@", key);
439         result = nil;
440     NS_ENDHANDLER
441
442     [unarchiver release];
443     [data release];
444     [filePath release];
445
446     LOG(Timing, "getting value for %@ took %f", key, (CFAbsoluteTimeGetCurrent() - touch));
447     
448     return result;
449 }
450
451 -(void)performSetObject:(id)object forKey:(id)key
452 {
453     NSString *filePath;
454     NSMutableData *data;
455     NSDictionary *attributes;
456     NSDictionary *directoryAttributes;
457     NSArchiver *archiver;
458     NSFileManager *defaultManager;
459     char uniqueKey[UniqueFilePathSize];
460     BOOL result;
461
462     ASSERT(object);
463     ASSERT(key);
464
465     UniqueFilePathForKey(key, uniqueKey);
466
467     LOG(FileDatabaseActivity, "%@ - %s", key, uniqueKey);
468
469     data = [NSMutableData data];
470     archiver = [[NSArchiver alloc] initForWritingWithMutableData:data];
471     [archiver encodeObject:key];
472     [archiver encodeObject:object];
473     
474     attributes = [NSDictionary dictionaryWithObjectsAndKeys:
475         NSUserName(), NSFileOwnerAccountName,
476         WebFilePOSIXPermissions, NSFilePosixPermissions,
477         NULL
478     ];
479
480     directoryAttributes = [NSDictionary dictionaryWithObjectsAndKeys:
481         NSUserName(), NSFileOwnerAccountName,
482         WebFileDirectoryPOSIXPermissions, NSFilePosixPermissions,
483         NULL
484     ];
485
486     defaultManager = [NSFileManager defaultManager];
487
488     filePath = [[NSString alloc] initWithFormat:@"%@/%s", path, uniqueKey];
489     attributes = [defaultManager fileAttributesAtPath:filePath traverseLink:YES];
490
491     // update usage and truncate before writing file
492     // this has the effect of _always_ keeping disk usage under sizeLimit by clearing away space in anticipation of the write.
493     WebLRUFileListSetFileData(lru, uniqueKey, [data length], CFAbsoluteTimeGetCurrent());
494     [self _truncateToSizeLimit:[self sizeLimit]];
495
496     result = [defaultManager _web_createFileAtPathWithIntermediateDirectories:filePath contents:data attributes:attributes directoryAttributes:directoryAttributes];
497
498     if (!result) {
499         WebLRUFileListRemoveFileWithPath(lru, uniqueKey);
500     }
501
502     [archiver release];
503     [filePath release];    
504 }
505
506 -(void)performRemoveObjectForKey:(id)key
507 {
508     NSString *filePath;
509     char uniqueKey[UniqueFilePathSize];
510     
511     ASSERT(key);
512
513     LOG(FileDatabaseActivity, "%@", key);
514
515     UniqueFilePathForKey(key, uniqueKey);
516     filePath = [[NSString alloc] initWithFormat:@"%@/%s", path, uniqueKey];
517     [[NSFileManager defaultManager] _web_removeFileOnlyAtPath:filePath];
518     WebLRUFileListRemoveFileWithPath(lru, uniqueKey);
519     [filePath release];
520 }
521
522 // database management functions ---------------------------------------------------------------------------
523 #pragma mark database management functions
524
525 -(void)open
526 {
527     NSFileManager *manager;
528     NSDictionary *attributes;
529     BOOL isDir;
530     
531     if (!isOpen) {
532         manager = [NSFileManager defaultManager];
533         if ([manager fileExistsAtPath:path isDirectory:&isDir]) {
534             if (isDir) {
535                 isOpen = YES;
536             }
537         }
538         else {
539             attributes = [NSDictionary dictionaryWithObjectsAndKeys:
540                 [NSDate date], @"NSFileModificationDate",
541                 NSUserName(), @"NSFileOwnerAccountName",
542                 WebFileDirectoryPOSIXPermissions, @"NSFilePosixPermissions",
543                 NULL
544             ];
545             
546             isOpen = [manager _web_createDirectoryAtPathWithIntermediateDirectories:path attributes:attributes];
547         }
548
549         // remove any leftover turds
550         [manager _web_backgroundRemoveLeftoverFiles:path];
551         
552         if (isOpen) {
553             [NSThread detachNewThreadSelector:@selector(_createLRUList:) toTarget:self withObject:nil];
554         }
555     }
556 }
557
558 -(void)close
559 {
560     if (isOpen) {
561         isOpen = NO;
562         if (lru) {
563             WebLRUFileListRelease(lru);
564             lru = NULL;
565         }
566     }
567 }
568
569 -(void)lazySync:(NSTimer *)theTimer
570 {
571     if (!lru) {
572         // wait for lru to finish getting created        
573         return;
574     }
575
576 #ifndef NDEBUG
577     CFTimeInterval mark = CFAbsoluteTimeGetCurrent();
578 #endif
579
580     LOG(FileDatabaseActivity, ">>> BEFORE lazySync\n%@", WebLRUFileListDescription(lru));
581
582     WebFileDatabaseOp *op;
583
584     ASSERT(theTimer);
585
586     while (touch + SYNC_IDLE_THRESHOLD < CFAbsoluteTimeGetCurrent() && [ops count] > 0) {
587         [mutex lock];
588
589         if (timer) {
590             [timer invalidate];
591             [timer autorelease];
592             timer = nil;
593         }
594         
595         op = [ops lastObject];
596         if (op) {
597             [op retain];
598             [ops removeLastObject];
599             [op perform:self];
600             [setCache removeObjectForKey:[op key]];
601             [removeCache removeObject:[op key]];
602             [op release];
603         }
604
605         [mutex unlock];
606     }
607
608     // come back later to finish the work...
609     if ([ops count] > 0) {
610         [mutex lock];
611         [self setTimer];
612         [mutex unlock];
613     }
614
615 #ifndef NDEBUG
616     if (lru)
617         LOG(FileDatabaseActivity, "<<< AFTER lazySync\n%@", WebLRUFileListDescription(lru));
618
619     CFTimeInterval now = CFAbsoluteTimeGetCurrent();
620     LOG(FileDatabaseActivity, "lazySync ran in %.3f secs.", now - mark);
621 #endif
622 }
623
624 -(void)sync
625 {
626     NSArray *array;
627
628     if (!lru) {
629         // wait for lru to finish getting created        
630         return;
631     }
632
633     touch = CFAbsoluteTimeGetCurrent();
634
635     LOG(FileDatabaseActivity, ">>> BEFORE sync\n%@", WebLRUFileListDescription(lru));
636     
637     [mutex lock];
638     array = [ops copy];
639     [ops removeAllObjects];
640     [timer invalidate];
641     [timer autorelease];
642     timer = nil;
643     [setCache removeAllObjects];
644     [removeCache removeAllObjects];
645     [mutex unlock];
646
647     [array makeObjectsPerformSelector:@selector(perform:) withObject:self];
648     [array release];
649
650     LOG(FileDatabaseActivity, "<<< AFTER sync\n%@", WebLRUFileListDescription(lru));
651 }
652
653 -(unsigned)count
654 {
655     if (lru)
656         return WebLRUFileListCountItems(lru);
657     
658     return 0;
659 }
660
661 -(unsigned)usage
662 {
663     if (lru)
664         return WebLRUFileListGetTotalSize(lru);
665     
666     return 0;
667 }
668
669 -(void)setSizeLimit:(unsigned)limit
670 {
671     sizeLimit = limit;
672     if (limit < [self usage]) {
673         [self _truncateToSizeLimit:limit];
674     }
675 }
676
677 @end