WebKit/Misc.subproj/WebFileDatabase.m

   1 /*
   2     WebFileDatabase.m
   3     Copyright (C) 2003 Apple Computer, Inc. All rights reserved.
   4 */
   5
   6 #import <WebKit/WebFileDatabase.h>
   7 #import <WebKit/WebKitLogging.h>
   8 #import <WebKit/WebLRUFileList.h>
   9 #import <WebKit/WebNSFileManagerExtras.h>
  10
  11 #import <fcntl.h>
  12 #import <fts.h>
  13 #import <pthread.h>
  14 #import <string.h>
  15 #import <sys/stat.h>
  16 #import <sys/types.h>
  17 #import <sys/mman.h>
  18
  19 #if ERROR_DISABLED
  20 #define BEGIN_EXCEPTION_HANDLER
  21 #define END_EXCEPTION_HANDLER
  22 #else
  23 #define BEGIN_EXCEPTION_HANDLER NS_DURING
  24 #define END_EXCEPTION_HANDLER \
  25     NS_HANDLER \
  26         ERROR("Uncaught exception: %@ [%@] [%@]", [localException class], [localException reason], [localException userInfo]); \
  27     NS_ENDHANDLER
  28 #endif
  29
  30 static pthread_once_t databaseInitControl = PTHREAD_ONCE_INIT;
  31 static NSNumber *WebFileDirectoryPOSIXPermissions;
  32 static NSNumber *WebFilePOSIXPermissions;
  33 static NSRunLoop *syncRunLoop;
  34
  35 #define UniqueFilePathSize (34)
  36 static void UniqueFilePathForKey(id key, char *buffer);
  37
  38 #define MinThreadPriority (10)
  39 static int SetThreadPriority(int priority);
  40
  41 typedef enum
  42 {
  43     WebFileDatabaseSetObjectOp,
  44     WebFileDatabaseRemoveObjectOp,
  45 } WebFileDatabaseOpcode;
  46
  47 enum
  48 {
  49     MAX_UNSIGNED_LENGTH = 20, // long enough to hold the string representation of a 64-bit unsigned number
  50     SYNC_IDLE_THRESHOLD = 10,
  51 };
  52
  53 // interface WebFileDatabaseOp -------------------------------------------------------------
  54
  55 @interface WebFileDatabaseOp : NSObject
  56 {
  57     WebFileDatabaseOpcode opcode;
  58     id key;
  59     id object;
  60 }
  61
  62 +(id)opWithCode:(WebFileDatabaseOpcode)opcode key:(id)key object:(id)object;
  63 -(id)initWithCode:(WebFileDatabaseOpcode)opcode key:(id)key object:(id)object;
  64
  65 -(WebFileDatabaseOpcode)opcode;
  66 -(id)key;
  67 -(id)object;
  68 -(void)perform:(WebFileDatabase *)target;
  69
  70 @end
  71
  72
  73 // implementation WebFileDatabaseOp -------------------------------------------------------------
  74
  75 @implementation WebFileDatabaseOp
  76
  77 +(id)opWithCode:(WebFileDatabaseOpcode)theOpcode key:(id)theKey object:(id)theObject
  78 {
  79     return [[[WebFileDatabaseOp alloc] initWithCode:theOpcode key:theKey object:theObject] autorelease];
  80 }
  81
  82 -(id)initWithCode:(WebFileDatabaseOpcode)theOpcode key:(id)theKey object:(id)theObject
  83 {
  84     ASSERT(theKey);
  85
  86     if ((self = [super init])) {
  87
  88         opcode = theOpcode;
  89         key = [theKey retain];
  90         object = [theObject retain];
  91
  92         return self;
  93     }
  94
  95     return nil;
  96 }
  97
  98 -(WebFileDatabaseOpcode)opcode
  99 {
 100     return opcode;
 101 }
 102
 103 -(id)key
 104 {
 105     return key;
 106 }
 107
 108 -(id)object
 109 {
 110     return object;
 111 }
 112
 113 -(void)perform:(WebFileDatabase *)target
 114 {
 115     ASSERT(target);
 116
 117     switch (opcode) {
 118         case WebFileDatabaseSetObjectOp:
 119             [target performSetObject:object forKey:key];
 120             break;
 121         case WebFileDatabaseRemoveObjectOp:
 122             [target performRemoveObjectForKey:key];
 123             break;
 124         default:
 125             ASSERT_NOT_REACHED();
 126             break;
 127     }
 128 }
 129
 130 -(void)dealloc
 131 {
 132     [key release];
 133     [object release];
 134
 135     [super dealloc];
 136 }
 137
 138 @end
 139
 140
 141 // interface WebFileDatabasePrivate -----------------------------------------------------------
 142
 143 @interface WebFileDatabase (WebFileDatabasePrivate)
 144
 145 -(void)_createLRUList:(id)arg;
 146 -(void)_truncateToSizeLimit:(unsigned)size;
 147
 148 @end
 149
 150 // implementation WebFileDatabasePrivate ------------------------------------------------------
 151
 152 @implementation WebFileDatabase (WebFileDatabasePrivate)
 153
 154 static int SetThreadPriority(int priority)
 155 {
 156     struct sched_param sp;
 157
 158     memset(&sp, 0, sizeof(struct sched_param));
 159     sp.sched_priority=priority;
 160     if (pthread_setschedparam(pthread_self(), SCHED_OTHER, &sp) == -1) {
 161         ERROR("Failed to change priority.");
 162         return -1;
 163     }
 164     return 0;
 165 }
 166
 167 static void UniqueFilePathForKey(id key, char *buffer)
 168 {
 169     const char *s;
 170     UInt32 hash1;
 171     UInt32 hash2;
 172     CFIndex len;
 173     CFIndex cnt;
 174
 175     s = [[[[key description] lowercaseString] stringByStandardizingPath] lossyCString];
 176     len = strlen(s);
 177
 178     // compute first hash
 179     hash1 = len;
 180     for (cnt = 0; cnt < len; cnt++) {
 181         hash1 += (hash1 << 8) + s[cnt];
 182     }
 183     hash1 += (hash1 << (len & 31));
 184
 185     // compute second hash
 186     hash2 = len;
 187     for (cnt = 0; cnt < len; cnt++) {
 188         hash2 = (37 * hash2) ^ s[cnt];
 189     }
 190
 191     snprintf(buffer, UniqueFilePathSize, "%.2lu/%.2lu/%.10lu-%.10lu.cache", ((hash1 & 0xff) >> 4), ((hash2 & 0xff) >> 4), hash1, hash2);
 192 }
 193
 194 -(void)_createLRUList:(id)arg
 195 {
 196     SetThreadPriority(MinThreadPriority + 1); // make this a little higher priority than the syncRunLoop thread
 197
 198     NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
 199
 200     BEGIN_EXCEPTION_HANDLER
 201
 202     WebLRUFileList *fileList = WebLRUFileListCreate();
 203     WebLRUFileListRebuildFileDataUsingRootDirectory(fileList, [path fileSystemRepresentation]);
 204     lru = fileList;
 205
 206     END_EXCEPTION_HANDLER
 207
 208     LOG(FileDatabaseActivity, "lru list created");
 209
 210 #if BUILDING_ON_PANTHER
 211     [pool release];
 212 #else
 213     [pool drain];
 214 #endif
 215 }
 216
 217 -(void)_truncateToSizeLimit:(unsigned)size
 218 {
 219     NSFileManager *defaultManager;
 220
 221     if (!lru || size > [self usage]) {
 222         return;
 223     }
 224
 225     if (size == 0) {
 226         [self removeAllObjects];
 227     }
 228     else {
 229         defaultManager = [NSFileManager defaultManager];
 230         [mutex lock];
 231         while ([self usage] > size) {
 232             char uniqueKey[UniqueFilePathSize];
 233             if (!WebLRUFileListGetPathOfOldestFile(lru, uniqueKey, UniqueFilePathSize)) {
 234                 break;
 235             }
 236             NSString *filePath = [[NSString alloc] initWithFormat:@"%@/%s", path, uniqueKey];
 237             [defaultManager _webkit_removeFileOnlyAtPath:filePath];
 238             [filePath release];
 239             WebLRUFileListRemoveOldestFileFromList(lru);
 240         }
 241         [mutex unlock];
 242     }
 243 }
 244
 245 @end
 246
 247
 248 // implementation WebFileDatabase -------------------------------------------------------------
 249
 250 @implementation WebFileDatabase
 251
 252 // creation functions ---------------------------------------------------------------------------
 253 #pragma mark creation functions
 254
 255 +(void)_syncLoop:(id)arg
 256 {
 257     SetThreadPriority(MinThreadPriority);
 258
 259     NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
 260     NSPort *placeholderPort;
 261
 262     BEGIN_EXCEPTION_HANDLER
 263
 264     syncRunLoop = [NSRunLoop currentRunLoop];
 265
 266     while (YES) {
 267         BEGIN_EXCEPTION_HANDLER
 268         // we specifically use an NSRunLoop here to get autorelease pool support
 269         placeholderPort = [NSPort port];
 270         [syncRunLoop addPort:placeholderPort forMode:NSDefaultRunLoopMode];
 271         [syncRunLoop run];
 272         [syncRunLoop removePort:placeholderPort forMode:NSDefaultRunLoopMode];
 273         END_EXCEPTION_HANDLER
 274     }
 275
 276     END_EXCEPTION_HANDLER
 277
 278 #if BUILDING_ON_PANTHER
 279     [pool release];
 280 #else
 281     [pool drain];
 282 #endif
 283 }
 284
 285 static void databaseInit()
 286 {
 287     // set file perms to owner read/write/execute only
 288     WebFileDirectoryPOSIXPermissions = [[NSNumber numberWithInt:(WEB_UREAD | WEB_UWRITE | WEB_UEXEC)] retain];
 289
 290     // set file perms to owner read/write only
 291     WebFilePOSIXPermissions = [[NSNumber numberWithInt:(WEB_UREAD | WEB_UWRITE)] retain];
 292
 293     [NSThread detachNewThreadSelector:@selector(_syncLoop:) toTarget:[WebFileDatabase class] withObject:nil];
 294 }
 295
 296 -(id)initWithPath:(NSString *)thePath
 297 {
 298     pthread_once(&databaseInitControl, databaseInit);
 299
 300     [super initWithPath:thePath];
 301
 302     if (self == nil || thePath == nil) {
 303         [self release];
 304         return nil;
 305     }
 306
 307     ops = [[NSMutableArray alloc] init];
 308     setCache = [[NSMutableDictionary alloc] init];
 309     removeCache = [[NSMutableSet alloc] init];
 310     timer = nil;
 311     mutex = [[NSRecursiveLock alloc] init];
 312
 313     return self;
 314 }
 315
 316 // WebFileDatabase objects are never released, so we need no dealloc implementation.
 317
 318 -(void)setTimer
 319 {
 320     if (timer == nil) {
 321         NSDate *fireDate = [[NSDate alloc] initWithTimeIntervalSinceNow:SYNC_IDLE_THRESHOLD];
 322         timer = [[NSTimer alloc] initWithFireDate:fireDate interval:SYNC_IDLE_THRESHOLD target:self selector:@selector(lazySync:) userInfo:nil repeats:YES];
 323         [fireDate release];
 324         [syncRunLoop addTimer:timer forMode:NSDefaultRunLoopMode];
 325     }
 326 }
 327
 328 // database functions ---------------------------------------------------------------------------
 329 #pragma mark database functions
 330
 331 -(void)setObject:(id)object forKey:(id)key
 332 {
 333     WebFileDatabaseOp *op;
 334
 335     ASSERT(object);
 336     ASSERT(key);
 337
 338     touch = CFAbsoluteTimeGetCurrent();
 339
 340     LOG(FileDatabaseActivity, "%p - %@", object, key);
 341
 342     [mutex lock];
 343
 344     [setCache setObject:object forKey:key];
 345     op = [[WebFileDatabaseOp alloc] initWithCode:WebFileDatabaseSetObjectOp key:key object:object];
 346     [ops addObject:op];
 347     [op release];
 348     [self setTimer];
 349
 350     [mutex unlock];
 351 }
 352
 353 -(void)removeObjectForKey:(id)key
 354 {
 355     WebFileDatabaseOp *op;
 356
 357     ASSERT(key);
 358
 359     touch = CFAbsoluteTimeGetCurrent();
 360
 361     [mutex lock];
 362
 363     [removeCache addObject:key];
 364     op = [[WebFileDatabaseOp alloc] initWithCode:WebFileDatabaseRemoveObjectOp key:key object:nil];
 365     [ops addObject:op];
 366     [op release];
 367     [self setTimer];
 368
 369     [mutex unlock];
 370 }
 371
 372 -(void)removeAllObjects
 373 {
 374     touch = CFAbsoluteTimeGetCurrent();
 375
 376     [mutex lock];
 377     [setCache removeAllObjects];
 378     [removeCache removeAllObjects];
 379     [ops removeAllObjects];
 380     [self close];
 381     [[NSFileManager defaultManager] _webkit_backgroundRemoveFileAtPath:path];
 382     [self open];
 383     [mutex unlock];
 384
 385     LOG(FileDatabaseActivity, "removeAllObjects");
 386 }
 387
 388 -(id)objectForKey:(id)key
 389 {
 390     volatile id result;
 391
 392     ASSERT(key);
 393
 394     touch = CFAbsoluteTimeGetCurrent();
 395
 396     // check caches
 397     [mutex lock];
 398     if ([removeCache containsObject:key]) {
 399         [mutex unlock];
 400         return nil;
 401     }
 402     if ((result = [setCache objectForKey:key])) {
 403         [mutex unlock];
 404         return result;
 405     }
 406     [mutex unlock];
 407
 408     // go to disk
 409     char uniqueKey[UniqueFilePathSize];
 410     UniqueFilePathForKey(key, uniqueKey);
 411     NSString *filePath = [[NSString alloc] initWithFormat:@"%@/%s", path, uniqueKey];
 412     NSData *data = [[NSData alloc] initWithContentsOfFile:filePath];
 413     NSUnarchiver * volatile unarchiver = nil;
 414
 415     NS_DURING
 416         if (data) {
 417             unarchiver = [[NSUnarchiver alloc] initForReadingWithData:data];
 418             if (unarchiver) {
 419                 id fileKey = [unarchiver decodeObject];
 420                 if ([fileKey isEqual:key]) {
 421                     id object = [unarchiver decodeObject];
 422                     if (object) {
 423                         // Decoded objects go away when the unarchiver does, so we need to
 424                         // retain this so we can return it to our caller.
 425                         result = [[object retain] autorelease];
 426                         if (lru) {
 427                             // if we can't update the list yet, that's too bad
 428                             // but not critically bad
 429                             WebLRUFileListTouchFileWithPath(lru, uniqueKey);
 430                         }
 431                         LOG(FileDatabaseActivity, "read disk cache file - %@", key);
 432                     }
 433                 }
 434             }
 435         }
 436     NS_HANDLER
 437         LOG(FileDatabaseActivity, "cannot unarchive cache file - %@", key);
 438         result = nil;
 439     NS_ENDHANDLER
 440
 441     [unarchiver release];
 442     [data release];
 443     [filePath release];
 444
 445     LOG(Timing, "getting value for %@ took %f", key, (CFAbsoluteTimeGetCurrent() - touch));
 446
 447     return result;
 448 }
 449
 450 -(void)performSetObject:(id)object forKey:(id)key
 451 {
 452     NSString *filePath;
 453     NSMutableData *data;
 454     NSDictionary *attributes;
 455     NSDictionary *directoryAttributes;
 456     NSArchiver *archiver;
 457     NSFileManager *defaultManager;
 458     char uniqueKey[UniqueFilePathSize];
 459     BOOL result;
 460
 461     ASSERT(object);
 462     ASSERT(key);
 463
 464     UniqueFilePathForKey(key, uniqueKey);
 465
 466     LOG(FileDatabaseActivity, "%@ - %s", key, uniqueKey);
 467
 468     data = [NSMutableData data];
 469     archiver = [[NSArchiver alloc] initForWritingWithMutableData:data];
 470     [archiver encodeObject:key];
 471     [archiver encodeObject:object];
 472
 473     attributes = [NSDictionary dictionaryWithObjectsAndKeys:
 474         NSUserName(), NSFileOwnerAccountName,
 475         WebFilePOSIXPermissions, NSFilePosixPermissions,
 476         NULL
 477     ];
 478
 479     directoryAttributes = [NSDictionary dictionaryWithObjectsAndKeys:
 480         NSUserName(), NSFileOwnerAccountName,
 481         WebFileDirectoryPOSIXPermissions, NSFilePosixPermissions,
 482         NULL
 483     ];
 484
 485     defaultManager = [NSFileManager defaultManager];
 486
 487     filePath = [[NSString alloc] initWithFormat:@"%@/%s", path, uniqueKey];
 488     attributes = [defaultManager fileAttributesAtPath:filePath traverseLink:YES];
 489
 490     // update usage and truncate before writing file
 491     // this has the effect of _always_ keeping disk usage under sizeLimit by clearing away space in anticipation of the write.
 492     WebLRUFileListSetFileData(lru, uniqueKey, [data length], CFAbsoluteTimeGetCurrent());
 493     [self _truncateToSizeLimit:[self sizeLimit]];
 494
 495     result = [defaultManager _webkit_createFileAtPathWithIntermediateDirectories:filePath contents:data attributes:attributes directoryAttributes:directoryAttributes];
 496
 497     if (!result) {
 498         WebLRUFileListRemoveFileWithPath(lru, uniqueKey);
 499     }
 500
 501     [archiver release];
 502     [filePath release];
 503 }
 504
 505 -(void)performRemoveObjectForKey:(id)key
 506 {
 507     NSString *filePath;
 508     char uniqueKey[UniqueFilePathSize];
 509
 510     ASSERT(key);
 511
 512     LOG(FileDatabaseActivity, "%@", key);
 513
 514     UniqueFilePathForKey(key, uniqueKey);
 515     filePath = [[NSString alloc] initWithFormat:@"%@/%s", path, uniqueKey];
 516     [[NSFileManager defaultManager] _webkit_removeFileOnlyAtPath:filePath];
 517     WebLRUFileListRemoveFileWithPath(lru, uniqueKey);
 518     [filePath release];
 519 }
 520
 521 // database management functions ---------------------------------------------------------------------------
 522 #pragma mark database management functions
 523
 524 -(void)open
 525 {
 526     NSFileManager *manager;
 527     NSDictionary *attributes;
 528     BOOL isDir;
 529
 530     if (!isOpen) {
 531         manager = [NSFileManager defaultManager];
 532         if ([manager fileExistsAtPath:path isDirectory:&isDir]) {
 533             if (isDir) {
 534                 isOpen = YES;
 535             }
 536         }
 537         else {
 538             attributes = [NSDictionary dictionaryWithObjectsAndKeys:
 539                 [NSDate date], @"NSFileModificationDate",
 540                 NSUserName(), @"NSFileOwnerAccountName",
 541                 WebFileDirectoryPOSIXPermissions, @"NSFilePosixPermissions",
 542                 NULL
 543             ];
 544
 545             isOpen = [manager _webkit_createDirectoryAtPathWithIntermediateDirectories:path attributes:attributes];
 546         }
 547
 548         // remove any leftover turds
 549         [manager _webkit_backgroundRemoveLeftoverFiles:path];
 550
 551         if (isOpen) {
 552             [NSThread detachNewThreadSelector:@selector(_createLRUList:) toTarget:self withObject:nil];
 553         }
 554     }
 555 }
 556
 557 -(void)close
 558 {
 559     if (isOpen) {
 560         isOpen = NO;
 561         if (lru) {
 562             WebLRUFileListRelease(lru);
 563             lru = NULL;
 564         }
 565     }
 566 }
 567
 568 -(void)lazySync:(NSTimer *)theTimer
 569 {
 570     if (!lru) {
 571         // wait for lru to finish getting created
 572         return;
 573     }
 574
 575 #ifndef NDEBUG
 576     CFTimeInterval mark = CFAbsoluteTimeGetCurrent();
 577 #endif
 578
 579     LOG(FileDatabaseActivity, ">>> BEFORE lazySync\n%@", WebLRUFileListDescription(lru));
 580
 581     WebFileDatabaseOp *op;
 582
 583     ASSERT(theTimer);
 584
 585     while (touch + SYNC_IDLE_THRESHOLD < CFAbsoluteTimeGetCurrent() && [ops count] > 0) {
 586         [mutex lock];
 587
 588         if (timer) {
 589             [timer invalidate];
 590             [timer autorelease];
 591             timer = nil;
 592         }
 593
 594         op = [ops lastObject];
 595         if (op) {
 596             [op retain];
 597             [ops removeLastObject];
 598             [op perform:self];
 599             [setCache removeObjectForKey:[op key]];
 600             [removeCache removeObject:[op key]];
 601             [op release];
 602         }
 603
 604         [mutex unlock];
 605     }
 606
 607     // come back later to finish the work...
 608     if ([ops count] > 0) {
 609         [mutex lock];
 610         [self setTimer];
 611         [mutex unlock];
 612     }
 613
 614 #ifndef NDEBUG
 615     if (lru)
 616         LOG(FileDatabaseActivity, "<<< AFTER lazySync\n%@", WebLRUFileListDescription(lru));
 617
 618     CFTimeInterval now = CFAbsoluteTimeGetCurrent();
 619     LOG(FileDatabaseActivity, "lazySync ran in %.3f secs.", now - mark);
 620 #endif
 621 }
 622
 623 -(void)sync
 624 {
 625     NSArray *array;
 626
 627     if (!lru) {
 628         // wait for lru to finish getting created
 629         return;
 630     }
 631
 632     touch = CFAbsoluteTimeGetCurrent();
 633
 634     LOG(FileDatabaseActivity, ">>> BEFORE sync\n%@", WebLRUFileListDescription(lru));
 635
 636     [mutex lock];
 637     array = [ops copy];
 638     [ops removeAllObjects];
 639     [timer invalidate];
 640     [timer autorelease];
 641     timer = nil;
 642     [setCache removeAllObjects];
 643     [removeCache removeAllObjects];
 644     [mutex unlock];
 645
 646     [array makeObjectsPerformSelector:@selector(perform:) withObject:self];
 647     [array release];
 648
 649     LOG(FileDatabaseActivity, "<<< AFTER sync\n%@", WebLRUFileListDescription(lru));
 650 }
 651
 652 -(unsigned)count
 653 {
 654     if (lru)
 655         return WebLRUFileListCountItems(lru);
 656
 657     return 0;
 658 }
 659
 660 -(unsigned)usage
 661 {
 662     if (lru)
 663         return WebLRUFileListGetTotalSize(lru);
 664
 665     return 0;
 666 }
 667
 668 -(void)setSizeLimit:(unsigned)limit
 669 {
 670     sizeLimit = limit;
 671     if (limit < [self usage]) {
 672         [self _truncateToSizeLimit:limit];
 673     }
 674 }
 675
 676 @end