Unreviewed, roll out http://trac.webkit.org/changeset/187972.
[WebKit-https.git] / Source / JavaScriptCore / heap / CopiedSpace.cpp
index 3310d2c..9592eaa 100644 (file)
 #include "config.h"
 #include "CopiedSpace.h"
 
-#include "CopiedSpaceInlineMethods.h"
+#include "CopiedSpaceInlines.h"
+#include "GCActivityCallback.h"
+#include "JSCInlines.h"
+#include "Options.h"
 
 namespace JSC {
 
 CopiedSpace::CopiedSpace(Heap* heap)
     : m_heap(heap)
-    , m_toSpace(0)
-    , m_fromSpace(0)
-    , m_totalMemoryAllocated(0)
-    , m_totalMemoryUtilized(0)
     , m_inCopyingPhase(false)
+    , m_shouldDoCopyPhase(false)
     , m_numberOfLoanedBlocks(0)
+    , m_bytesRemovedFromOldSpaceDueToReallocation(0)
 {
 }
 
+CopiedSpace::~CopiedSpace()
+{
+    while (!m_oldGen.toSpace->isEmpty())
+        CopiedBlock::destroy(m_oldGen.toSpace->removeHead());
+
+    while (!m_oldGen.fromSpace->isEmpty())
+        CopiedBlock::destroy(m_oldGen.fromSpace->removeHead());
+
+    while (!m_oldGen.oversizeBlocks.isEmpty())
+        CopiedBlock::destroy(m_oldGen.oversizeBlocks.removeHead());
+
+    while (!m_newGen.toSpace->isEmpty())
+        CopiedBlock::destroy(m_newGen.toSpace->removeHead());
+
+    while (!m_newGen.fromSpace->isEmpty())
+        CopiedBlock::destroy(m_newGen.fromSpace->removeHead());
+
+    while (!m_newGen.oversizeBlocks.isEmpty())
+        CopiedBlock::destroy(m_newGen.oversizeBlocks.removeHead());
+
+    ASSERT(m_oldGen.toSpace->isEmpty());
+    ASSERT(m_oldGen.fromSpace->isEmpty());
+    ASSERT(m_oldGen.oversizeBlocks.isEmpty());
+    ASSERT(m_newGen.toSpace->isEmpty());
+    ASSERT(m_newGen.fromSpace->isEmpty());
+    ASSERT(m_newGen.oversizeBlocks.isEmpty());
+}
+
 void CopiedSpace::init()
 {
-    m_toSpace = &m_blocks1;
-    m_fromSpace = &m_blocks2;
+    m_oldGen.toSpace = &m_oldGen.blocks1;
+    m_oldGen.fromSpace = &m_oldGen.blocks2;
     
-    m_totalMemoryAllocated += HeapBlock::s_blockSize * s_initialBlockNum;
+    m_newGen.toSpace = &m_newGen.blocks1;
+    m_newGen.fromSpace = &m_newGen.blocks2;
 
-    if (!addNewBlock())
-        CRASH();
+    allocateBlock();
 }   
 
 CheckedBoolean CopiedSpace::tryAllocateSlowCase(size_t bytes, void** outPtr)
@@ -57,13 +86,12 @@ CheckedBoolean CopiedSpace::tryAllocateSlowCase(size_t bytes, void** outPtr)
     if (isOversize(bytes))
         return tryAllocateOversize(bytes, outPtr);
     
-    m_totalMemoryUtilized += m_allocator.currentUtilization();
-    if (!addNewBlock()) {
-        *outPtr = 0;
-        return false;
-    }
-    *outPtr = m_allocator.allocate(bytes);
-    ASSERT(*outPtr);
+    ASSERT(m_heap->vm()->currentThreadIsHoldingAPILock());
+    m_heap->didAllocate(m_allocator.currentCapacity());
+
+    allocateBlock();
+
+    *outPtr = m_allocator.forceAllocate(bytes);
     return true;
 }
 
@@ -71,22 +99,19 @@ CheckedBoolean CopiedSpace::tryAllocateOversize(size_t bytes, void** outPtr)
 {
     ASSERT(isOversize(bytes));
     
-    size_t blockSize = WTF::roundUpToMultipleOf<s_pageSize>(sizeof(CopiedBlock) + bytes);
-    PageAllocationAligned allocation = PageAllocationAligned::allocate(blockSize, s_pageSize, OSAllocator::JSGCHeapPages);
-    if (!static_cast<bool>(allocation)) {
-        *outPtr = 0;
-        return false;
-    }
-    CopiedBlock* block = new (NotNull, allocation.base()) CopiedBlock(allocation);
-    m_oversizeBlocks.push(block);
-    ASSERT(is8ByteAligned(block->m_offset));
-
-    m_oversizeFilter.add(reinterpret_cast<Bits>(block));
+    CopiedBlock* block = CopiedBlock::create(WTF::roundUpToMultipleOf<sizeof(double)>(sizeof(CopiedBlock) + bytes));
+    m_newGen.oversizeBlocks.push(block);
+    m_newGen.blockFilter.add(reinterpret_cast<Bits>(block));
+    m_blockSet.add(block);
+    ASSERT(!block->isOld());
     
-    m_totalMemoryAllocated += blockSize;
-    m_totalMemoryUtilized += bytes;
+    CopiedAllocator allocator;
+    allocator.setCurrentBlock(block);
+    *outPtr = allocator.forceAllocate(bytes);
+    allocator.resetCurrentBlock();
+
+    m_heap->didAllocate(block->capacity());
 
-    *outPtr = block->m_offset;
     return true;
 }
 
@@ -96,19 +121,13 @@ CheckedBoolean CopiedSpace::tryReallocate(void** ptr, size_t oldSize, size_t new
         return true;
     
     void* oldPtr = *ptr;
-    ASSERT(!m_heap->globalData()->isInitializingObject());
-
-    if (isOversize(oldSize) || isOversize(newSize))
+    ASSERT(!m_heap->vm()->isInitializingObject());
+    
+    if (CopiedSpace::blockFor(oldPtr)->isOversize() || isOversize(newSize))
         return tryReallocateOversize(ptr, oldSize, newSize);
-
-    if (m_allocator.wasLastAllocation(oldPtr, oldSize)) {
-        m_allocator.resetLastAllocation(oldPtr);
-        if (m_allocator.fitsInCurrentBlock(newSize)) {
-            m_totalMemoryUtilized += newSize - oldSize;
-            return m_allocator.allocate(newSize);
-        }
-    }
-    m_totalMemoryUtilized -= oldSize;
+    
+    if (m_allocator.tryReallocate(oldPtr, oldSize, newSize))
+        return true;
 
     void* result = 0;
     if (!tryAllocate(newSize, &result)) {
@@ -132,53 +151,83 @@ CheckedBoolean CopiedSpace::tryReallocateOversize(void** ptr, size_t oldSize, si
         *ptr = 0;
         return false;
     }
+
     memcpy(newPtr, oldPtr, oldSize);
 
-    if (isOversize(oldSize)) {
-        CopiedBlock* oldBlock = oversizeBlockFor(oldPtr);
-        m_oversizeBlocks.remove(oldBlock);
-        oldBlock->m_allocation.deallocate();
-        m_totalMemoryAllocated -= oldSize + sizeof(CopiedBlock);
+    CopiedBlock* oldBlock = CopiedSpace::blockFor(oldPtr);
+    if (oldBlock->isOversize()) {
+        // FIXME: Eagerly deallocating the old space block probably buys more confusion than
+        // value.
+        // https://bugs.webkit.org/show_bug.cgi?id=144750
+        if (oldBlock->isOld()) {
+            m_bytesRemovedFromOldSpaceDueToReallocation += oldBlock->size();
+            m_oldGen.oversizeBlocks.remove(oldBlock);
+        } else
+            m_newGen.oversizeBlocks.remove(oldBlock);
+        m_blockSet.remove(oldBlock);
+        CopiedBlock::destroy(oldBlock);
     }
     
-    m_totalMemoryUtilized -= oldSize;
-
     *ptr = newPtr;
     return true;
 }
 
-void CopiedSpace::doneFillingBlock(CopiedBlock* block)
+void CopiedSpace::doneFillingBlock(CopiedBlock* block, CopiedBlock** exchange)
 {
-    ASSERT(block);
-    ASSERT(block->m_offset < reinterpret_cast<char*>(block) + HeapBlock::s_blockSize);
     ASSERT(m_inCopyingPhase);
+    
+    if (exchange)
+        *exchange = allocateBlockForCopyingPhase();
 
-    if (block->m_offset == block->m_payload) {
-        recycleBlock(block);
+    if (!block)
         return;
-    }
 
-    {
-        MutexLocker locker(m_toSpaceLock);
-        m_toSpace->push(block);
-        m_toSpaceSet.add(block);
-        m_toSpaceFilter.add(reinterpret_cast<Bits>(block));
+    if (!block->dataSize()) {
+        recycleBorrowedBlock(block);
+        return;
     }
 
+    block->zeroFillWilderness();
+
     {
-        MutexLocker locker(m_memoryStatsLock);
-        m_totalMemoryUtilized += static_cast<size_t>(static_cast<char*>(block->m_offset) - block->m_payload);
+        // Always put the block into the old gen because it's being promoted!
+        SpinLockHolder locker(&m_toSpaceLock);
+        m_oldGen.toSpace->push(block);
+        m_blockSet.add(block);
+        m_oldGen.blockFilter.add(reinterpret_cast<Bits>(block));
     }
 
     {
         MutexLocker locker(m_loanedBlocksLock);
         ASSERT(m_numberOfLoanedBlocks > 0);
+        ASSERT(m_inCopyingPhase);
         m_numberOfLoanedBlocks--;
         if (!m_numberOfLoanedBlocks)
             m_loanedBlocksCondition.signal();
     }
 }
 
+void CopiedSpace::didStartFullCollection()
+{
+    ASSERT(heap()->operationInProgress() == FullCollection);
+    ASSERT(m_oldGen.fromSpace->isEmpty());
+    ASSERT(m_newGen.fromSpace->isEmpty());
+
+#ifndef NDEBUG
+    for (CopiedBlock* block = m_newGen.toSpace->head(); block; block = block->next())
+        ASSERT(!block->liveBytes());
+
+    for (CopiedBlock* block = m_newGen.oversizeBlocks.head(); block; block = block->next())
+        ASSERT(!block->liveBytes());
+#endif
+
+    for (CopiedBlock* block = m_oldGen.toSpace->head(); block; block = block->next())
+        block->didSurviveGC();
+
+    for (CopiedBlock* block = m_oldGen.oversizeBlocks.head(); block; block = block->next())
+        block->didSurviveGC();
+}
+
 void CopiedSpace::doneCopying()
 {
     {
@@ -187,78 +236,123 @@ void CopiedSpace::doneCopying()
             m_loanedBlocksCondition.wait(m_loanedBlocksLock);
     }
 
-    ASSERT(m_inCopyingPhase);
+    ASSERT(m_inCopyingPhase == m_shouldDoCopyPhase);
     m_inCopyingPhase = false;
-    while (!m_fromSpace->isEmpty()) {
-        CopiedBlock* block = static_cast<CopiedBlock*>(m_fromSpace->removeHead());
-        if (block->m_isPinned) {
-            block->m_isPinned = false;
-            m_toSpace->push(block);
-            continue;
-        }
 
-        m_toSpaceSet.remove(block);
-        {
-            MutexLocker locker(m_heap->m_freeBlockLock);
-            m_heap->m_freeBlocks.push(block);
-            m_heap->m_numberOfFreeBlocks++;
-        }
+    DoublyLinkedList<CopiedBlock>* toSpace;
+    DoublyLinkedList<CopiedBlock>* fromSpace;
+    TinyBloomFilter* blockFilter;
+    if (heap()->operationInProgress() == FullCollection) {
+        toSpace = m_oldGen.toSpace;
+        fromSpace = m_oldGen.fromSpace;
+        blockFilter = &m_oldGen.blockFilter;
+    } else {
+        toSpace = m_newGen.toSpace;
+        fromSpace = m_newGen.fromSpace;
+        blockFilter = &m_newGen.blockFilter;
     }
 
-    CopiedBlock* curr = static_cast<CopiedBlock*>(m_oversizeBlocks.head());
-    while (curr) {
-        CopiedBlock* next = static_cast<CopiedBlock*>(curr->next());
-        if (!curr->m_isPinned) {
-            m_oversizeBlocks.remove(curr);
-            m_totalMemoryAllocated -= curr->m_allocation.size();
-            m_totalMemoryUtilized -= curr->m_allocation.size() - sizeof(CopiedBlock);
-            curr->m_allocation.deallocate();
-        } else
-            curr->m_isPinned = false;
-        curr = next;
+    while (!fromSpace->isEmpty()) {
+        CopiedBlock* block = fromSpace->removeHead();
+        // We don't add the block to the blockSet because it was never removed.
+        ASSERT(m_blockSet.contains(block));
+        blockFilter->add(reinterpret_cast<Bits>(block));
+        block->didSurviveGC();
+        toSpace->push(block);
     }
 
-    if (!m_toSpace->head()) {
-        if (!addNewBlock())
-            CRASH();
-    } else
-        m_allocator.resetCurrentBlock(static_cast<CopiedBlock*>(m_toSpace->head()));
+    if (heap()->operationInProgress() == EdenCollection) {
+        m_oldGen.toSpace->append(*m_newGen.toSpace);
+        m_oldGen.oversizeBlocks.append(m_newGen.oversizeBlocks);
+        m_oldGen.blockFilter.add(m_newGen.blockFilter);
+        m_newGen.blockFilter.reset();
+    }
+
+    ASSERT(m_newGen.toSpace->isEmpty());
+    ASSERT(m_newGen.fromSpace->isEmpty());
+    ASSERT(m_newGen.oversizeBlocks.isEmpty());
+
+    allocateBlock();
+
+    m_shouldDoCopyPhase = false;
 }
 
-CheckedBoolean CopiedSpace::getFreshBlock(AllocationEffort allocationEffort, CopiedBlock** outBlock)
+size_t CopiedSpace::size()
 {
-    HeapBlock* heapBlock = 0;
-    CopiedBlock* block = 0;
-    {
-        MutexLocker locker(m_heap->m_freeBlockLock);
-        if (!m_heap->m_freeBlocks.isEmpty()) {
-            heapBlock = m_heap->m_freeBlocks.removeHead();
-            m_heap->m_numberOfFreeBlocks--;
-        }
-    }
-    if (heapBlock)
-        block = new (NotNull, heapBlock) CopiedBlock(heapBlock->m_allocation);
-    else if (allocationEffort == AllocationMustSucceed) {
-        if (!allocateNewBlock(&block)) {
-            *outBlock = 0;
-            ASSERT_NOT_REACHED();
-            return false;
-        }
-    } else {
-        ASSERT(allocationEffort == AllocationCanFail);
-        if (m_heap->waterMark() >= m_heap->highWaterMark() && m_heap->m_isSafeToCollect)
-            m_heap->collect(Heap::DoNotSweep);
-        
-        if (!getFreshBlock(AllocationMustSucceed, &block)) {
-            *outBlock = 0;
-            ASSERT_NOT_REACHED();
-            return false;
+    size_t calculatedSize = 0;
+
+    for (CopiedBlock* block = m_oldGen.toSpace->head(); block; block = block->next())
+        calculatedSize += block->size();
+
+    for (CopiedBlock* block = m_oldGen.fromSpace->head(); block; block = block->next())
+        calculatedSize += block->size();
+
+    for (CopiedBlock* block = m_oldGen.oversizeBlocks.head(); block; block = block->next())
+        calculatedSize += block->size();
+
+    for (CopiedBlock* block = m_newGen.toSpace->head(); block; block = block->next())
+        calculatedSize += block->size();
+
+    for (CopiedBlock* block = m_newGen.fromSpace->head(); block; block = block->next())
+        calculatedSize += block->size();
+
+    for (CopiedBlock* block = m_newGen.oversizeBlocks.head(); block; block = block->next())
+        calculatedSize += block->size();
+
+    return calculatedSize;
+}
+
+size_t CopiedSpace::capacity()
+{
+    size_t calculatedCapacity = 0;
+
+    for (CopiedBlock* block = m_oldGen.toSpace->head(); block; block = block->next())
+        calculatedCapacity += block->capacity();
+
+    for (CopiedBlock* block = m_oldGen.fromSpace->head(); block; block = block->next())
+        calculatedCapacity += block->capacity();
+
+    for (CopiedBlock* block = m_oldGen.oversizeBlocks.head(); block; block = block->next())
+        calculatedCapacity += block->capacity();
+
+    for (CopiedBlock* block = m_newGen.toSpace->head(); block; block = block->next())
+        calculatedCapacity += block->capacity();
+
+    for (CopiedBlock* block = m_newGen.fromSpace->head(); block; block = block->next())
+        calculatedCapacity += block->capacity();
+
+    for (CopiedBlock* block = m_newGen.oversizeBlocks.head(); block; block = block->next())
+        calculatedCapacity += block->capacity();
+
+    return calculatedCapacity;
+}
+
+static bool isBlockListPagedOut(double deadline, DoublyLinkedList<CopiedBlock>* list)
+{
+    unsigned itersSinceLastTimeCheck = 0;
+    CopiedBlock* current = list->head();
+    while (current) {
+        current = current->next();
+        ++itersSinceLastTimeCheck;
+        if (itersSinceLastTimeCheck >= Heap::s_timeCheckResolution) {
+            double currentTime = WTF::monotonicallyIncreasingTime();
+            if (currentTime > deadline)
+                return true;
+            itersSinceLastTimeCheck = 0;
         }
     }
-    ASSERT(block);
-    ASSERT(is8ByteAligned(block->m_offset));
-    *outBlock = block;
-    return true;
+
+    return false;
+}
+
+bool CopiedSpace::isPagedOut(double deadline)
+{
+    return isBlockListPagedOut(deadline, m_oldGen.toSpace) 
+        || isBlockListPagedOut(deadline, m_oldGen.fromSpace) 
+        || isBlockListPagedOut(deadline, &m_oldGen.oversizeBlocks)
+        || isBlockListPagedOut(deadline, m_newGen.toSpace) 
+        || isBlockListPagedOut(deadline, m_newGen.fromSpace) 
+        || isBlockListPagedOut(deadline, &m_newGen.oversizeBlocks);
 }
 
 } // namespace JSC