Unreviewed, roll out http://trac.webkit.org/changeset/187972.
[WebKit-https.git] / Source / JavaScriptCore / heap / CopiedSpace.cpp
index ff051d9..9592eaa 100644 (file)
 #include "config.h"
 #include "CopiedSpace.h"
 
-#include "CopiedSpaceInlineMethods.h"
+#include "CopiedSpaceInlines.h"
+#include "GCActivityCallback.h"
+#include "JSCInlines.h"
+#include "Options.h"
 
 namespace JSC {
 
+CopiedSpace::CopiedSpace(Heap* heap)
+    : m_heap(heap)
+    , m_inCopyingPhase(false)
+    , m_shouldDoCopyPhase(false)
+    , m_numberOfLoanedBlocks(0)
+    , m_bytesRemovedFromOldSpaceDueToReallocation(0)
+{
+}
+
+CopiedSpace::~CopiedSpace()
+{
+    while (!m_oldGen.toSpace->isEmpty())
+        CopiedBlock::destroy(m_oldGen.toSpace->removeHead());
+
+    while (!m_oldGen.fromSpace->isEmpty())
+        CopiedBlock::destroy(m_oldGen.fromSpace->removeHead());
+
+    while (!m_oldGen.oversizeBlocks.isEmpty())
+        CopiedBlock::destroy(m_oldGen.oversizeBlocks.removeHead());
+
+    while (!m_newGen.toSpace->isEmpty())
+        CopiedBlock::destroy(m_newGen.toSpace->removeHead());
+
+    while (!m_newGen.fromSpace->isEmpty())
+        CopiedBlock::destroy(m_newGen.fromSpace->removeHead());
+
+    while (!m_newGen.oversizeBlocks.isEmpty())
+        CopiedBlock::destroy(m_newGen.oversizeBlocks.removeHead());
+
+    ASSERT(m_oldGen.toSpace->isEmpty());
+    ASSERT(m_oldGen.fromSpace->isEmpty());
+    ASSERT(m_oldGen.oversizeBlocks.isEmpty());
+    ASSERT(m_newGen.toSpace->isEmpty());
+    ASSERT(m_newGen.fromSpace->isEmpty());
+    ASSERT(m_newGen.oversizeBlocks.isEmpty());
+}
+
+void CopiedSpace::init()
+{
+    m_oldGen.toSpace = &m_oldGen.blocks1;
+    m_oldGen.fromSpace = &m_oldGen.blocks2;
+    
+    m_newGen.toSpace = &m_newGen.blocks1;
+    m_newGen.fromSpace = &m_newGen.blocks2;
+
+    allocateBlock();
+}   
+
 CheckedBoolean CopiedSpace::tryAllocateSlowCase(size_t bytes, void** outPtr)
 {
     if (isOversize(bytes))
         return tryAllocateOversize(bytes, outPtr);
     
-    m_totalMemoryUtilized += static_cast<size_t>(static_cast<char*>(m_currentBlock->m_offset) - m_currentBlock->m_payload);
-    if (!addNewBlock()) {
-        *outPtr = 0;
+    ASSERT(m_heap->vm()->currentThreadIsHoldingAPILock());
+    m_heap->didAllocate(m_allocator.currentCapacity());
+
+    allocateBlock();
+
+    *outPtr = m_allocator.forceAllocate(bytes);
+    return true;
+}
+
+CheckedBoolean CopiedSpace::tryAllocateOversize(size_t bytes, void** outPtr)
+{
+    ASSERT(isOversize(bytes));
+    
+    CopiedBlock* block = CopiedBlock::create(WTF::roundUpToMultipleOf<sizeof(double)>(sizeof(CopiedBlock) + bytes));
+    m_newGen.oversizeBlocks.push(block);
+    m_newGen.blockFilter.add(reinterpret_cast<Bits>(block));
+    m_blockSet.add(block);
+    ASSERT(!block->isOld());
+    
+    CopiedAllocator allocator;
+    allocator.setCurrentBlock(block);
+    *outPtr = allocator.forceAllocate(bytes);
+    allocator.resetCurrentBlock();
+
+    m_heap->didAllocate(block->capacity());
+
+    return true;
+}
+
+CheckedBoolean CopiedSpace::tryReallocate(void** ptr, size_t oldSize, size_t newSize)
+{
+    if (oldSize >= newSize)
+        return true;
+    
+    void* oldPtr = *ptr;
+    ASSERT(!m_heap->vm()->isInitializingObject());
+    
+    if (CopiedSpace::blockFor(oldPtr)->isOversize() || isOversize(newSize))
+        return tryReallocateOversize(ptr, oldSize, newSize);
+    
+    if (m_allocator.tryReallocate(oldPtr, oldSize, newSize))
+        return true;
+
+    void* result = 0;
+    if (!tryAllocate(newSize, &result)) {
+        *ptr = 0;
         return false;
     }
-    m_toSpaceFilter.add(reinterpret_cast<Bits>(m_currentBlock));
-    m_toSpaceSet.add(m_currentBlock);
-    *outPtr = allocateFromBlock(m_currentBlock, bytes);
+    memcpy(result, oldPtr, oldSize);
+    *ptr = result;
     return true;
 }
 
+CheckedBoolean CopiedSpace::tryReallocateOversize(void** ptr, size_t oldSize, size_t newSize)
+{
+    ASSERT(isOversize(oldSize) || isOversize(newSize));
+    ASSERT(newSize > oldSize);
+
+    void* oldPtr = *ptr;
+    
+    void* newPtr = 0;
+    if (!tryAllocateOversize(newSize, &newPtr)) {
+        *ptr = 0;
+        return false;
+    }
+
+    memcpy(newPtr, oldPtr, oldSize);
+
+    CopiedBlock* oldBlock = CopiedSpace::blockFor(oldPtr);
+    if (oldBlock->isOversize()) {
+        // FIXME: Eagerly deallocating the old space block probably buys more confusion than
+        // value.
+        // https://bugs.webkit.org/show_bug.cgi?id=144750
+        if (oldBlock->isOld()) {
+            m_bytesRemovedFromOldSpaceDueToReallocation += oldBlock->size();
+            m_oldGen.oversizeBlocks.remove(oldBlock);
+        } else
+            m_newGen.oversizeBlocks.remove(oldBlock);
+        m_blockSet.remove(oldBlock);
+        CopiedBlock::destroy(oldBlock);
+    }
+    
+    *ptr = newPtr;
+    return true;
+}
+
+void CopiedSpace::doneFillingBlock(CopiedBlock* block, CopiedBlock** exchange)
+{
+    ASSERT(m_inCopyingPhase);
+    
+    if (exchange)
+        *exchange = allocateBlockForCopyingPhase();
+
+    if (!block)
+        return;
+
+    if (!block->dataSize()) {
+        recycleBorrowedBlock(block);
+        return;
+    }
+
+    block->zeroFillWilderness();
+
+    {
+        // Always put the block into the old gen because it's being promoted!
+        SpinLockHolder locker(&m_toSpaceLock);
+        m_oldGen.toSpace->push(block);
+        m_blockSet.add(block);
+        m_oldGen.blockFilter.add(reinterpret_cast<Bits>(block));
+    }
+
+    {
+        MutexLocker locker(m_loanedBlocksLock);
+        ASSERT(m_numberOfLoanedBlocks > 0);
+        ASSERT(m_inCopyingPhase);
+        m_numberOfLoanedBlocks--;
+        if (!m_numberOfLoanedBlocks)
+            m_loanedBlocksCondition.signal();
+    }
+}
+
+void CopiedSpace::didStartFullCollection()
+{
+    ASSERT(heap()->operationInProgress() == FullCollection);
+    ASSERT(m_oldGen.fromSpace->isEmpty());
+    ASSERT(m_newGen.fromSpace->isEmpty());
+
+#ifndef NDEBUG
+    for (CopiedBlock* block = m_newGen.toSpace->head(); block; block = block->next())
+        ASSERT(!block->liveBytes());
+
+    for (CopiedBlock* block = m_newGen.oversizeBlocks.head(); block; block = block->next())
+        ASSERT(!block->liveBytes());
+#endif
+
+    for (CopiedBlock* block = m_oldGen.toSpace->head(); block; block = block->next())
+        block->didSurviveGC();
+
+    for (CopiedBlock* block = m_oldGen.oversizeBlocks.head(); block; block = block->next())
+        block->didSurviveGC();
+}
+
+void CopiedSpace::doneCopying()
+{
+    {
+        MutexLocker locker(m_loanedBlocksLock);
+        while (m_numberOfLoanedBlocks > 0)
+            m_loanedBlocksCondition.wait(m_loanedBlocksLock);
+    }
+
+    ASSERT(m_inCopyingPhase == m_shouldDoCopyPhase);
+    m_inCopyingPhase = false;
+
+    DoublyLinkedList<CopiedBlock>* toSpace;
+    DoublyLinkedList<CopiedBlock>* fromSpace;
+    TinyBloomFilter* blockFilter;
+    if (heap()->operationInProgress() == FullCollection) {
+        toSpace = m_oldGen.toSpace;
+        fromSpace = m_oldGen.fromSpace;
+        blockFilter = &m_oldGen.blockFilter;
+    } else {
+        toSpace = m_newGen.toSpace;
+        fromSpace = m_newGen.fromSpace;
+        blockFilter = &m_newGen.blockFilter;
+    }
+
+    while (!fromSpace->isEmpty()) {
+        CopiedBlock* block = fromSpace->removeHead();
+        // We don't add the block to the blockSet because it was never removed.
+        ASSERT(m_blockSet.contains(block));
+        blockFilter->add(reinterpret_cast<Bits>(block));
+        block->didSurviveGC();
+        toSpace->push(block);
+    }
+
+    if (heap()->operationInProgress() == EdenCollection) {
+        m_oldGen.toSpace->append(*m_newGen.toSpace);
+        m_oldGen.oversizeBlocks.append(m_newGen.oversizeBlocks);
+        m_oldGen.blockFilter.add(m_newGen.blockFilter);
+        m_newGen.blockFilter.reset();
+    }
+
+    ASSERT(m_newGen.toSpace->isEmpty());
+    ASSERT(m_newGen.fromSpace->isEmpty());
+    ASSERT(m_newGen.oversizeBlocks.isEmpty());
+
+    allocateBlock();
+
+    m_shouldDoCopyPhase = false;
+}
+
+size_t CopiedSpace::size()
+{
+    size_t calculatedSize = 0;
+
+    for (CopiedBlock* block = m_oldGen.toSpace->head(); block; block = block->next())
+        calculatedSize += block->size();
+
+    for (CopiedBlock* block = m_oldGen.fromSpace->head(); block; block = block->next())
+        calculatedSize += block->size();
+
+    for (CopiedBlock* block = m_oldGen.oversizeBlocks.head(); block; block = block->next())
+        calculatedSize += block->size();
+
+    for (CopiedBlock* block = m_newGen.toSpace->head(); block; block = block->next())
+        calculatedSize += block->size();
+
+    for (CopiedBlock* block = m_newGen.fromSpace->head(); block; block = block->next())
+        calculatedSize += block->size();
+
+    for (CopiedBlock* block = m_newGen.oversizeBlocks.head(); block; block = block->next())
+        calculatedSize += block->size();
+
+    return calculatedSize;
+}
+
+size_t CopiedSpace::capacity()
+{
+    size_t calculatedCapacity = 0;
+
+    for (CopiedBlock* block = m_oldGen.toSpace->head(); block; block = block->next())
+        calculatedCapacity += block->capacity();
+
+    for (CopiedBlock* block = m_oldGen.fromSpace->head(); block; block = block->next())
+        calculatedCapacity += block->capacity();
+
+    for (CopiedBlock* block = m_oldGen.oversizeBlocks.head(); block; block = block->next())
+        calculatedCapacity += block->capacity();
+
+    for (CopiedBlock* block = m_newGen.toSpace->head(); block; block = block->next())
+        calculatedCapacity += block->capacity();
+
+    for (CopiedBlock* block = m_newGen.fromSpace->head(); block; block = block->next())
+        calculatedCapacity += block->capacity();
+
+    for (CopiedBlock* block = m_newGen.oversizeBlocks.head(); block; block = block->next())
+        calculatedCapacity += block->capacity();
+
+    return calculatedCapacity;
+}
+
+static bool isBlockListPagedOut(double deadline, DoublyLinkedList<CopiedBlock>* list)
+{
+    unsigned itersSinceLastTimeCheck = 0;
+    CopiedBlock* current = list->head();
+    while (current) {
+        current = current->next();
+        ++itersSinceLastTimeCheck;
+        if (itersSinceLastTimeCheck >= Heap::s_timeCheckResolution) {
+            double currentTime = WTF::monotonicallyIncreasingTime();
+            if (currentTime > deadline)
+                return true;
+            itersSinceLastTimeCheck = 0;
+        }
+    }
+
+    return false;
+}
+
+bool CopiedSpace::isPagedOut(double deadline)
+{
+    return isBlockListPagedOut(deadline, m_oldGen.toSpace) 
+        || isBlockListPagedOut(deadline, m_oldGen.fromSpace) 
+        || isBlockListPagedOut(deadline, &m_oldGen.oversizeBlocks)
+        || isBlockListPagedOut(deadline, m_newGen.toSpace) 
+        || isBlockListPagedOut(deadline, m_newGen.fromSpace) 
+        || isBlockListPagedOut(deadline, &m_newGen.oversizeBlocks);
+}
+
 } // namespace JSC