[bmalloc] IsoHeap should have lower tier using shared IsoPage
[WebKit-https.git] / Source / bmalloc / bmalloc / IsoHeapImplInlines.h
index 9a11e9e..23e2b3c 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2017 Apple Inc. All rights reserved.
+ * Copyright (C) 2017-2018 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -27,6 +27,8 @@
 
 #include "IsoHeapImpl.h"
 #include "IsoTLSDeallocatorEntry.h"
+#include "IsoSharedHeapInlines.h"
+#include "IsoSharedPageInlines.h"
 
 namespace bmalloc {
 
@@ -60,11 +62,13 @@ EligibilityResult<Config> IsoHeapImpl<Config>::takeFirstEligible()
     
     for (; m_firstEligibleDirectory; m_firstEligibleDirectory = m_firstEligibleDirectory->next) {
         EligibilityResult<Config> result = m_firstEligibleDirectory->payload.takeFirstEligible();
-        if (result.kind != EligibilityKind::Full)
+        if (result.kind != EligibilityKind::Full) {
+            m_directoryHighWatermark = std::max(m_directoryHighWatermark, m_firstEligibleDirectory->index());
             return result;
+        }
     }
     
-    auto* newDirectory = new IsoDirectoryPage<Config>(*this, m_numDirectoryPages++);
+    auto* newDirectory = new IsoDirectoryPage<Config>(*this, m_nextDirectoryPageIndex++);
     if (m_headDirectory) {
         m_tailDirectory->next = newDirectory;
         m_tailDirectory = newDirectory;
@@ -73,6 +77,7 @@ EligibilityResult<Config> IsoHeapImpl<Config>::takeFirstEligible()
         m_headDirectory = newDirectory;
         m_tailDirectory = newDirectory;
     }
+    m_directoryHighWatermark = newDirectory->index();
     m_firstEligibleDirectory = newDirectory;
     EligibilityResult<Config> result = newDirectory->payload.takeFirstEligible();
     RELEASE_BASSERT(result.kind != EligibilityKind::Full);
@@ -98,21 +103,20 @@ void IsoHeapImpl<Config>::didBecomeEligible(IsoDirectory<Config, IsoDirectoryPag
 template<typename Config>
 void IsoHeapImpl<Config>::scavenge(Vector<DeferredDecommit>& decommits)
 {
+    std::lock_guard<Mutex> locker(this->lock);
     forEachDirectory(
         [&] (auto& directory) {
             directory.scavenge(decommits);
         });
+    m_directoryHighWatermark = 0;
+    m_numberOfAllocationsFromSharedInOneCycle = 0;
+    m_allocationMode = AllocationMode::Init;
 }
 
 template<typename Config>
 size_t IsoHeapImpl<Config>::freeableMemory()
 {
-    size_t result = 0;
-    forEachDirectory(
-        [&] (auto& directory) {
-            result += directory.freeableMemory();
-        });
-    return result;
+    return m_freeableMemory;
 }
 
 template<typename Config>
@@ -176,6 +180,11 @@ void IsoHeapImpl<Config>::forEachLiveObject(const Func& func)
         [&] (IsoPage<Config>& page) {
             page.forEachLiveObject(func);
         });
+    for (unsigned index = 0; index < maxAllocationFromShared; ++index) {
+        void* pointer = m_sharedCells[index];
+        if (pointer && !(m_usableBits & (1U << index)))
+            func(pointer);
+    }
 }
 
 template<typename Config>
@@ -207,5 +216,94 @@ void IsoHeapImpl<Config>::didDecommit(void* ptr, size_t bytes)
 #endif
 }
 
+template<typename Config>
+void IsoHeapImpl<Config>::isNowFreeable(void* ptr, size_t bytes)
+{
+    BUNUSED_PARAM(ptr);
+    m_freeableMemory += bytes;
+}
+
+template<typename Config>
+void IsoHeapImpl<Config>::isNoLongerFreeable(void* ptr, size_t bytes)
+{
+    BUNUSED_PARAM(ptr);
+    m_freeableMemory -= bytes;
+}
+
+template<typename Config>
+AllocationMode IsoHeapImpl<Config>::updateAllocationMode()
+{
+    // Exhaust shared free cells, which means we should start activating the fast allocation mode for this type.
+    if (!m_usableBits) {
+        m_slowPathTimePoint = std::chrono::steady_clock::now();
+        return AllocationMode::Fast;
+    }
+
+    switch (m_allocationMode) {
+    case AllocationMode::Shared:
+        // Currently in the shared allocation mode. Until we exhaust shared free cells, continue using the shared allocation mode.
+        // But if we allocate so many shared cells within very short period, we should use the fast allocation mode instead.
+        // This avoids the following pathological case.
+        //
+        //     for (int i = 0; i < 1e6; ++i) {
+        //         auto* ptr = allocate();
+        //         ...
+        //         free(ptr);
+        //     }
+        if (m_numberOfAllocationsFromSharedInOneCycle <= IsoPage<Config>::numObjects)
+            return AllocationMode::Shared;
+        BFALLTHROUGH;
+
+    case AllocationMode::Fast: {
+        // The allocation pattern may change. We should check the allocation rate and decide which mode is more appropriate.
+        // If we don't go to the allocation slow path during 1~ seconds, we think the allocation becomes quiescent state.
+        auto now = std::chrono::steady_clock::now();
+        if ((now - m_slowPathTimePoint) < std::chrono::seconds(1)) {
+            m_slowPathTimePoint = now;
+            return AllocationMode::Fast;
+        }
+
+        m_numberOfAllocationsFromSharedInOneCycle = 0;
+        m_slowPathTimePoint = now;
+        return AllocationMode::Shared;
+    }
+
+    case AllocationMode::Init:
+        m_slowPathTimePoint = std::chrono::steady_clock::now();
+        return AllocationMode::Shared;
+    }
+
+    return AllocationMode::Shared;
+}
+
+template<typename Config>
+void* IsoHeapImpl<Config>::allocateFromShared(bool abortOnFailure)
+{
+    static constexpr bool verbose = false;
+
+    unsigned indexPlusOne = __builtin_ffs(m_usableBits);
+    BASSERT(indexPlusOne);
+    unsigned index = indexPlusOne - 1;
+    void* result = result = m_sharedCells[index];
+    if (result) {
+        if (verbose)
+            fprintf(stderr, "%p: allocated %p from shared again of size %u\n", this, result, Config::objectSize);
+    } else {
+        constexpr unsigned objectSizeWithHeapImplPointer = Config::objectSize + sizeof(uint8_t);
+        result = IsoSharedHeap::get()->allocateNew<objectSizeWithHeapImplPointer>(abortOnFailure);
+        if (!result)
+            return nullptr;
+        if (verbose)
+            fprintf(stderr, "%p: allocated %p from shared of size %u\n", this, result, Config::objectSize);
+        BASSERT(index < IsoHeapImplBase::maxAllocationFromShared);
+        *indexSlotFor<Config>(result) = index;
+        m_sharedCells[index] = result;
+    }
+    BASSERT(result);
+    m_usableBits &= (~(1U << index));
+    ++m_numberOfAllocationsFromSharedInOneCycle;
+    return result;
+}
+
 } // namespace bmalloc