bmalloc: moved line caches from the deallocator to the allocator

https://bugs.webkit.org/show_bug.cgi?id=136868

Reviewed by Gavin Barraclough.

I did this mostly as a simplification, to make it easier to change the
allocation strategy.

No throughput change on MallocBench. Saves about 50kB.

Since the deallocator needs to lock the heap when freeing lines anyway,
there isn't much benefit to giving the deallocator a local cache of
deallocated lines.

We still give the allocator a local cache of lines because that does
reduce the frequency at which it needs to lock the heap in order to
acquire more lines.

* bmalloc/Allocator.cpp:
(bmalloc::Allocator::scavenge):
(bmalloc::Allocator::allocateSmallLine):
(bmalloc::Allocator::allocateMediumLine):
(bmalloc::Allocator::allocateMedium):
(bmalloc::Allocator::allocateSlowCase):
* bmalloc/Allocator.h:
* bmalloc/Deallocator.cpp:
(bmalloc::Deallocator::Deallocator):
(bmalloc::Deallocator::scavenge):
(bmalloc::Deallocator::processObjectLog):
(bmalloc::Deallocator::deallocateSmallLine): Deleted.
(bmalloc::Deallocator::allocateSmallLine): Deleted.
(bmalloc::Deallocator::deallocateMediumLine): Deleted.
(bmalloc::Deallocator::allocateMediumLine): Deleted.
* bmalloc/Deallocator.h:

* bmalloc/Sizes.h:
* bmalloc/VMAllocate.h: Took the opportunity to make the line cache size
exactly one page in size. That's about what we were shooting for anyway,
and it may make it easier to switch to per-page allocation in future.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@173675 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/Source/bmalloc/ChangeLog b/Source/bmalloc/ChangeLog
index f0ca1a178ce41555906180cc99e76422bad29fd6..2608d6a03f34ce8f94175fcbecf9695d151c99ae 100644 (file)
--- a/Source/bmalloc/ChangeLog
+++ b/Source/bmalloc/ChangeLog
@@ -1,3 +1,45 @@
+2014-09-16  Geoffrey Garen  <ggaren@apple.com>
+
+        bmalloc: moved line caches from the deallocator to the allocator
+        https://bugs.webkit.org/show_bug.cgi?id=136868
+
+        Reviewed by Gavin Barraclough.
+
+        I did this mostly as a simplification, to make it easier to change the
+        allocation strategy.
+
+        No throughput change on MallocBench. Saves about 50kB.
+
+        Since the deallocator needs to lock the heap when freeing lines anyway,
+        there isn't much benefit to giving the deallocator a local cache of
+        deallocated lines.
+
+        We still give the allocator a local cache of lines because that does
+        reduce the frequency at which it needs to lock the heap in order to
+        acquire more lines.
+
+        * bmalloc/Allocator.cpp:
+        (bmalloc::Allocator::scavenge):
+        (bmalloc::Allocator::allocateSmallLine):
+        (bmalloc::Allocator::allocateMediumLine):
+        (bmalloc::Allocator::allocateMedium):
+        (bmalloc::Allocator::allocateSlowCase):
+        * bmalloc/Allocator.h:
+        * bmalloc/Deallocator.cpp:
+        (bmalloc::Deallocator::Deallocator):
+        (bmalloc::Deallocator::scavenge):
+        (bmalloc::Deallocator::processObjectLog):
+        (bmalloc::Deallocator::deallocateSmallLine): Deleted.
+        (bmalloc::Deallocator::allocateSmallLine): Deleted.
+        (bmalloc::Deallocator::deallocateMediumLine): Deleted.
+        (bmalloc::Deallocator::allocateMediumLine): Deleted.
+        * bmalloc/Deallocator.h:
+
+        * bmalloc/Sizes.h:
+        * bmalloc/VMAllocate.h: Took the opportunity to make the line cache size
+        exactly one page in size. That's about what we were shooting for anyway,
+        and it may make it easier to switch to per-page allocation in future.
+
 2014-09-15  Geoffrey Garen  <ggaren@apple.com>
 
         bmalloc: allocate small and medium objects using the same bump pointer class

diff --git a/Source/bmalloc/bmalloc/Allocator.cpp b/Source/bmalloc/bmalloc/Allocator.cpp
index c498a0f4025d906e3599d0714bb9a2020638b555..00e33c0cfe977334e595f9598bd70bfdc1876b2a 100644 (file)
--- a/Source/bmalloc/bmalloc/Allocator.cpp
+++ b/Source/bmalloc/bmalloc/Allocator.cpp
@@ -63,6 +63,43 @@ void Allocator::scavenge()
             m_deallocator.deallocate(allocator.allocate());
         allocator.clear();
     }
+
+    std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
+    Heap* heap = PerProcess<Heap>::getFastCase();
+    
+    for (auto& smallLineCache : m_smallLineCaches) {
+        while (smallLineCache.size())
+            heap->deallocateSmallLine(lock, smallLineCache.pop());
+    }
+    while (m_mediumLineCache.size())
+        heap->deallocateMediumLine(lock, m_mediumLineCache.pop());
+}
+
+SmallLine* Allocator::allocateSmallLine(size_t smallSizeClass)
+{
+    SmallLineCache& smallLineCache = m_smallLineCaches[smallSizeClass];
+    if (!smallLineCache.size()) {
+        std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
+        Heap* heap = PerProcess<Heap>::getFastCase();
+
+        while (smallLineCache.size() != smallLineCache.capacity())
+            smallLineCache.push(heap->allocateSmallLine(lock, smallSizeClass));
+    }
+
+    return smallLineCache.pop();
+}
+
+MediumLine* Allocator::allocateMediumLine()
+{
+    if (!m_mediumLineCache.size()) {
+        std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
+        Heap* heap = PerProcess<Heap>::getFastCase();
+
+        while (m_mediumLineCache.size() != m_mediumLineCache.capacity())
+            m_mediumLineCache.push(heap->allocateMediumLine(lock));
+    }
+
+    return m_mediumLineCache.pop();
 }
 
 void* Allocator::allocateLarge(size_t size)
@@ -84,7 +121,7 @@ void* Allocator::allocateMedium(size_t size)
     BumpAllocator& allocator = m_mediumAllocators[mediumSizeClassFor(size)];
 
     if (!allocator.canAllocate())
-        allocator.refill(m_deallocator.allocateMediumLine());
+        allocator.refill(allocateMediumLine());
     return allocator.allocate();
 }
 
@@ -97,7 +134,7 @@ IF_DEBUG(
     if (size <= smallMax) {
         size_t smallSizeClass = smallSizeClassFor(size);
         BumpAllocator& allocator = m_smallAllocators[smallSizeClass];
-        allocator.refill(m_deallocator.allocateSmallLine(smallSizeClass));
+        allocator.refill(allocateSmallLine(smallSizeClass));
         return allocator.allocate();
     }
 

diff --git a/Source/bmalloc/bmalloc/Allocator.h b/Source/bmalloc/bmalloc/Allocator.h
index 65f149af8fb56b890332ef59a4f63eae82c76655..ff23b01007d65771876a4bcaad0a460ce1cf82f9 100644 (file)
--- a/Source/bmalloc/bmalloc/Allocator.h
+++ b/Source/bmalloc/bmalloc/Allocator.h
@@ -51,16 +51,25 @@ public:
     void scavenge();
 
 private:
+    typedef FixedVector<SmallLine*, smallLineCacheCapacity> SmallLineCache;
+    typedef FixedVector<MediumLine*, mediumLineCacheCapacity> MediumLineCache;
+
     void* allocateFastCase(BumpAllocator&);
 
     void* allocateMedium(size_t);
     void* allocateLarge(size_t);
     void* allocateXLarge(size_t);
     
+    SmallLine* allocateSmallLine(size_t smallSizeClass);
+    MediumLine* allocateMediumLine();
+    
     Deallocator& m_deallocator;
 
     std::array<BumpAllocator, smallMax / alignment> m_smallAllocators;
     std::array<BumpAllocator, mediumMax / alignment> m_mediumAllocators;
+
+    std::array<SmallLineCache, smallMax / alignment> m_smallLineCaches;
+    MediumLineCache m_mediumLineCache;
 };
 
 inline bool Allocator::allocateFastCase(size_t size, void*& object)

diff --git a/Source/bmalloc/bmalloc/Deallocator.cpp b/Source/bmalloc/bmalloc/Deallocator.cpp
index fec941b22ff53816d8921ca664e8de4a54bc062c..b8dae219f4c73390401001938280eb50a993e7f6 100644 (file)
--- a/Source/bmalloc/bmalloc/Deallocator.cpp
+++ b/Source/bmalloc/bmalloc/Deallocator.cpp
@@ -39,9 +39,6 @@ using namespace std;
 namespace bmalloc {
 
 Deallocator::Deallocator()
-    : m_objectLog()
-    , m_smallLineCaches()
-    , m_mediumLineCache()
 {
 }
 
@@ -53,16 +50,6 @@ Deallocator::~Deallocator()
 void Deallocator::scavenge()
 {
     processObjectLog();
-    
-    std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
-    Heap* heap = PerProcess<Heap>::getFastCase();
-    
-    for (auto& smallLineCache : m_smallLineCaches) {
-        while (smallLineCache.size())
-            heap->deallocateSmallLine(lock, smallLineCache.pop());
-    }
-    while (m_mediumLineCache.size())
-        heap->deallocateMediumLine(lock, m_mediumLineCache.pop());
 }
 
 void Deallocator::deallocateLarge(void* object)
@@ -80,19 +67,20 @@ void Deallocator::deallocateXLarge(void* object)
 void Deallocator::processObjectLog()
 {
     std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
+    Heap* heap = PerProcess<Heap>::getFastCase();
     
     for (auto object : m_objectLog) {
         if (isSmall(object)) {
             SmallLine* line = SmallLine::get(object);
             if (!line->deref(lock))
                 continue;
-            deallocateSmallLine(lock, line);
+            heap->deallocateSmallLine(lock, line);
         } else {
             BASSERT(isSmallOrMedium(object));
             MediumLine* line = MediumLine::get(object);
             if (!line->deref(lock))
                 continue;
-            deallocateMediumLine(lock, line);
+            heap->deallocateMediumLine(lock, line);
         }
     }
     
@@ -119,48 +107,4 @@ void Deallocator::deallocateSlowCase(void* object)
     return deallocateXLarge(object);
 }
 
-void Deallocator::deallocateSmallLine(std::lock_guard<StaticMutex>& lock, SmallLine* line)
-{
-    SmallLineCache& smallLineCache = m_smallLineCaches[SmallPage::get(line)->smallSizeClass()];
-    if (smallLineCache.size() == smallLineCache.capacity())
-        return PerProcess<Heap>::getFastCase()->deallocateSmallLine(lock, line);
-
-    smallLineCache.push(line);
-}
-
-SmallLine* Deallocator::allocateSmallLine(size_t smallSizeClass)
-{
-    SmallLineCache& smallLineCache = m_smallLineCaches[smallSizeClass];
-    if (!smallLineCache.size()) {
-        std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
-        Heap* heap = PerProcess<Heap>::getFastCase();
-
-        while (smallLineCache.size() != smallLineCache.capacity())
-            smallLineCache.push(heap->allocateSmallLine(lock, smallSizeClass));
-    }
-
-    return smallLineCache.pop();
-}
-
-void Deallocator::deallocateMediumLine(std::lock_guard<StaticMutex>& lock, MediumLine* line)
-{
-    if (m_mediumLineCache.size() == m_mediumLineCache.capacity())
-        return PerProcess<Heap>::getFastCase()->deallocateMediumLine(lock, line);
-
-    m_mediumLineCache.push(line);
-}
-
-MediumLine* Deallocator::allocateMediumLine()
-{
-    if (!m_mediumLineCache.size()) {
-        std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
-        Heap* heap = PerProcess<Heap>::getFastCase();
-
-        while (m_mediumLineCache.size() != m_mediumLineCache.capacity())
-            m_mediumLineCache.push(heap->allocateMediumLine(lock));
-    }
-
-    return m_mediumLineCache.pop();
-}
-
 } // namespace bmalloc

diff --git a/Source/bmalloc/bmalloc/Deallocator.h b/Source/bmalloc/bmalloc/Deallocator.h
index 50a91ff0d4782ec538ad44b49640ef863c7e05a3..4fbcf3bf20b9adecc7ad6e6c6e255736bd6bf66a 100644 (file)
--- a/Source/bmalloc/bmalloc/Deallocator.h
+++ b/Source/bmalloc/bmalloc/Deallocator.h
@@ -44,25 +44,14 @@ public:
     bool deallocateFastCase(void*);
     void deallocateSlowCase(void*);
 
-    void deallocateSmallLine(std::lock_guard<StaticMutex>&, SmallLine*);
-    SmallLine* allocateSmallLine(size_t smallSizeClass);
-
-    void deallocateMediumLine(std::lock_guard<StaticMutex>&, MediumLine*);
-    MediumLine* allocateMediumLine();
-    
     void scavenge();
     
 private:
-    typedef FixedVector<SmallLine*, smallLineCacheCapacity> SmallLineCache;
-    typedef FixedVector<MediumLine*, mediumLineCacheCapacity> MediumLineCache;
-
     void deallocateLarge(void*);
     void deallocateXLarge(void*);
     void processObjectLog();
 
     FixedVector<void*, deallocatorLogCapacity> m_objectLog;
-    std::array<SmallLineCache, smallMax / alignment> m_smallLineCaches;
-    MediumLineCache m_mediumLineCache;
 };
 
 inline bool Deallocator::deallocateFastCase(void* object)

diff --git a/Source/bmalloc/bmalloc/Sizes.h b/Source/bmalloc/bmalloc/Sizes.h
index ad834953d33cf1af2953f79c130b1dd50c7fea9e..8f19b93bb92f3c8fa15b9403a1c9a89b6827ef86 100644 (file)
--- a/Source/bmalloc/bmalloc/Sizes.h
+++ b/Source/bmalloc/bmalloc/Sizes.h
@@ -27,6 +27,7 @@
 #define Sizes_h
 
 #include "Algorithm.h"
+#include "BPlatform.h"
 #include <algorithm>
 #include <cstdint>
 #include <cstddef>
@@ -45,6 +46,13 @@ namespace Sizes {
     static const size_t alignment = 8;
     static const size_t alignmentMask = alignment - 1ul;
 
+#if BPLATFORM(IOS)
+    static const size_t vmPageSize = 16 * kB;
+#else
+    static const size_t vmPageSize = 4 * kB;
+#endif
+    static const size_t vmPageMask = ~(vmPageSize - 1);
+    
     static const size_t superChunkSize = 32 * MB;
 
     static const size_t smallMax = 256;
@@ -84,8 +92,8 @@ namespace Sizes {
 
     static const size_t deallocatorLogCapacity = 256;
 
-    static const size_t smallLineCacheCapacity = 16;
-    static const size_t mediumLineCacheCapacity = 8;
+    static const size_t smallLineCacheCapacity = vmPageSize / smallLineSize;
+    static const size_t mediumLineCacheCapacity = vmPageSize / mediumLineSize;
     
     static const std::chrono::milliseconds scavengeSleepDuration = std::chrono::milliseconds(512);
 

diff --git a/Source/bmalloc/bmalloc/VMAllocate.h b/Source/bmalloc/bmalloc/VMAllocate.h
index 04307f5db88dff7ee8f62be0afa533ee94354e64..7f41dd4628f47e3d6ab7165e07614faf84cc1d8e 100644 (file)
--- a/Source/bmalloc/bmalloc/VMAllocate.h
+++ b/Source/bmalloc/bmalloc/VMAllocate.h
@@ -27,7 +27,6 @@
 #define VMAllocate_h
 
 #include "BAssert.h"
-#include "BPlatform.h"
 #include "Range.h"
 #include "Sizes.h"
 #include "Syscall.h"
@@ -40,14 +39,6 @@ namespace bmalloc {
 
 #define BMALLOC_VM_TAG VM_MAKE_TAG(VM_MEMORY_TCMALLOC)
 
-#if BPLATFORM(IOS)
-static const size_t vmPageSize = 16 * kB;
-#else
-static const size_t vmPageSize = 4 * kB;
-#endif
-
-static const size_t vmPageMask = ~(vmPageSize - 1);
-    
 inline size_t vmSize(size_t size)
 {
     return roundUpToMultipleOf<vmPageSize>(size);