[bmalloc] IsoHeap should have lower tier using shared IsoPage
[WebKit-https.git] / Source / bmalloc / bmalloc / IsoTLSInlines.h
index f43fddc..1d14b7a 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2017 Apple Inc. All rights reserved.
+ * Copyright (C) 2017-2018 Apple Inc. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
 
 #pragma once
 
+#include "Environment.h"
 #include "IsoHeapImpl.h"
 #include "IsoTLS.h"
+#include "bmalloc.h"
 
 namespace bmalloc {
 
 template<typename Type>
 void* IsoTLS::allocate(api::IsoHeap<Type>& handle, bool abortOnFailure)
 {
-    return allocator<typename api::IsoHeap<Type>::Config>(handle).allocate(abortOnFailure);
+    return allocateImpl<typename api::IsoHeap<Type>::Config>(handle, abortOnFailure);
 }
 
 template<typename Type>
@@ -41,7 +43,7 @@ void IsoTLS::deallocate(api::IsoHeap<Type>& handle, void* p)
 {
     if (!p)
         return;
-    deallocator<typename api::IsoHeap<Type>::Config>(handle).deallocate(p);
+    deallocateImpl<typename api::IsoHeap<Type>::Config>(handle, p);
 }
 
 template<typename Type>
@@ -62,24 +64,88 @@ void IsoTLS::scavenge(api::IsoHeap<Type>& handle)
 }
 
 template<typename Config, typename Type>
-IsoAllocator<Config>& IsoTLS::allocator(api::IsoHeap<Type>& handle)
+void* IsoTLS::allocateImpl(api::IsoHeap<Type>& handle, bool abortOnFailure)
 {
     unsigned offset = handle.allocatorOffset();
     IsoTLS* tls = get();
-    if (!tls || offset >= tls->m_extent) {
-        tls = ensureHeapAndEntries(handle);
-        offset = handle.allocatorOffset();
+    if (!tls || offset >= tls->m_extent)
+        return allocateSlow<Config>(handle, abortOnFailure);
+    return tls->allocateFast<Config>(offset, abortOnFailure);
+}
+
+template<typename Config>
+void* IsoTLS::allocateFast(unsigned offset, bool abortOnFailure)
+{
+    return reinterpret_cast<IsoAllocator<Config>*>(m_data + offset)->allocate(abortOnFailure);
+}
+
+template<typename Config, typename Type>
+BNO_INLINE void* IsoTLS::allocateSlow(api::IsoHeap<Type>& handle, bool abortOnFailure)
+{
+    for (;;) {
+        switch (s_mallocFallbackState) {
+        case MallocFallbackState::Undecided:
+            determineMallocFallbackState();
+            continue;
+        case MallocFallbackState::FallBackToMalloc:
+            return api::tryMalloc(Config::objectSize);
+        case MallocFallbackState::DoNotFallBack:
+            break;
+        }
+        break;
     }
-    return *reinterpret_cast<IsoAllocator<Config>*>(tls->m_data + offset);
+    
+    // If debug heap is enabled, s_mallocFallbackState becomes MallocFallbackState::FallBackToMalloc.
+    BASSERT(!Environment::get()->isDebugHeapEnabled());
+    
+    IsoTLS* tls = ensureHeapAndEntries(handle);
+    
+    return tls->allocateFast<Config>(handle.allocatorOffset(), abortOnFailure);
 }
 
 template<typename Config, typename Type>
-IsoDeallocator<Config>& IsoTLS::deallocator(api::IsoHeap<Type>& handle)
+void IsoTLS::deallocateImpl(api::IsoHeap<Type>& handle, void* p)
 {
     unsigned offset = handle.deallocatorOffset();
     IsoTLS* tls = get();
-    RELEASE_BASSERT(offset < tls->m_extent);
-    return *reinterpret_cast<IsoDeallocator<Config>*>(tls->m_data + offset);
+    // Note that this bounds check would be here even if we didn't have to support DebugHeap,
+    // since we don't want unpredictable behavior if offset or m_extent ever got corrupted.
+    if (!tls || offset >= tls->m_extent)
+        deallocateSlow<Config>(handle, p);
+    else
+        tls->deallocateFast<Config>(handle, offset, p);
+}
+
+template<typename Config, typename Type>
+void IsoTLS::deallocateFast(api::IsoHeap<Type>& handle, unsigned offset, void* p)
+{
+    reinterpret_cast<IsoDeallocator<Config>*>(m_data + offset)->deallocate(handle, p);
+}
+
+template<typename Config, typename Type>
+BNO_INLINE void IsoTLS::deallocateSlow(api::IsoHeap<Type>& handle, void* p)
+{
+    for (;;) {
+        switch (s_mallocFallbackState) {
+        case MallocFallbackState::Undecided:
+            determineMallocFallbackState();
+            continue;
+        case MallocFallbackState::FallBackToMalloc:
+            return api::free(p);
+        case MallocFallbackState::DoNotFallBack:
+            break;
+        }
+        break;
+    }
+    
+    // If debug heap is enabled, s_mallocFallbackState becomes MallocFallbackState::FallBackToMalloc.
+    BASSERT(!Environment::get()->isDebugHeapEnabled());
+    
+    RELEASE_BASSERT(handle.isInitialized());
+    
+    IsoTLS* tls = ensureEntries(std::max(handle.allocatorOffset(), handle.deallocatorOffset()));
+    
+    tls->deallocateFast<Config>(handle, handle.deallocatorOffset(), p);
 }
 
 inline IsoTLS* IsoTLS::get()
@@ -106,7 +172,7 @@ template<typename Type>
 void IsoTLS::ensureHeap(api::IsoHeap<Type>& handle)
 {
     if (!handle.isInitialized()) {
-        std::lock_guard<StaticMutex> locker(handle.m_initializationLock);
+        std::lock_guard<Mutex> locker(handle.m_initializationLock);
         if (!handle.isInitialized()) {
             auto* heap = new IsoHeapImpl<typename api::IsoHeap<Type>::Config>();
             std::atomic_thread_fence(std::memory_order_seq_cst);