Isolated Heaps caused an increase in reported leaks on the bots
authorfpizlo@apple.com <fpizlo@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 24 Nov 2017 00:47:58 +0000 (00:47 +0000)
committerfpizlo@apple.com <fpizlo@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 24 Nov 2017 00:47:58 +0000 (00:47 +0000)
https://bugs.webkit.org/show_bug.cgi?id=179463

Reviewed by Darin Adler.

This fixes the way isoheaps interact with system tools:

- Opts into the VMHeap API so that the leaks tool can find isoheap memory.

- Opts into the DebugHeap/Environment APIs so that we turn off isoheap allocation if memory
  debugging options are in use.

* bmalloc.xcodeproj/project.pbxproj:
* bmalloc/DebugHeap.h:
* bmalloc/IsoHeap.h:
* bmalloc/IsoPage.cpp: Added.
(bmalloc::IsoPageBase::allocatePageMemory):
* bmalloc/IsoPage.h:
* bmalloc/IsoPageInlines.h:
(bmalloc::IsoPage<Config>::tryCreate):
* bmalloc/IsoTLS.cpp:
(bmalloc::IsoTLS::deallocateSlow):
(bmalloc::IsoTLS::ensureEntries):
(bmalloc::IsoTLS::isUsingDebugHeap):
(bmalloc::IsoTLS::debugMalloc):
* bmalloc/IsoTLS.h:
* bmalloc/IsoTLSInlines.h:
(bmalloc::IsoTLS::allocate):
(bmalloc::IsoTLS::deallocate):
(bmalloc::IsoTLS::allocateImpl):
(bmalloc::IsoTLS::allocateFast):
(bmalloc::IsoTLS::allocateSlow):
(bmalloc::IsoTLS::deallocateImpl):
(bmalloc::IsoTLS::deallocateFast):
(bmalloc::IsoTLS::ensureHeapAndEntries):
(bmalloc::IsoTLS::allocator): Deleted.
(bmalloc::IsoTLS::deallocator): Deleted.
* bmalloc/bmalloc.cpp:
(bmalloc::api::tryLargeMemalignVirtual):
(bmalloc::api::freeLargeVirtual):
(bmalloc::api::scavenge):
(bmalloc::api::isEnabled):
(bmalloc::api::setScavengerThreadQOSClass):
* bmalloc/bmalloc.h:
(bmalloc::api::tryLargeMemalignVirtual): Deleted.
(bmalloc::api::freeLargeVirtual): Deleted.
(bmalloc::api::scavenge): Deleted.
(bmalloc::api::isEnabled): Deleted.
(bmalloc::api::setScavengerThreadQOSClass): Deleted.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@225125 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/bmalloc/CMakeLists.txt
Source/bmalloc/ChangeLog
Source/bmalloc/bmalloc.xcodeproj/project.pbxproj
Source/bmalloc/bmalloc/IsoPage.cpp [new file with mode: 0644]
Source/bmalloc/bmalloc/IsoPage.h
Source/bmalloc/bmalloc/IsoPageInlines.h
Source/bmalloc/bmalloc/IsoTLS.cpp
Source/bmalloc/bmalloc/IsoTLS.h
Source/bmalloc/bmalloc/IsoTLSInlines.h
Source/bmalloc/bmalloc/bmalloc.cpp
Source/bmalloc/bmalloc/bmalloc.h

index 114f9e7..d276d78 100644 (file)
@@ -18,6 +18,7 @@ set(bmalloc_SOURCES
     bmalloc/Heap.cpp
     bmalloc/HeapKind.cpp
     bmalloc/IsoHeapImpl.cpp
+    bmalloc/IsoPage.cpp
     bmalloc/IsoTLS.cpp
     bmalloc/IsoTLSEntry.cpp
     bmalloc/IsoTLSLayout.cpp
index 43e0685..e4866a9 100644 (file)
@@ -1,3 +1,55 @@
+2017-11-16  Filip Pizlo  <fpizlo@apple.com>
+
+        Isolated Heaps caused an increase in reported leaks on the bots
+        https://bugs.webkit.org/show_bug.cgi?id=179463
+
+        Reviewed by Darin Adler.
+        
+        This fixes the way isoheaps interact with system tools:
+        
+        - Opts into the VMHeap API so that the leaks tool can find isoheap memory.
+        
+        - Opts into the DebugHeap/Environment APIs so that we turn off isoheap allocation if memory
+          debugging options are in use.
+
+        * bmalloc.xcodeproj/project.pbxproj:
+        * bmalloc/DebugHeap.h:
+        * bmalloc/IsoHeap.h:
+        * bmalloc/IsoPage.cpp: Added.
+        (bmalloc::IsoPageBase::allocatePageMemory):
+        * bmalloc/IsoPage.h:
+        * bmalloc/IsoPageInlines.h:
+        (bmalloc::IsoPage<Config>::tryCreate):
+        * bmalloc/IsoTLS.cpp:
+        (bmalloc::IsoTLS::deallocateSlow):
+        (bmalloc::IsoTLS::ensureEntries):
+        (bmalloc::IsoTLS::isUsingDebugHeap):
+        (bmalloc::IsoTLS::debugMalloc):
+        * bmalloc/IsoTLS.h:
+        * bmalloc/IsoTLSInlines.h:
+        (bmalloc::IsoTLS::allocate):
+        (bmalloc::IsoTLS::deallocate):
+        (bmalloc::IsoTLS::allocateImpl):
+        (bmalloc::IsoTLS::allocateFast):
+        (bmalloc::IsoTLS::allocateSlow):
+        (bmalloc::IsoTLS::deallocateImpl):
+        (bmalloc::IsoTLS::deallocateFast):
+        (bmalloc::IsoTLS::ensureHeapAndEntries):
+        (bmalloc::IsoTLS::allocator): Deleted.
+        (bmalloc::IsoTLS::deallocator): Deleted.
+        * bmalloc/bmalloc.cpp:
+        (bmalloc::api::tryLargeMemalignVirtual):
+        (bmalloc::api::freeLargeVirtual):
+        (bmalloc::api::scavenge):
+        (bmalloc::api::isEnabled):
+        (bmalloc::api::setScavengerThreadQOSClass):
+        * bmalloc/bmalloc.h:
+        (bmalloc::api::tryLargeMemalignVirtual): Deleted.
+        (bmalloc::api::freeLargeVirtual): Deleted.
+        (bmalloc::api::scavenge): Deleted.
+        (bmalloc::api::isEnabled): Deleted.
+        (bmalloc::api::setScavengerThreadQOSClass): Deleted.
+
 2017-11-14  Saam Barati  <sbarati@apple.com>
 
         Make the gigacage runway 32GB
index 014b760..879c256 100644 (file)
@@ -23,6 +23,7 @@
 /* Begin PBXBuildFile section */
                0F3DA0141F267AB800342C08 /* AllocationKind.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F3DA0131F267AB800342C08 /* AllocationKind.h */; settings = {ATTRIBUTES = (Private, ); }; };
                0F5167741FAD685C008236A8 /* bmalloc.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F5167731FAD6852008236A8 /* bmalloc.cpp */; };
+               0F5549EF1FB54704007FF75A /* IsoPage.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 0F5549EE1FB54701007FF75A /* IsoPage.cpp */; };
                0F5BF1471F22A8B10029D91D /* HeapKind.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5BF1461F22A8B10029D91D /* HeapKind.h */; settings = {ATTRIBUTES = (Private, ); }; };
                0F5BF1491F22A8D80029D91D /* PerHeapKind.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5BF1481F22A8D80029D91D /* PerHeapKind.h */; settings = {ATTRIBUTES = (Private, ); }; };
                0F5BF14D1F22B0C30029D91D /* Gigacage.h in Headers */ = {isa = PBXBuildFile; fileRef = 0F5BF14C1F22B0C30029D91D /* Gigacage.h */; settings = {ATTRIBUTES = (Private, ); }; };
 /* Begin PBXFileReference section */
                0F3DA0131F267AB800342C08 /* AllocationKind.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = AllocationKind.h; path = bmalloc/AllocationKind.h; sourceTree = "<group>"; };
                0F5167731FAD6852008236A8 /* bmalloc.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = bmalloc.cpp; path = bmalloc/bmalloc.cpp; sourceTree = "<group>"; };
+               0F5549EE1FB54701007FF75A /* IsoPage.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = IsoPage.cpp; path = bmalloc/IsoPage.cpp; sourceTree = "<group>"; };
                0F5BF1461F22A8B10029D91D /* HeapKind.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = HeapKind.h; path = bmalloc/HeapKind.h; sourceTree = "<group>"; };
                0F5BF1481F22A8D80029D91D /* PerHeapKind.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = PerHeapKind.h; path = bmalloc/PerHeapKind.h; sourceTree = "<group>"; };
                0F5BF14C1F22B0C30029D91D /* Gigacage.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = Gigacage.h; path = bmalloc/Gigacage.h; sourceTree = "<group>"; };
                                0F7EB81E1F9541B000F1ABCB /* IsoHeapImpl.h */,
                                0F7EB7FD1F9541AD00F1ABCB /* IsoHeapImplInlines.h */,
                                0F7EB8091F9541AD00F1ABCB /* IsoHeapInlines.h */,
+                               0F5549EE1FB54701007FF75A /* IsoPage.cpp */,
                                0F7EB8071F9541AD00F1ABCB /* IsoPage.h */,
                                0F7EB80A1F9541AE00F1ABCB /* IsoPageInlines.h */,
                                0F7EB8041F9541AD00F1ABCB /* IsoPageTrigger.h */,
                                0F7EB8361F9541B000F1ABCB /* IsoTLS.cpp in Sources */,
                                0FD557331F7EDB7B00B1F0A3 /* HeapKind.cpp in Sources */,
                                0F7EB83B1F9541B000F1ABCB /* IsoHeapImpl.cpp in Sources */,
+                               0F5549EF1FB54704007FF75A /* IsoPage.cpp in Sources */,
                                14F271C318EA3978008C152F /* Allocator.cpp in Sources */,
                                6599C5CC1EC3F15900A2F7BB /* AvailableMemory.cpp in Sources */,
                                14F271C418EA397B008C152F /* Cache.cpp in Sources */,
diff --git a/Source/bmalloc/bmalloc/IsoPage.cpp b/Source/bmalloc/bmalloc/IsoPage.cpp
new file mode 100644 (file)
index 0000000..2b3ff3e
--- /dev/null
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2017 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
+ */
+
+#include "IsoPage.h"
+
+#include "PerProcess.h"
+#include "VMHeap.h"
+
+namespace bmalloc {
+
+void* IsoPageBase::allocatePageMemory()
+{
+    return PerProcess<VMHeap>::get()->tryAllocateLargeChunk(pageSize, pageSize, AllocationKind::Physical).begin();
+}
+
+} // namespace bmalloc
index 5fee57a..b909f04 100644 (file)
@@ -38,6 +38,9 @@ template<typename Config> class IsoHeapImpl;
 class IsoPageBase {
 public:    
     static constexpr size_t pageSize = 16384;
+    
+protected:
+    BEXPORT static void* allocatePageMemory();
 };
 
 template<typename Config>
index 628c16b..0c47864 100644 (file)
@@ -35,7 +35,7 @@ namespace bmalloc {
 template<typename Config>
 IsoPage<Config>* IsoPage<Config>::tryCreate(IsoDirectoryBase<Config>& directory, unsigned index)
 {
-    void* memory = tryVMAllocate(pageSize, pageSize);
+    void* memory = allocatePageMemory();
     if (!memory)
         return nullptr;
     
index f3de6cd..041f4ba 100644 (file)
@@ -25,6 +25,8 @@
 
 #include "IsoTLS.h"
 
+#include "DebugHeap.h"
+#include "Environment.h"
 #include "IsoTLSEntryInlines.h"
 #include "IsoTLSInlines.h"
 #include "IsoTLSLayout.h"
@@ -50,6 +52,15 @@ IsoTLS::IsoTLS()
 {
 }
 
+void IsoTLS::deallocateSlow(void* p)
+{
+    // If we go down this path and we aren't in debug heap mode, then this means we have some corruption.
+    // Think of this as really being an assertion about offset < tls->m_extent.
+    RELEASE_BASSERT(PerProcess<Environment>::get()->isDebugHeapEnabled());
+    
+    PerProcess<DebugHeap>::get()->free(p);
+}
+
 IsoTLS* IsoTLS::ensureEntries(unsigned offset)
 {
     RELEASE_BASSERT(!get() || offset >= get()->m_extent);
@@ -76,16 +87,19 @@ IsoTLS* IsoTLS::ensureEntries(unsigned offset)
     IsoTLSEntry* startEntry = oldLastEntry ? oldLastEntry : layout.head();
     
     IsoTLSEntry* targetEntry = startEntry;
-    for (;;) {
+    size_t requiredCapacity = 0;
+    if (startEntry) {
+        for (;;) {
+            RELEASE_BASSERT(targetEntry);
+            RELEASE_BASSERT(targetEntry->offset() <= offset);
+            if (targetEntry->offset() == offset)
+                break;
+            targetEntry = targetEntry->m_next;
+        }
         RELEASE_BASSERT(targetEntry);
-        RELEASE_BASSERT(targetEntry->offset() <= offset);
-        if (targetEntry->offset() == offset)
-            break;
-        targetEntry = targetEntry->m_next;
+        requiredCapacity = targetEntry->extent();
     }
-    RELEASE_BASSERT(targetEntry);
-
-    size_t requiredCapacity = targetEntry->extent();
+    
     if (!tls || requiredCapacity > tls->m_capacity) {
         size_t requiredSize = sizeForCapacity(requiredCapacity);
         size_t goodSize = roundUpToMultipleOf(vmPageSize(), requiredSize);
@@ -110,14 +124,16 @@ IsoTLS* IsoTLS::ensureEntries(unsigned offset)
         set(tls);
     }
     
-    startEntry->walkUpToInclusive(
-        targetEntry,
-        [&] (IsoTLSEntry* entry) {
-            entry->construct(tls->m_data + entry->offset());
-        });
-    
-    tls->m_lastEntry = targetEntry;
-    tls->m_extent = targetEntry->extent();
+    if (startEntry) {
+        startEntry->walkUpToInclusive(
+            targetEntry,
+            [&] (IsoTLSEntry* entry) {
+                entry->construct(tls->m_data + entry->offset());
+            });
+        
+        tls->m_lastEntry = targetEntry;
+        tls->m_extent = targetEntry->extent();
+    }
     
     return tls;
 }
@@ -159,5 +175,18 @@ void IsoTLS::forEachEntry(const Func& func)
         });
 }
 
+bool IsoTLS::isUsingDebugHeap()
+{
+    return PerProcess<Environment>::get()->isDebugHeapEnabled();
+}
+
+auto IsoTLS::debugMalloc(size_t size) -> DebugMallocResult
+{
+    DebugMallocResult result;
+    if ((result.usingDebugHeap = isUsingDebugHeap()))
+        result.ptr = PerProcess<DebugHeap>::get()->malloc(size);
+    return result;
+}
+
 } // namespace bmalloc
 
index a5f24b1..99a4501 100644 (file)
@@ -58,10 +58,21 @@ private:
     IsoTLS();
     
     template<typename Config, typename Type>
-    static IsoAllocator<Config>& allocator(api::IsoHeap<Type>&);
+    static void* allocateImpl(api::IsoHeap<Type>&, bool abortOnFailure);
+    
+    template<typename Config>
+    void* allocateFast(unsigned offset, bool abortOnFailure);
+    
+    template<typename Config, typename Type>
+    static void* allocateSlow(api::IsoHeap<Type>&, bool abortOnFailure);
     
     template<typename Config, typename Type>
-    static IsoDeallocator<Config>& deallocator(api::IsoHeap<Type>&);
+    static void deallocateImpl(api::IsoHeap<Type>&, void* p);
+    
+    template<typename Config>
+    void deallocateFast(unsigned offset, void* p);
+    
+    BEXPORT static void deallocateSlow(void* p);
     
     static IsoTLS* get();
     static void set(IsoTLS*);
@@ -81,6 +92,15 @@ private:
     template<typename Func>
     void forEachEntry(const Func&);
     
+    BEXPORT static bool isUsingDebugHeap();
+    
+    struct DebugMallocResult {
+        void* ptr { nullptr };
+        bool usingDebugHeap { false };
+    };
+    
+    BEXPORT static DebugMallocResult debugMalloc(size_t);
+    
     IsoTLSEntry* m_lastEntry { nullptr };
     unsigned m_extent { 0 };
     unsigned m_capacity { 0 };
index f43fddc..6dbcb61 100644 (file)
@@ -33,7 +33,7 @@ namespace bmalloc {
 template<typename Type>
 void* IsoTLS::allocate(api::IsoHeap<Type>& handle, bool abortOnFailure)
 {
-    return allocator<typename api::IsoHeap<Type>::Config>(handle).allocate(abortOnFailure);
+    return allocateImpl<typename api::IsoHeap<Type>::Config>(handle, abortOnFailure);
 }
 
 template<typename Type>
@@ -41,7 +41,7 @@ void IsoTLS::deallocate(api::IsoHeap<Type>& handle, void* p)
 {
     if (!p)
         return;
-    deallocator<typename api::IsoHeap<Type>::Config>(handle).deallocate(p);
+    deallocateImpl<typename api::IsoHeap<Type>::Config>(handle, p);
 }
 
 template<typename Type>
@@ -62,24 +62,51 @@ void IsoTLS::scavenge(api::IsoHeap<Type>& handle)
 }
 
 template<typename Config, typename Type>
-IsoAllocator<Config>& IsoTLS::allocator(api::IsoHeap<Type>& handle)
+void* IsoTLS::allocateImpl(api::IsoHeap<Type>& handle, bool abortOnFailure)
 {
     unsigned offset = handle.allocatorOffset();
     IsoTLS* tls = get();
-    if (!tls || offset >= tls->m_extent) {
-        tls = ensureHeapAndEntries(handle);
-        offset = handle.allocatorOffset();
-    }
-    return *reinterpret_cast<IsoAllocator<Config>*>(tls->m_data + offset);
+    if (!tls || offset >= tls->m_extent)
+        return allocateSlow<typename api::IsoHeap<Type>::Config>(handle, abortOnFailure);
+    return tls->allocateFast<Config>(offset, abortOnFailure);
+}
+
+template<typename Config>
+void* IsoTLS::allocateFast(unsigned offset, bool abortOnFailure)
+{
+    return reinterpret_cast<IsoAllocator<Config>*>(m_data + offset)->allocate(abortOnFailure);
 }
 
 template<typename Config, typename Type>
-IsoDeallocator<Config>& IsoTLS::deallocator(api::IsoHeap<Type>& handle)
+BNO_INLINE void* IsoTLS::allocateSlow(api::IsoHeap<Type>& handle, bool abortOnFailure)
+{
+    IsoTLS* tls = ensureHeapAndEntries(handle);
+    
+    auto debugMallocResult = debugMalloc(Config::objectSize);
+    if (debugMallocResult.usingDebugHeap)
+        return debugMallocResult.ptr;
+    
+    unsigned offset = handle.allocatorOffset();
+    return tls->allocateFast<Config>(offset, abortOnFailure);
+}
+
+template<typename Config, typename Type>
+void IsoTLS::deallocateImpl(api::IsoHeap<Type>& handle, void* p)
 {
     unsigned offset = handle.deallocatorOffset();
     IsoTLS* tls = get();
-    RELEASE_BASSERT(offset < tls->m_extent);
-    return *reinterpret_cast<IsoDeallocator<Config>*>(tls->m_data + offset);
+    // Note that this bounds check would be here even if we didn't have to support DebugHeap,
+    // since we don't want unpredictable behavior if offset or m_extent ever got corrupted.
+    if (offset >= tls->m_extent)
+        deallocateSlow(p);
+    else
+        tls->deallocateFast<Config>(offset, p);
+}
+
+template<typename Config>
+void IsoTLS::deallocateFast(unsigned offset, void* p)
+{
+    reinterpret_cast<IsoDeallocator<Config>*>(m_data + offset)->deallocate(p);
 }
 
 inline IsoTLS* IsoTLS::get()
@@ -124,8 +151,16 @@ BNO_INLINE IsoTLS* IsoTLS::ensureHeapAndEntries(api::IsoHeap<Type>& handle)
         !get()
         || handle.allocatorOffset() >= get()->m_extent
         || handle.deallocatorOffset() >= get()->m_extent);
-    ensureHeap(handle);
-    return ensureEntries(std::max(handle.allocatorOffset(), handle.deallocatorOffset()));
+    unsigned offset;
+    if (isUsingDebugHeap()) {
+        if (IsoTLS* result = get())
+            return result;
+        offset = 0;
+    } else {
+        ensureHeap(handle);
+        offset = std::max(handle.allocatorOffset(), handle.deallocatorOffset());
+    }
+    return ensureEntries(offset);
 }
 
 } // namespace bmalloc
index 1401b82..bde8775 100644 (file)
@@ -25,6 +25,8 @@
 
 #include "bmalloc.h"
 
+#include "PerProcess.h"
+
 namespace bmalloc { namespace api {
 
 void* mallocOutOfLine(size_t size, HeapKind kind)
@@ -37,5 +39,43 @@ void freeOutOfLine(void* object, HeapKind kind)
     free(object, kind);
 }
 
+void* tryLargeMemalignVirtual(size_t alignment, size_t size, HeapKind kind)
+{
+    kind = mapToActiveHeapKind(kind);
+    Heap& heap = PerProcess<PerHeapKind<Heap>>::get()->at(kind);
+    std::lock_guard<StaticMutex> lock(Heap::mutex());
+    return heap.tryAllocateLarge(lock, alignment, size, AllocationKind::Virtual);
+}
+
+void freeLargeVirtual(void* object, HeapKind kind)
+{
+    kind = mapToActiveHeapKind(kind);
+    Heap& heap = PerProcess<PerHeapKind<Heap>>::get()->at(kind);
+    std::lock_guard<StaticMutex> lock(Heap::mutex());
+    heap.deallocateLarge(lock, object, AllocationKind::Virtual);
+}
+
+void scavenge()
+{
+    scavengeThisThread();
+
+    PerProcess<Scavenger>::get()->scavenge();
+}
+
+bool isEnabled(HeapKind kind)
+{
+    kind = mapToActiveHeapKind(kind);
+    std::unique_lock<StaticMutex> lock(Heap::mutex());
+    return !PerProcess<PerHeapKind<Heap>>::getFastCase()->at(kind).debugHeap();
+}
+
+#if BOS(DARWIN)
+void setScavengerThreadQOSClass(qos_class_t overrideClass)
+{
+    std::unique_lock<StaticMutex> lock(Heap::mutex());
+    PerProcess<Scavenger>::get()->setScavengerThreadQOSClass(overrideClass);
+}
+#endif
+
 } } // namespace bmalloc::api
 
index ce60ecc..a23389b 100644 (file)
@@ -31,7 +31,6 @@
 #include "Heap.h"
 #include "IsoTLS.h"
 #include "PerHeapKind.h"
-#include "PerProcess.h"
 #include "Scavenger.h"
 #include "StaticMutex.h"
 
@@ -71,13 +70,7 @@ inline void* realloc(void* object, size_t newSize, HeapKind kind = HeapKind::Pri
 }
 
 // Returns null for failure
-inline void* tryLargeMemalignVirtual(size_t alignment, size_t size, HeapKind kind = HeapKind::Primary)
-{
-    kind = mapToActiveHeapKind(kind);
-    Heap& heap = PerProcess<PerHeapKind<Heap>>::get()->at(kind);
-    std::lock_guard<StaticMutex> lock(Heap::mutex());
-    return heap.tryAllocateLarge(lock, alignment, size, AllocationKind::Virtual);
-}
+BEXPORT void* tryLargeMemalignVirtual(size_t alignment, size_t size, HeapKind kind = HeapKind::Primary);
 
 inline void free(void* object, HeapKind kind = HeapKind::Primary)
 {
@@ -86,13 +79,7 @@ inline void free(void* object, HeapKind kind = HeapKind::Primary)
 
 BEXPORT void freeOutOfLine(void* object, HeapKind kind = HeapKind::Primary);
 
-inline void freeLargeVirtual(void* object, HeapKind kind = HeapKind::Primary)
-{
-    kind = mapToActiveHeapKind(kind);
-    Heap& heap = PerProcess<PerHeapKind<Heap>>::get()->at(kind);
-    std::lock_guard<StaticMutex> lock(Heap::mutex());
-    heap.deallocateLarge(lock, object, AllocationKind::Virtual);
-}
+BEXPORT void freeLargeVirtual(void* object, HeapKind kind = HeapKind::Primary);
 
 inline void scavengeThisThread()
 {
@@ -101,19 +88,9 @@ inline void scavengeThisThread()
     IsoTLS::scavenge();
 }
 
-inline void scavenge()
-{
-    scavengeThisThread();
+BEXPORT void scavenge();
 
-    PerProcess<Scavenger>::get()->scavenge();
-}
-
-inline bool isEnabled(HeapKind kind = HeapKind::Primary)
-{
-    kind = mapToActiveHeapKind(kind);
-    std::unique_lock<StaticMutex> lock(Heap::mutex());
-    return !PerProcess<PerHeapKind<Heap>>::getFastCase()->at(kind).debugHeap();
-}
+BEXPORT bool isEnabled(HeapKind kind = HeapKind::Primary);
     
 inline size_t availableMemory()
 {
@@ -133,11 +110,7 @@ inline double percentAvailableMemoryInUse()
 #endif
 
 #if BOS(DARWIN)
-inline void setScavengerThreadQOSClass(qos_class_t overrideClass)
-{
-    std::unique_lock<StaticMutex> lock(Heap::mutex());
-    PerProcess<Scavenger>::get()->setScavengerThreadQOSClass(overrideClass);
-}
+BEXPORT void setScavengerThreadQOSClass(qos_class_t overrideClass);
 #endif
 
 } // namespace api