6a7ca731d0c4d7a6631fdc085a68a4192687ae10
[WebKit-https.git] / Source / bmalloc / bmalloc / Scavenger.cpp
1 /*
2  * Copyright (C) 2017-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "Scavenger.h"
27
28 #include "AllIsoHeapsInlines.h"
29 #include "AvailableMemory.h"
30 #include "BulkDecommit.h"
31 #include "Environment.h"
32 #include "Heap.h"
33 #if BOS(DARWIN)
34 #import <dispatch/dispatch.h>
35 #import <mach/host_info.h>
36 #import <mach/mach.h>
37 #import <mach/mach_error.h>
38 #endif
39 #include <stdio.h>
40 #include <thread>
41
42 namespace bmalloc {
43
44 static constexpr bool verbose = false;
45
46 struct PrintTime {
47     PrintTime(const char* str) 
48         : string(str)
49     { }
50
51     ~PrintTime()
52     {
53         if (!printed)
54             print();
55     }
56     void print()
57     {
58         if (verbose) {
59             fprintf(stderr, "%s %lfms\n", string, static_cast<double>(std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - start).count()) / 1000);
60             printed = true;
61         }
62     }
63     const char* string;
64     std::chrono::steady_clock::time_point start { std::chrono::steady_clock::now() };
65     bool printed { false };
66 };
67
68 DEFINE_STATIC_PER_PROCESS_STORAGE(Scavenger);
69
70 Scavenger::Scavenger(std::lock_guard<Mutex>&)
71 {
72     BASSERT(!Environment::get()->isDebugHeapEnabled());
73
74 #if BOS(DARWIN)
75     auto queue = dispatch_queue_create("WebKit Malloc Memory Pressure Handler", DISPATCH_QUEUE_SERIAL);
76     m_pressureHandlerDispatchSource = dispatch_source_create(DISPATCH_SOURCE_TYPE_MEMORYPRESSURE, 0, DISPATCH_MEMORYPRESSURE_CRITICAL, queue);
77     dispatch_source_set_event_handler(m_pressureHandlerDispatchSource, ^{
78         scavenge();
79     });
80     dispatch_resume(m_pressureHandlerDispatchSource);
81     dispatch_release(queue);
82 #endif
83     m_waitTime = std::chrono::milliseconds(10);
84
85     m_thread = std::thread(&threadEntryPoint, this);
86 }
87
88 void Scavenger::run()
89 {
90     std::lock_guard<Mutex> lock(m_mutex);
91     runHoldingLock();
92 }
93
94 void Scavenger::runHoldingLock()
95 {
96     m_state = State::Run;
97     m_condition.notify_all();
98 }
99
100 void Scavenger::runSoon()
101 {
102     std::lock_guard<Mutex> lock(m_mutex);
103     runSoonHoldingLock();
104 }
105
106 void Scavenger::runSoonHoldingLock()
107 {
108     if (willRunSoon())
109         return;
110     m_state = State::RunSoon;
111     m_condition.notify_all();
112 }
113
114 void Scavenger::didStartGrowing()
115 {
116     // We don't really need to lock here, since this is just a heuristic.
117     m_isProbablyGrowing = true;
118 }
119
120 void Scavenger::scheduleIfUnderMemoryPressure(size_t bytes)
121 {
122     std::lock_guard<Mutex> lock(m_mutex);
123     scheduleIfUnderMemoryPressureHoldingLock(bytes);
124 }
125
126 void Scavenger::scheduleIfUnderMemoryPressureHoldingLock(size_t bytes)
127 {
128     m_scavengerBytes += bytes;
129     if (m_scavengerBytes < scavengerBytesPerMemoryPressureCheck)
130         return;
131
132     m_scavengerBytes = 0;
133
134     if (willRun())
135         return;
136
137     if (!isUnderMemoryPressure())
138         return;
139
140     m_isProbablyGrowing = false;
141     runHoldingLock();
142 }
143
144 void Scavenger::schedule(size_t bytes)
145 {
146     std::lock_guard<Mutex> lock(m_mutex);
147     scheduleIfUnderMemoryPressureHoldingLock(bytes);
148     
149     if (willRunSoon())
150         return;
151     
152     m_isProbablyGrowing = false;
153     runSoonHoldingLock();
154 }
155
156 inline void dumpStats()
157 {
158     auto dump = [] (auto* string, auto size) {
159         fprintf(stderr, "%s %zuMB\n", string, static_cast<size_t>(size) / 1024 / 1024);
160     };
161
162 #if BOS(DARWIN)
163     task_vm_info_data_t vmInfo;
164     mach_msg_type_number_t vmSize = TASK_VM_INFO_COUNT;
165     if (KERN_SUCCESS == task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)(&vmInfo), &vmSize)) {
166         dump("phys_footprint", vmInfo.phys_footprint);
167         dump("internal+compressed", vmInfo.internal + vmInfo.compressed);
168     }
169 #endif
170
171     dump("bmalloc-freeable", Scavenger::get()->freeableMemory());
172     dump("bmalloc-footprint", Scavenger::get()->footprint());
173 }
174
175 std::chrono::milliseconds Scavenger::timeSinceLastFullScavenge()
176 {
177     std::unique_lock<Mutex> lock(m_mutex);
178     return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - m_lastFullScavengeTime);
179 }
180
181 void Scavenger::enableMiniMode()
182 {
183     m_isInMiniMode = true; // We just store to this racily. The scavenger thread will eventually pick up the right value.
184     if (m_state == State::RunSoon)
185         run();
186 }
187
188 void Scavenger::scavenge()
189 {
190     std::unique_lock<Mutex> lock(m_scavengingMutex);
191
192     if (verbose) {
193         fprintf(stderr, "--------------------------------\n");
194         fprintf(stderr, "--before scavenging--\n");
195         dumpStats();
196     }
197
198     {
199         BulkDecommit decommitter;
200
201         {
202             PrintTime printTime("\nfull scavenge under lock time");
203             size_t deferredDecommits = 0;
204             std::lock_guard<Mutex> lock(Heap::mutex());
205             for (unsigned i = numHeaps; i--;) {
206                 if (!isActiveHeapKind(static_cast<HeapKind>(i)))
207                     continue;
208                 PerProcess<PerHeapKind<Heap>>::get()->at(i).scavenge(lock, decommitter, deferredDecommits);
209             }
210             decommitter.processEager();
211
212             if (deferredDecommits)
213                 m_state = State::RunSoon;
214         }
215
216         {
217             PrintTime printTime("full scavenge lazy decommit time");
218             decommitter.processLazy();
219         }
220
221         {
222             PrintTime printTime("full scavenge mark all as eligible time");
223             std::lock_guard<Mutex> lock(Heap::mutex());
224             for (unsigned i = numHeaps; i--;) {
225                 if (!isActiveHeapKind(static_cast<HeapKind>(i)))
226                     continue;
227                 PerProcess<PerHeapKind<Heap>>::get()->at(i).markAllLargeAsEligibile(lock);
228             }
229         }
230     }
231
232     {
233         RELEASE_BASSERT(!m_deferredDecommits.size());
234         AllIsoHeaps::get()->forEach(
235             [&] (IsoHeapImplBase& heap) {
236                 heap.scavenge(m_deferredDecommits);
237             });
238         IsoHeapImplBase::finishScavenging(m_deferredDecommits);
239         m_deferredDecommits.shrink(0);
240     }
241
242     if (verbose) {
243         fprintf(stderr, "--after scavenging--\n");
244         dumpStats();
245         fprintf(stderr, "--------------------------------\n");
246     }
247
248     {
249         std::unique_lock<Mutex> lock(m_mutex);
250         m_lastFullScavengeTime = std::chrono::steady_clock::now();
251     }
252 }
253
254 size_t Scavenger::freeableMemory()
255 {
256     size_t result = 0;
257     {
258         std::lock_guard<Mutex> lock(Heap::mutex());
259         for (unsigned i = numHeaps; i--;) {
260             if (!isActiveHeapKind(static_cast<HeapKind>(i)))
261                 continue;
262             result += PerProcess<PerHeapKind<Heap>>::get()->at(i).freeableMemory(lock);
263         }
264     }
265
266     AllIsoHeaps::get()->forEach(
267         [&] (IsoHeapImplBase& heap) {
268             result += heap.freeableMemory();
269         });
270
271     return result;
272 }
273
274 size_t Scavenger::footprint()
275 {
276     RELEASE_BASSERT(!Environment::get()->isDebugHeapEnabled());
277
278     size_t result = 0;
279     for (unsigned i = numHeaps; i--;) {
280         if (!isActiveHeapKind(static_cast<HeapKind>(i)))
281             continue;
282         result += PerProcess<PerHeapKind<Heap>>::get()->at(i).footprint();
283     }
284
285     AllIsoHeaps::get()->forEach(
286         [&] (IsoHeapImplBase& heap) {
287             result += heap.footprint();
288         });
289
290     return result;
291 }
292
293 void Scavenger::threadEntryPoint(Scavenger* scavenger)
294 {
295     scavenger->threadRunLoop();
296 }
297
298 void Scavenger::threadRunLoop()
299 {
300     setSelfQOSClass();
301 #if BOS(DARWIN)
302     setThreadName("JavaScriptCore bmalloc scavenger");
303 #else
304     setThreadName("BMScavenger");
305 #endif
306     
307     // This loop ratchets downward from most active to least active state. While
308     // we ratchet downward, any other thread may reset our state.
309     
310     // We require any state change while we are sleeping to signal to our
311     // condition variable and wake us up.
312     
313     while (true) {
314         if (m_state == State::Sleep) {
315             std::unique_lock<Mutex> lock(m_mutex);
316             m_condition.wait(lock, [&]() { return m_state != State::Sleep; });
317         }
318         
319         if (m_state == State::RunSoon) {
320             std::unique_lock<Mutex> lock(m_mutex);
321             m_condition.wait_for(lock, m_waitTime, [&]() { return m_state != State::RunSoon; });
322         }
323         
324         m_state = State::Sleep;
325         
326         setSelfQOSClass();
327         
328         if (verbose) {
329             fprintf(stderr, "--------------------------------\n");
330             fprintf(stderr, "considering running scavenger\n");
331             dumpStats();
332             fprintf(stderr, "--------------------------------\n");
333         }
334
335         std::chrono::steady_clock::time_point start { std::chrono::steady_clock::now() };
336         
337         scavenge();
338
339         auto timeSpentScavenging = std::chrono::steady_clock::now() - start;
340
341         if (verbose) {
342             fprintf(stderr, "time spent scavenging %lfms\n",
343                 static_cast<double>(std::chrono::duration_cast<std::chrono::microseconds>(timeSpentScavenging).count()) / 1000);
344         }
345
346         std::chrono::milliseconds newWaitTime;
347
348         if (m_isInMiniMode) {
349             timeSpentScavenging *= 50;
350             newWaitTime = std::chrono::duration_cast<std::chrono::milliseconds>(timeSpentScavenging);
351             newWaitTime = std::min(std::max(newWaitTime, std::chrono::milliseconds(25)), std::chrono::milliseconds(500));
352         } else {
353             timeSpentScavenging *= 150;
354             newWaitTime = std::chrono::duration_cast<std::chrono::milliseconds>(timeSpentScavenging);
355             m_waitTime = std::min(std::max(newWaitTime, std::chrono::milliseconds(100)), std::chrono::milliseconds(10000));
356         }
357
358         if (verbose)
359             fprintf(stderr, "new wait time %lldms\n", m_waitTime.count());
360     }
361 }
362
363 void Scavenger::setThreadName(const char* name)
364 {
365     BUNUSED(name);
366 #if BOS(DARWIN)
367     pthread_setname_np(name);
368 #elif BOS(LINUX)
369     // Truncate the given name since Linux limits the size of the thread name 16 including null terminator.
370     std::array<char, 16> buf;
371     strncpy(buf.data(), name, buf.size() - 1);
372     buf[buf.size() - 1] = '\0';
373     pthread_setname_np(pthread_self(), buf.data());
374 #endif
375 }
376
377 void Scavenger::setSelfQOSClass()
378 {
379 #if BOS(DARWIN)
380     pthread_set_qos_class_self_np(requestedScavengerThreadQOSClass(), 0);
381 #endif
382 }
383
384 } // namespace bmalloc
385