d668fe0347cf9c1decaad50aee10434ee87f8ded
[WebKit-https.git] / Source / bmalloc / bmalloc / Scavenger.cpp
1 /*
2  * Copyright (C) 2017-2018 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
14  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE INC. OR
17  * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18  * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19  * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20  * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21  * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
24  */
25
26 #include "Scavenger.h"
27
28 #include "AllIsoHeapsInlines.h"
29 #include "AvailableMemory.h"
30 #include "BulkDecommit.h"
31 #include "Environment.h"
32 #include "Heap.h"
33 #if BOS(DARWIN)
34 #import <dispatch/dispatch.h>
35 #import <mach/host_info.h>
36 #import <mach/mach.h>
37 #import <mach/mach_error.h>
38 #endif
39 #include <stdio.h>
40 #include <thread>
41
42 namespace bmalloc {
43
44 static constexpr bool verbose = false;
45
46 struct PrintTime {
47     PrintTime(const char* str) 
48         : string(str)
49     { }
50
51     ~PrintTime()
52     {
53         if (!printed)
54             print();
55     }
56     void print()
57     {
58         if (verbose) {
59             fprintf(stderr, "%s %lfms\n", string, static_cast<double>(std::chrono::duration_cast<std::chrono::microseconds>(std::chrono::steady_clock::now() - start).count()) / 1000);
60             printed = true;
61         }
62     }
63     const char* string;
64     std::chrono::steady_clock::time_point start { std::chrono::steady_clock::now() };
65     bool printed { false };
66 };
67
68 DEFINE_STATIC_PER_PROCESS_STORAGE(Scavenger);
69
70 Scavenger::Scavenger(std::lock_guard<Mutex>&)
71 {
72     BASSERT(!Environment::get()->isDebugHeapEnabled());
73
74 #if BOS(DARWIN)
75     auto queue = dispatch_queue_create("WebKit Malloc Memory Pressure Handler", DISPATCH_QUEUE_SERIAL);
76     m_pressureHandlerDispatchSource = dispatch_source_create(DISPATCH_SOURCE_TYPE_MEMORYPRESSURE, 0, DISPATCH_MEMORYPRESSURE_CRITICAL, queue);
77     dispatch_source_set_event_handler(m_pressureHandlerDispatchSource, ^{
78         scavenge();
79     });
80     dispatch_resume(m_pressureHandlerDispatchSource);
81     dispatch_release(queue);
82 #endif
83     
84     m_thread = std::thread(&threadEntryPoint, this);
85 }
86
87 void Scavenger::run()
88 {
89     std::lock_guard<Mutex> lock(m_mutex);
90     runHoldingLock();
91 }
92
93 void Scavenger::runHoldingLock()
94 {
95     m_state = State::Run;
96     m_condition.notify_all();
97 }
98
99 void Scavenger::runSoon()
100 {
101     std::lock_guard<Mutex> lock(m_mutex);
102     runSoonHoldingLock();
103 }
104
105 void Scavenger::runSoonHoldingLock()
106 {
107     if (willRunSoon())
108         return;
109     m_state = State::RunSoon;
110     m_condition.notify_all();
111 }
112
113 void Scavenger::didStartGrowing()
114 {
115     // We don't really need to lock here, since this is just a heuristic.
116     m_isProbablyGrowing = true;
117 }
118
119 void Scavenger::scheduleIfUnderMemoryPressure(size_t bytes)
120 {
121     std::lock_guard<Mutex> lock(m_mutex);
122     scheduleIfUnderMemoryPressureHoldingLock(bytes);
123 }
124
125 void Scavenger::scheduleIfUnderMemoryPressureHoldingLock(size_t bytes)
126 {
127     m_scavengerBytes += bytes;
128     if (m_scavengerBytes < scavengerBytesPerMemoryPressureCheck)
129         return;
130
131     m_scavengerBytes = 0;
132
133     if (willRun())
134         return;
135
136     if (!isUnderMemoryPressure())
137         return;
138
139     m_isProbablyGrowing = false;
140     runHoldingLock();
141 }
142
143 void Scavenger::schedule(size_t bytes)
144 {
145     std::lock_guard<Mutex> lock(m_mutex);
146     scheduleIfUnderMemoryPressureHoldingLock(bytes);
147     
148     if (willRunSoon())
149         return;
150     
151     m_isProbablyGrowing = false;
152     runSoonHoldingLock();
153 }
154
155 inline void dumpStats()
156 {
157     auto dump = [] (auto* string, auto size) {
158         fprintf(stderr, "%s %zuMB\n", string, static_cast<size_t>(size) / 1024 / 1024);
159     };
160
161 #if BOS(DARWIN)
162     task_vm_info_data_t vmInfo;
163     mach_msg_type_number_t vmSize = TASK_VM_INFO_COUNT;
164     if (KERN_SUCCESS == task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)(&vmInfo), &vmSize)) {
165         dump("phys_footprint", vmInfo.phys_footprint);
166         dump("internal+compressed", vmInfo.internal + vmInfo.compressed);
167     }
168 #endif
169
170     dump("bmalloc-freeable", Scavenger::get()->freeableMemory());
171     dump("bmalloc-footprint", Scavenger::get()->footprint());
172 }
173
174 std::chrono::milliseconds Scavenger::timeSinceLastFullScavenge()
175 {
176     std::unique_lock<Mutex> lock(m_mutex);
177     return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - m_lastFullScavengeTime);
178 }
179
180 std::chrono::milliseconds Scavenger::timeSinceLastPartialScavenge()
181 {
182     std::unique_lock<Mutex> lock(m_mutex);
183     return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now() - m_lastPartialScavengeTime);
184 }
185
186 void Scavenger::enableMiniMode()
187 {
188     m_isInMiniMode = true; // We just store to this racily. The scavenger thread will eventually pick up the right value.
189     if (m_state == State::RunSoon)
190         run();
191 }
192
193 void Scavenger::scavenge()
194 {
195     std::unique_lock<Mutex> lock(m_scavengingMutex);
196
197     if (verbose) {
198         fprintf(stderr, "--------------------------------\n");
199         fprintf(stderr, "--before scavenging--\n");
200         dumpStats();
201     }
202
203     {
204         BulkDecommit decommitter;
205
206         {
207             PrintTime printTime("\nfull scavenge under lock time");
208             std::lock_guard<Mutex> lock(Heap::mutex());
209             for (unsigned i = numHeaps; i--;) {
210                 if (!isActiveHeapKind(static_cast<HeapKind>(i)))
211                     continue;
212                 PerProcess<PerHeapKind<Heap>>::get()->at(i).scavenge(lock, decommitter);
213             }
214             decommitter.processEager();
215         }
216
217         {
218             PrintTime printTime("full scavenge lazy decommit time");
219             decommitter.processLazy();
220         }
221
222         {
223             PrintTime printTime("full scavenge mark all as eligible time");
224             std::lock_guard<Mutex> lock(Heap::mutex());
225             for (unsigned i = numHeaps; i--;) {
226                 if (!isActiveHeapKind(static_cast<HeapKind>(i)))
227                     continue;
228                 PerProcess<PerHeapKind<Heap>>::get()->at(i).markAllLargeAsEligibile(lock);
229             }
230         }
231     }
232
233     {
234         RELEASE_BASSERT(!m_deferredDecommits.size());
235         AllIsoHeaps::get()->forEach(
236             [&] (IsoHeapImplBase& heap) {
237                 heap.scavenge(m_deferredDecommits);
238             });
239         IsoHeapImplBase::finishScavenging(m_deferredDecommits);
240         m_deferredDecommits.shrink(0);
241     }
242
243     if (verbose) {
244         fprintf(stderr, "--after scavenging--\n");
245         dumpStats();
246         fprintf(stderr, "--------------------------------\n");
247     }
248
249     {
250         std::unique_lock<Mutex> lock(m_mutex);
251         m_lastFullScavengeTime = std::chrono::steady_clock::now();
252     }
253 }
254
255 void Scavenger::partialScavenge()
256 {
257     std::unique_lock<Mutex> lock(m_scavengingMutex);
258
259     if (verbose) {
260         fprintf(stderr, "--------------------------------\n");
261         fprintf(stderr, "--before partial scavenging--\n");
262         dumpStats();
263     }
264
265     {
266         BulkDecommit decommitter;
267         {
268             PrintTime printTime("\npartialScavenge under lock time");
269             std::lock_guard<Mutex> lock(Heap::mutex());
270             for (unsigned i = numHeaps; i--;) {
271                 if (!isActiveHeapKind(static_cast<HeapKind>(i)))
272                     continue;
273                 Heap& heap = PerProcess<PerHeapKind<Heap>>::get()->at(i);
274                 size_t freeableMemory = heap.freeableMemory(lock);
275                 if (freeableMemory < 4 * MB)
276                     continue;
277                 heap.scavengeToHighWatermark(lock, decommitter);
278             }
279
280             decommitter.processEager();
281         }
282
283         {
284             PrintTime printTime("partialScavenge lazy decommit time");
285             decommitter.processLazy();
286         }
287
288         {
289             PrintTime printTime("partialScavenge mark all as eligible time");
290             std::lock_guard<Mutex> lock(Heap::mutex());
291             for (unsigned i = numHeaps; i--;) {
292                 if (!isActiveHeapKind(static_cast<HeapKind>(i)))
293                     continue;
294                 Heap& heap = PerProcess<PerHeapKind<Heap>>::get()->at(i);
295                 heap.markAllLargeAsEligibile(lock);
296             }
297         }
298     }
299
300     {
301         RELEASE_BASSERT(!m_deferredDecommits.size());
302         AllIsoHeaps::get()->forEach(
303             [&] (IsoHeapImplBase& heap) {
304                 heap.scavengeToHighWatermark(m_deferredDecommits);
305             });
306         IsoHeapImplBase::finishScavenging(m_deferredDecommits);
307         m_deferredDecommits.shrink(0);
308     }
309
310     if (verbose) {
311         fprintf(stderr, "--after partial scavenging--\n");
312         dumpStats();
313         fprintf(stderr, "--------------------------------\n");
314     }
315
316     {
317         std::unique_lock<Mutex> lock(m_mutex);
318         m_lastPartialScavengeTime = std::chrono::steady_clock::now();
319     }
320 }
321
322 size_t Scavenger::freeableMemory()
323 {
324     size_t result = 0;
325     {
326         std::lock_guard<Mutex> lock(Heap::mutex());
327         for (unsigned i = numHeaps; i--;) {
328             if (!isActiveHeapKind(static_cast<HeapKind>(i)))
329                 continue;
330             result += PerProcess<PerHeapKind<Heap>>::get()->at(i).freeableMemory(lock);
331         }
332     }
333
334     AllIsoHeaps::get()->forEach(
335         [&] (IsoHeapImplBase& heap) {
336             result += heap.freeableMemory();
337         });
338
339     return result;
340 }
341
342 size_t Scavenger::footprint()
343 {
344     RELEASE_BASSERT(!Environment::get()->isDebugHeapEnabled());
345
346     size_t result = 0;
347     for (unsigned i = numHeaps; i--;) {
348         if (!isActiveHeapKind(static_cast<HeapKind>(i)))
349             continue;
350         result += PerProcess<PerHeapKind<Heap>>::get()->at(i).footprint();
351     }
352
353     AllIsoHeaps::get()->forEach(
354         [&] (IsoHeapImplBase& heap) {
355             result += heap.footprint();
356         });
357
358     return result;
359 }
360
361 void Scavenger::threadEntryPoint(Scavenger* scavenger)
362 {
363     scavenger->threadRunLoop();
364 }
365
366 void Scavenger::threadRunLoop()
367 {
368     setSelfQOSClass();
369 #if BOS(DARWIN)
370     setThreadName("JavaScriptCore bmalloc scavenger");
371 #else
372     setThreadName("BMScavenger");
373 #endif
374     
375     // This loop ratchets downward from most active to least active state. While
376     // we ratchet downward, any other thread may reset our state.
377     
378     // We require any state change while we are sleeping to signal to our
379     // condition variable and wake us up.
380     
381     while (true) {
382         if (m_state == State::Sleep) {
383             std::unique_lock<Mutex> lock(m_mutex);
384             m_condition.wait(lock, [&]() { return m_state != State::Sleep; });
385         }
386         
387         if (m_state == State::RunSoon) {
388             std::unique_lock<Mutex> lock(m_mutex);
389             m_condition.wait_for(lock, std::chrono::milliseconds(m_isInMiniMode ? 200 : 2000), [&]() { return m_state != State::RunSoon; });
390         }
391         
392         m_state = State::Sleep;
393         
394         setSelfQOSClass();
395         
396         if (verbose) {
397             fprintf(stderr, "--------------------------------\n");
398             fprintf(stderr, "considering running scavenger\n");
399             dumpStats();
400             fprintf(stderr, "--------------------------------\n");
401         }
402
403         enum class ScavengeMode {
404             None,
405             Partial,
406             Full
407         };
408
409         size_t freeableMemory = this->freeableMemory();
410
411         ScavengeMode scavengeMode = [&] {
412             auto timeSinceLastFullScavenge = this->timeSinceLastFullScavenge();
413             auto timeSinceLastPartialScavenge = this->timeSinceLastPartialScavenge();
414             auto timeSinceLastScavenge = std::min(timeSinceLastPartialScavenge, timeSinceLastFullScavenge);
415
416             if (isUnderMemoryPressure() && freeableMemory > 1 * MB && timeSinceLastScavenge > std::chrono::milliseconds(5))
417                 return ScavengeMode::Full;
418
419             if (!m_isProbablyGrowing) {
420                 if (timeSinceLastFullScavenge < std::chrono::milliseconds(1000) && !m_isInMiniMode)
421                     return ScavengeMode::Partial;
422                 return ScavengeMode::Full;
423             }
424
425             if (m_isInMiniMode) {
426                 if (timeSinceLastFullScavenge < std::chrono::milliseconds(200))
427                     return ScavengeMode::Partial;
428                 return ScavengeMode::Full;
429             }
430
431 #if BCPU(X86_64)
432             auto partialScavengeInterval = std::chrono::milliseconds(12000);
433 #else
434             auto partialScavengeInterval = std::chrono::milliseconds(8000);
435 #endif
436             if (timeSinceLastScavenge < partialScavengeInterval) {
437                 // Rate limit partial scavenges.
438                 return ScavengeMode::None;
439             }
440             if (freeableMemory < 25 * MB)
441                 return ScavengeMode::None;
442             if (5 * freeableMemory < footprint())
443                 return ScavengeMode::None;
444             return ScavengeMode::Partial;
445         }();
446
447         m_isProbablyGrowing = false;
448
449         switch (scavengeMode) {
450         case ScavengeMode::None: {
451             runSoon();
452             break;
453         }
454         case ScavengeMode::Partial: {
455             partialScavenge();
456             runSoon();
457             break;
458         }
459         case ScavengeMode::Full: {
460             scavenge();
461             break;
462         }
463         }
464     }
465 }
466
467 void Scavenger::setThreadName(const char* name)
468 {
469     BUNUSED(name);
470 #if BOS(DARWIN)
471     pthread_setname_np(name);
472 #elif BOS(LINUX)
473     // Truncate the given name since Linux limits the size of the thread name 16 including null terminator.
474     std::array<char, 16> buf;
475     strncpy(buf.data(), name, buf.size() - 1);
476     buf[buf.size() - 1] = '\0';
477     pthread_setname_np(pthread_self(), buf.data());
478 #endif
479 }
480
481 void Scavenger::setSelfQOSClass()
482 {
483 #if BOS(DARWIN)
484     pthread_set_qos_class_self_np(requestedScavengerThreadQOSClass(), 0);
485 #endif
486 }
487
488 } // namespace bmalloc
489