Some MallocBench refinements
[WebKit-https.git] / PerformanceTests / MallocBench / run-malloc-benchmarks
1 #!/usr/bin/env ruby
2
3 require 'getoptlong'
4 require 'pathname'
5
6 $binDir = "#{File.expand_path(File.dirname(__FILE__))}"
7 $productDir = `perl -e 'use lib \"#{$binDir}/../../Tools/Scripts\"; use webkitdirs; print productDir()'`
8
9 $benchmarks = [
10     # Single-threaded benchmarks.
11     "churn",
12     "list_allocate",
13     "tree_allocate",
14     "tree_churn",
15     "fragment",
16     "fragment_iterate",
17     "medium",
18     "big",
19
20     # Benchmarks based on browser recordings.
21     "facebook",
22     "reddit",
23     "flickr",
24     "theverge",
25
26     # Multi-threaded benchmark variants.
27     "message_one",
28     "message_many",
29     "churn --parallel",
30     "list_allocate --parallel",
31     "tree_allocate --parallel",
32     "tree_churn --parallel",
33     # "facebook --parallel",
34     # "reddit --parallel",
35     # "flickr --parallel",
36     # "theverge --parallel",
37     "fragment --parallel",
38     "fragment_iterate --parallel",
39
40     # These tests often crash TCMalloc: <rdar://problem/13657137>.
41     "medium --parallel",
42     "big --parallel",
43
44     # Enable these tests to test memory footprint. The way they run is not
45     # really compatible with throughput testing.
46     # "reddit_memory_warning --runs 0",
47     # "flickr_memory_warning --runs 0",
48     # "theverge_memory_warning --runs 0",
49
50     # Enable this test to test shrinking back down from a large heap while a process remains active.
51     # The way it runs is not really compatible with throughput testing.
52     # "balloon"
53 ]
54
55 $heap = 0
56
57 def usage
58         puts "run-malloc-benchmarks [options] <Name:/path/to/dylib> [<Name:/path/to/dylib>]"
59         puts
60         puts "    Runs a suite of memory allocation and access benchmarks."
61     puts
62     puts "    <Name:/path/to/dylib> is a symbolic name followed by a folder containing a libmbmalloc.dylib."
63     puts
64     puts "    Specify \"SystemMalloc\" to test the built-in libc malloc."
65     puts "    Specify \"NanoMalloc\" to test the built-in libc malloc using the NanoMalloc zone."
66     puts
67     puts "    Example usage:"
68     puts
69     puts "        run-malloc-benchmarks SystemMalloc NanoMalloc"
70     puts "        run-malloc-benchmarks FastMalloc:/path/to/FastMalloc/Build/Products/Release/"
71     puts "        run-malloc-benchmarks --benchmark churn SystemMalloc FastMalloc:/path/to/FastMalloc/Build/Products/Release/"
72     puts
73         puts "Options:"
74     puts
75     puts "    --benchmark <benchmark>      Select a single benchmark to run instead of the full suite."
76     puts "    --heap <heap>           Set a baseline heap size."
77     puts
78 end
79
80 class Dylib
81     attr_reader :name
82     attr_reader :path
83
84     def initialize(name, path)
85         @name = name
86         @path = File.join(path, "libmbmalloc.dylib")
87     end
88 end
89
90 class Results
91     attr_reader :executionTime
92     attr_reader :peakMemory
93     attr_reader :memoryAtEnd
94
95     def initialize(executionTime, peakMemory, memoryAtEnd)
96         @executionTime = executionTime
97         @peakMemory = peakMemory
98         @memoryAtEnd = memoryAtEnd
99     end
100 end
101
102 class Stat
103     attr_reader :benchmark
104     attr_reader :result
105
106     def initialize(benchmark, result)
107         @benchmark = benchmark
108         @result = result[/\d+/].to_i
109     end
110 end
111
112 class TimeStat < Stat
113     def to_s
114         @result + "ms"
115     end
116 end
117
118 class MemoryStat < Stat
119     def to_s
120         @result + "kB"
121     end
122 end
123
124 class PeakMemoryStat < Stat
125     def to_s
126         @result + "kB"
127     end
128 end
129
130 def lpad(str, chars)
131     if str.length > chars
132         str
133     else
134         "%#{chars}s"%(str)
135     end
136 end
137
138 def rpad(str, chars)
139     while str.length < chars
140         str += " "
141     end
142     str
143 end
144
145 def computeArithmeticMean(array)
146   sum = 0.0
147   array.each {
148     | value |
149     sum += value
150   }
151   (sum / array.length)
152 end
153
154 def computeGeometricMean(array)
155   mult = 1.0
156   array.each {
157     | value |
158     mult *= value ? value : 1.0
159   }
160   (mult ** (1.0 / array.length))
161 end
162
163 def computeHarmonicMean(array)
164   1.0 / computeArithmeticMean(array.collect{ | value | 1.0 / value })
165 end
166
167 def lowerIsBetter(a, b, better, worse)
168     if b < a
169         return "^ " + (a.to_f / b.to_f).round(2).to_s + "x " + better
170     end
171
172     if b == a
173         return ""
174     end
175
176     "! " + (b.to_f / a.to_f).round(2).to_s + "x " + worse
177 end
178
179
180 def lowerIsFaster(a, b)
181     lowerIsBetter(a, b, "faster", "slower")
182 end
183
184 def lowerIsSmaller(a, b)
185     lowerIsBetter(a, b, "smaller", "bigger")
186 end
187
188 def numberWithDelimiter(number)
189     number.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse
190 end
191
192 def prettify(number, suffix)
193     numberWithDelimiter(number) + suffix
194 end
195
196 def parseOptions
197     GetoptLong.new(
198         ['--benchmark', GetoptLong::REQUIRED_ARGUMENT],
199         ['--heap', GetoptLong::REQUIRED_ARGUMENT],
200         ['--help', GetoptLong::NO_ARGUMENT],
201     ).each {
202         | opt, arg |
203         case opt
204         when '--benchmark'
205             $benchmarks = [ arg ]
206         when '--heap'
207             $heap = arg
208         when '--help'
209             usage
210             exit 1
211         else
212           raise "bad option: #{opt}"
213         end
214     }
215
216     if ARGV.length < 1
217         puts "Error: No dylib specified."
218         exit 1
219     end
220
221     dylibs = []
222     ARGV.each {
223         | arg |
224         if arg == "SystemMalloc"
225             dylib = Dylib.new("SystemMalloc", $productDir)
226         elsif arg == "NanoMalloc"
227             dylib = Dylib.new("NanoMalloc", $productDir)
228         else
229             name = arg.split(":")[0]
230             path = arg.split(":")[1]
231             if !name || name.length < 1 ||
232                 !path || path.length < 1
233                 puts "Invalid <Name:/path/to/dylib>: '#{arg}'."
234                 exit 1
235             end
236
237             dylib = Dylib.new(name, File.expand_path(path))
238         end
239
240         if !File.exists?(dylib.path)
241             puts "File not found: #{dylib.path}."
242             exit 1
243         end
244
245         dylibs.push(dylib)
246     }
247     dylibs
248 end
249
250 def runBenchmarks(dylibs)
251     executionTime = []
252     peakMemory = []
253     memoryAtEnd = []
254
255     $benchmarks.each {
256         | benchmark |
257
258         executionTime.push([])
259         peakMemory.push([])
260         memoryAtEnd.push([])
261
262         dylibs.each {
263             | dylib |
264
265             $stderr.print "\rRUNNING #{dylib.name}: #{benchmark}...                                "
266             env = "DYLD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' "
267             if dylib.name == "NanoMalloc"
268                 env += "MallocNanoZone=1 "
269             end
270             input = "cd '#{$productDir}'; #{env} '#{$productDir}/MallocBench' --benchmark #{benchmark} --heap #{$heap}}"
271             output =`#{input}`
272             splitOutput = output.split("\n")
273
274             executionTime[-1].push(TimeStat.new(benchmark, splitOutput[1]))
275             peakMemory[-1].push(PeakMemoryStat.new(benchmark, splitOutput.length > 3 ? splitOutput[3] : "0"))
276             memoryAtEnd[-1].push(MemoryStat.new(benchmark, splitOutput.length > 2 ? splitOutput[2] : "0"))
277         }
278     }
279     $stderr.print "\r                                                                                \n"
280
281     Results.new(executionTime, peakMemory, memoryAtEnd)
282 end
283
284 def printResults(dylibs, results)
285     def printHeader(dylibs, fieldSize)
286         print
287         print lpad("", fieldSize)
288         print lpad(dylibs[0].name, fieldSize)
289         if dylibs.length > 1
290             print lpad(dylibs[1].name, fieldSize)
291             print lpad("Δ", fieldSize)
292         end
293         print "\n"
294     end
295
296     def printMetric(name, results, compareFunction, suffix, fieldSize)
297         def printMean(name, results, meanFunction, compareFunction, suffix, fieldSize)
298             means = []
299
300             means.push(meanFunction.call(results.collect { | stats | stats[0].result }))
301             print rpad("    " + name, fieldSize)
302             print lpad("#{prettify(means[0].round, suffix)}", fieldSize)
303
304             if results[0][1]
305                 means.push(meanFunction.call(results.collect { | stats | stats[1].result }))
306                 print lpad("#{prettify(means[1].round, suffix)}", fieldSize)
307                 print lpad(compareFunction.call(means[0], means[1]), fieldSize)
308             end
309
310             print "\n"
311         end
312
313         if results[0][0].result == 0
314             return
315         end
316
317         print name + ":\n"
318         results.each {
319             | stats |
320
321             print rpad("    " + stats[0].benchmark, fieldSize)
322             print lpad("#{prettify(stats[0].result, suffix)}", fieldSize)
323
324             if stats[1]
325                 print lpad("#{prettify(stats[1].result, suffix)}", fieldSize)
326                 print lpad(compareFunction.call(stats[0].result, stats[1].result), fieldSize)
327             end
328
329             print "\n"
330         }
331
332         print "\n"
333
334         printMean("<geometric mean>", results, method(:computeGeometricMean), compareFunction, suffix, fieldSize)
335         printMean("<arithmetic mean>", results, method(:computeArithmeticMean), compareFunction, suffix, fieldSize)
336         printMean("<harmonic mean>", results, method(:computeHarmonicMean), compareFunction, suffix, fieldSize)
337
338         print "\n"
339     end
340
341     fieldSize = ($benchmarks + ["<arithmetic mean>"]).collect {
342         | benchmark |
343         benchmark.size
344     }.max + 4
345
346     printHeader(dylibs, fieldSize)
347     printMetric("Execution Time", results.executionTime, method(:lowerIsFaster), "ms", fieldSize)
348     printMetric("Peak Memory", results.peakMemory, method(:lowerIsSmaller), "kB", fieldSize)
349     printMetric("Memory at End", results.memoryAtEnd, method(:lowerIsSmaller), "kB", fieldSize)
350 end
351
352 def main
353     begin
354         dylibs = parseOptions()
355         results = runBenchmarks(dylibs)
356         printResults(dylibs, results)
357     rescue => exception
358         puts
359         puts
360         puts exception
361         puts exception.backtrace
362         puts
363     end
364 end
365
366 main()