bmalloc: Misc improvements to MallocBench
[WebKit.git] / PerformanceTests / MallocBench / run-malloc-benchmarks
1 #!/usr/bin/env ruby
2
3 require 'getoptlong'
4 require 'pathname'
5
6 $binDir = "#{File.expand_path(File.dirname(__FILE__))}"
7 $productDir = `perl -e 'use lib \"#{$binDir}/../../Tools/Scripts\"; use webkitdirs; print productDir()'`
8
9 $benchmarks_all = [
10     # Single-threaded benchmarks.
11     "churn",
12     "list_allocate",
13     "tree_allocate",
14     "tree_churn",
15     "fragment",
16     "fragment_iterate",
17     "medium",
18     "big",
19
20     # Benchmarks based on browser recordings.
21     "facebook",
22     "reddit",
23     "flickr",
24     "theverge",
25     "nimlang",
26
27     # Multi-threaded benchmark variants.
28     "message_one",
29     "message_many",
30     "churn --parallel",
31     "list_allocate --parallel",
32     "tree_allocate --parallel",
33     "tree_churn --parallel",
34     # "facebook --parallel",
35     # "reddit --parallel",
36     # "flickr --parallel",
37     # "theverge --parallel",
38     # "nimlang --use-thread-id",
39     "fragment --parallel",
40     "fragment_iterate --parallel",
41
42     # These tests often crash TCMalloc: <rdar://problem/13657137>.
43     "medium --parallel",
44     "big --parallel",
45
46     # Enable these tests to test memory footprint. The way they run is not
47     # really compatible with throughput testing.
48     # "reddit_memory_warning --runs 0",
49     # "flickr_memory_warning --runs 0",
50     # "theverge_memory_warning --runs 0",
51
52     # Enable this test to test shrinking back down from a large heap while a process remains active.
53     # The way it runs is not really compatible with throughput testing.
54     # "balloon"
55 ]
56
57 $benchmarks_memory = [
58     "facebook",
59     "reddit",
60     "flickr",
61     "theverge",
62     "nimlang"
63 ]
64
65 $benchmarks_memory_warning = [
66     "reddit_memory_warning --runs 0",
67     "flickr_memory_warning --runs 0",
68     "theverge_memory_warning --runs 0",
69 ]
70
71 $benchmarks = $benchmarks_all
72 $heap = 0
73
74 def usage
75         puts "run-malloc-benchmarks [options] <Name:/path/to/dylib> [<Name:/path/to/dylib>]"
76         puts
77         puts "    Runs a suite of memory allocation and access benchmarks."
78     puts
79     puts "    <Name:/path/to/dylib> is a symbolic name followed by a folder containing a libmbmalloc.dylib."
80     puts
81     puts "    Specify \"SystemMalloc\" to test the built-in libc malloc."
82     puts "    Specify \"NanoMalloc\" to test the built-in libc malloc using the NanoMalloc zone."
83     puts
84     puts "    Example usage:"
85     puts
86     puts "        run-malloc-benchmarks SystemMalloc NanoMalloc"
87     puts "        run-malloc-benchmarks FastMalloc:/path/to/FastMalloc/Build/Products/Release/"
88     puts "        run-malloc-benchmarks --benchmark churn SystemMalloc FastMalloc:/path/to/FastMalloc/Build/Products/Release/"
89     puts
90         puts "Options:"
91     puts
92     puts "    --benchmark <benchmark>      Select a single benchmark to run instead of the full suite."
93     puts "    --heap <heap>           Set a baseline heap size."
94     puts
95 end
96
97 class Dylib
98     attr_reader :name
99     attr_reader :path
100
101     def initialize(name, path)
102         @name = name
103         @path = File.join(path, "libmbmalloc.dylib")
104     end
105 end
106
107 class Results
108     attr_reader :executionTime
109     attr_reader :peakMemory
110     attr_reader :memoryAtEnd
111
112     def initialize(executionTime, peakMemory, memoryAtEnd)
113         @executionTime = executionTime
114         @peakMemory = peakMemory
115         @memoryAtEnd = memoryAtEnd
116     end
117 end
118
119 class Stat
120     attr_reader :benchmark
121     attr_reader :result
122
123     def initialize(benchmark, result)
124         @benchmark = benchmark
125         @result = result[/\d+/].to_i
126     end
127 end
128
129 class TimeStat < Stat
130     def to_s
131         @result + "ms"
132     end
133 end
134
135 class MemoryStat < Stat
136     def to_s
137         @result + "kB"
138     end
139 end
140
141 class PeakMemoryStat < Stat
142     def to_s
143         @result + "kB"
144     end
145 end
146
147 def lpad(str, chars)
148     if str.length > chars
149         str
150     else
151         "%#{chars}s"%(str)
152     end
153 end
154
155 def rpad(str, chars)
156     while str.length < chars
157         str += " "
158     end
159     str
160 end
161
162 def computeArithmeticMean(array)
163   sum = 0.0
164   array.each {
165     | value |
166     sum += value
167   }
168   (sum / array.length)
169 end
170
171 def computeGeometricMean(array)
172   mult = 1.0
173   array.each {
174     | value |
175     mult *= value ? value : 1.0
176   }
177   (mult ** (1.0 / array.length))
178 end
179
180 def computeHarmonicMean(array)
181   1.0 / computeArithmeticMean(array.collect{ | value | 1.0 / value })
182 end
183
184 def lowerIsBetter(a, b, better, worse)
185     if b < a
186         return "^ " + (a.to_f / b.to_f).round(2).to_s + "x " + better
187     end
188
189     if b == a
190         return ""
191     end
192
193     "! " + (b.to_f / a.to_f).round(2).to_s + "x " + worse
194 end
195
196
197 def lowerIsFaster(a, b)
198     lowerIsBetter(a, b, "faster", "slower")
199 end
200
201 def lowerIsSmaller(a, b)
202     lowerIsBetter(a, b, "smaller", "bigger")
203 end
204
205 def numberWithDelimiter(number)
206     number.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse
207 end
208
209 def prettify(number, suffix)
210     numberWithDelimiter(number) + suffix
211 end
212
213 def parseOptions
214     GetoptLong.new(
215         ['--benchmark', GetoptLong::REQUIRED_ARGUMENT],
216         ['--memory', GetoptLong::NO_ARGUMENT],
217         ['--memory_warning', GetoptLong::NO_ARGUMENT],
218         ['--heap', GetoptLong::REQUIRED_ARGUMENT],
219         ['--help', GetoptLong::NO_ARGUMENT],
220     ).each {
221         | opt, arg |
222         case opt
223         when '--benchmark'
224             $benchmarks = [ arg ]
225         when '--memory'
226             $benchmarks = $benchmarks_memory
227         when '--memory_warning'
228             $benchmarks = $benchmarks_memory_warning
229         when '--heap'
230             $heap = arg
231         when '--help'
232             usage
233             exit 1
234         else
235           raise "bad option: #{opt}"
236         end
237     }
238
239     if ARGV.length < 1
240         puts "Error: No dylib specified."
241         exit 1
242     end
243
244     dylibs = []
245     ARGV.each {
246         | arg |
247         if arg == "SystemMalloc"
248             dylib = Dylib.new("SystemMalloc", $productDir)
249         elsif arg == "NanoMalloc"
250             dylib = Dylib.new("NanoMalloc", $productDir)
251         else
252             name = arg.split(":")[0]
253             path = arg.split(":")[1]
254             if !name || name.length < 1 ||
255                 !path || path.length < 1
256                 puts "Invalid <Name:/path/to/dylib>: '#{arg}'."
257                 exit 1
258             end
259
260             dylib = Dylib.new(name, File.expand_path(path))
261         end
262
263         if !File.exists?(dylib.path)
264             puts "File not found: #{dylib.path}."
265             exit 1
266         end
267
268         dylibs.push(dylib)
269     }
270     dylibs
271 end
272
273 def runBenchmarks(dylibs)
274     executionTime = []
275     peakMemory = []
276     memoryAtEnd = []
277
278     $benchmarks.each {
279         | benchmark |
280
281         executionTime.push([])
282         peakMemory.push([])
283         memoryAtEnd.push([])
284
285         dylibs.each {
286             | dylib |
287
288             $stderr.print "\rRUNNING #{dylib.name}: #{benchmark}...                                "
289             env = "DYLD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' "
290             if dylib.name == "NanoMalloc"
291                 env += "MallocNanoZone=1 "
292             end
293             input = "cd '#{$productDir}'; #{env} '#{$productDir}/MallocBench' --benchmark #{benchmark} --heap #{$heap}}"
294             output =`#{input}`
295             splitOutput = output.split("\n")
296
297             executionTime[-1].push(TimeStat.new(benchmark, splitOutput[1]))
298             peakMemory[-1].push(PeakMemoryStat.new(benchmark, splitOutput.length > 3 ? splitOutput[2] : "0"))
299             memoryAtEnd[-1].push(MemoryStat.new(benchmark, splitOutput.length > 2 ? splitOutput[3] : "0"))
300         }
301     }
302     $stderr.print "\r                                                                                \n"
303
304     Results.new(executionTime, peakMemory, memoryAtEnd)
305 end
306
307 def printResults(dylibs, results)
308     def printHeader(dylibs, fieldSize)
309         print
310         print lpad("", fieldSize)
311         print lpad(dylibs[0].name, fieldSize)
312         if dylibs.length > 1
313             print lpad(dylibs[1].name, fieldSize)
314             print lpad("Δ", fieldSize)
315         end
316         print "\n"
317     end
318
319     def printMetric(name, results, compareFunction, suffix, fieldSize)
320         def printMean(name, results, meanFunction, compareFunction, suffix, fieldSize)
321             means = []
322
323             means.push(meanFunction.call(results.collect { | stats | stats[0].result }))
324             print rpad("    " + name, fieldSize)
325             print lpad("#{prettify(means[0].round, suffix)}", fieldSize)
326
327             if results[0][1]
328                 means.push(meanFunction.call(results.collect { | stats | stats[1].result }))
329                 print lpad("#{prettify(means[1].round, suffix)}", fieldSize)
330                 print lpad(compareFunction.call(means[0], means[1]), fieldSize)
331             end
332
333             print "\n"
334         end
335
336         if results[0][0].result == 0
337             return
338         end
339
340         print name + ":\n"
341         results.each {
342             | stats |
343
344             print rpad("    " + stats[0].benchmark, fieldSize)
345             print lpad("#{prettify(stats[0].result, suffix)}", fieldSize)
346
347             if stats[1]
348                 print lpad("#{prettify(stats[1].result, suffix)}", fieldSize)
349                 print lpad(compareFunction.call(stats[0].result, stats[1].result), fieldSize)
350             end
351
352             print "\n"
353         }
354
355         print "\n"
356
357         printMean("<geometric mean>", results, method(:computeGeometricMean), compareFunction, suffix, fieldSize)
358         printMean("<arithmetic mean>", results, method(:computeArithmeticMean), compareFunction, suffix, fieldSize)
359         printMean("<harmonic mean>", results, method(:computeHarmonicMean), compareFunction, suffix, fieldSize)
360
361         print "\n"
362     end
363
364     fieldSize = ($benchmarks + ["<arithmetic mean>"]).collect {
365         | benchmark |
366         benchmark.size
367     }.max + 4
368
369     printHeader(dylibs, fieldSize)
370     printMetric("Execution Time", results.executionTime, method(:lowerIsFaster), "ms", fieldSize)
371     printMetric("Peak Memory", results.peakMemory, method(:lowerIsSmaller), "kB", fieldSize)
372     printMetric("Memory at End", results.memoryAtEnd, method(:lowerIsSmaller), "kB", fieldSize)
373 end
374
375 def main
376     begin
377         dylibs = parseOptions()
378         results = runBenchmarks(dylibs)
379         printResults(dylibs, results)
380     rescue => exception
381         puts
382         puts
383         puts exception
384         puts exception.backtrace
385         puts
386     end
387 end
388
389 main()