Let's benchmark malloc
[WebKit.git] / PerformanceTests / MallocBench / run-malloc-benchmarks
1 #!/usr/bin/env ruby
2
3 require 'getoptlong'
4 require 'pathname'
5
6 $binDir = "#{File.expand_path(File.dirname(__FILE__))}"
7 $productDir = `perl -e 'use lib \"#{$binDir}/../../Tools/Scripts\"; use webkitdirs; print productDir()'`
8
9 $benchmarks = [
10     "churn",
11     "churn --parallel",
12     "list_allocate",
13     "list_allocate --parallel",
14     "tree_allocate",
15     "tree_allocate --parallel",
16     "tree_churn",
17     "tree_churn --parallel",
18     "facebook",
19     "facebook --parallel",
20     "fragment",
21     "fragment --parallel",
22     "fragment_iterate",
23     "fragment_iterate --parallel",
24     "message_one",
25     "message_many",
26     "medium",
27     "medium --parallel",
28     "big",
29     "big --parallel",
30 ]
31
32 $heap = 0
33 $measureHeap = false
34
35 def usage
36         puts "run-malloc-benchmarks [options] <Name:/path/to/dylib> [<Name:/path/to/dylib>]"
37         puts
38         puts "    Runs a suite of memory allocation and access benchmarks."
39     puts
40     puts "    <Name:/path/to/dylib> is a symbolic name followed by a folder containing a libmbmalloc.dylib."
41     puts
42     puts "    Specify \"SystemMalloc\" to test the built-in libc malloc."
43     puts "    Specify \"NanoMalloc\" to test the built-in libc malloc using the NanoMalloc zone."
44     puts
45     puts "    Example usage:"
46     puts
47     puts "        run-malloc-benchmarks SystemMalloc NanoMalloc"
48     puts "        run-malloc-benchmarks FastMalloc:/path/to/FastMalloc/Build/Products/Release/"
49     puts "        run-malloc-benchmarks --benchmark churn SystemMalloc FastMalloc:/path/to/FastMalloc/Build/Products/Release/"
50     puts
51         puts "Options:"
52     puts
53     puts "    --benchmark <benchmark>      Select a single benchmark to run instead of the full suite."
54     puts "    --heap <heap>           Set a baseline heap size."
55     puts
56 end
57
58 class Dylib
59     attr_reader :name
60     attr_reader :path
61
62     def initialize(name, path)
63         @name = name
64         @path = File.join(path, "libmbmalloc.dylib")
65     end
66 end
67
68 class Results
69     attr_reader :executionTime
70     attr_reader :peakMemory
71     attr_reader :memoryAtEnd
72
73     def initialize(executionTime, peakMemory, memoryAtEnd)
74         @executionTime = executionTime
75         @peakMemory = peakMemory
76         @memoryAtEnd = memoryAtEnd
77     end
78 end
79
80 class Stat
81     attr_reader :benchmark
82     attr_reader :result
83
84     def initialize(benchmark, result)
85         @benchmark = benchmark
86         @result = result[/\d+/].to_i
87     end
88 end
89
90 class TimeStat < Stat
91     def to_s
92         @result + "ms"
93     end
94 end
95
96 class MemoryStat < Stat
97     def to_s
98         @result + "kB"
99     end
100 end
101
102 class PeakMemoryStat < Stat
103     def to_s
104         @result + "kB"
105     end
106 end
107
108 def lpad(str, chars)
109     if str.length > chars
110         str
111     else
112         "%#{chars}s"%(str)
113     end
114 end
115
116 def rpad(str, chars)
117     while str.length < chars
118         str += " "
119     end
120     str
121 end
122
123 def computeArithmeticMean(array)
124   sum = 0.0
125   array.each {
126     | value |
127     sum += value
128   }
129   (sum / array.length)
130 end
131
132 def computeGeometricMean(array)
133   mult = 1.0
134   array.each {
135     | value |
136     mult *= value ? value : 1.0
137   }
138   (mult ** (1.0 / array.length))
139 end
140
141 def computeHarmonicMean(array)
142   1.0 / computeArithmeticMean(array.collect{ | value | 1.0 / value })
143 end
144
145 def lowerIsBetter(a, b, better, worse)
146     if b < a
147         return "^ " + (a.to_f / b.to_f).round(2).to_s + "x " + better
148     end
149
150     if b == a
151         return ""
152     end
153
154     "! " + (b.to_f / a.to_f).round(2).to_s + "x " + worse
155 end
156
157
158 def lowerIsFaster(a, b)
159     lowerIsBetter(a, b, "faster", "slower")
160 end
161
162 def lowerIsSmaller(a, b)
163     lowerIsBetter(a, b, "smaller", "bigger")
164 end
165
166 def numberWithDelimiter(number)
167     number.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse
168 end
169
170 def prettify(number, suffix)
171     numberWithDelimiter(number) + suffix
172 end
173
174 def parseOptions
175     GetoptLong.new(
176         ['--benchmark', GetoptLong::REQUIRED_ARGUMENT],
177         ['--heap', GetoptLong::REQUIRED_ARGUMENT],
178         ['--measure-heap', GetoptLong::NO_ARGUMENT],
179         ['--help', GetoptLong::NO_ARGUMENT],
180     ).each {
181         | opt, arg |
182         case opt
183         when '--benchmark'
184             $benchmarks = [ arg ]
185         when '--heap'
186             $heap = arg
187         when '--measure-heap'
188             $measureHeap = true
189         when '--help'
190             usage
191             exit 1
192         else
193           raise "bad option: #{opt}"
194         end
195     }
196
197     if ARGV.length < 1
198         puts "Error: No dylib specified."
199         exit 1
200     end
201
202     dylibs = []
203     ARGV.each {
204         | arg |
205         if arg == "SystemMalloc"
206             dylib = Dylib.new("SystemMalloc", $productDir)
207         elsif arg == "NanoMalloc"
208             dylib = Dylib.new("NanoMalloc", $productDir)
209         else
210             name = arg.split(":")[0]
211             path = arg.split(":")[1]
212             if !name || name.length < 1 ||
213                 !path || path.length < 1
214                 puts "Invalid <Name:/path/to/dylib>: '#{arg}'."
215                 exit 1
216             end
217
218             dylib = Dylib.new(name, File.expand_path(path))
219         end
220
221         if !File.exists?(dylib.path)
222             puts "File not found: #{dylib.path}."
223             exit 1
224         end
225
226         dylibs.push(dylib)
227     }
228     dylibs
229 end
230
231 def runBenchmarks(dylibs)
232     executionTime = []
233     peakMemory = []
234     memoryAtEnd = []
235
236     $benchmarks.each {
237         | benchmark |
238
239         executionTime.push([])
240         peakMemory.push([])
241         memoryAtEnd.push([])
242
243         dylibs.each {
244             | dylib |
245
246             $stderr.print "\rRUNNING #{dylib.name}: #{benchmark}...                "
247             env = "DYLD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' "
248             if dylib.name == "NanoMalloc"
249                 env += "MallocNanoZone=1 "
250             end
251             input = "cd '#{$productDir}'; #{env} '#{$productDir}/MallocBench' --benchmark #{benchmark} --heap #{$heap} #{$measureHeap ? '--measure-heap' : ''}"
252             output =`#{input}`
253             splitOutput = output.split("\n")
254
255             executionTime[-1].push(TimeStat.new(benchmark, splitOutput[1]))
256             peakMemory[-1].push(PeakMemoryStat.new(benchmark, splitOutput.length > 3 ? splitOutput[3] : "0"))
257             memoryAtEnd[-1].push(MemoryStat.new(benchmark, splitOutput.length > 2 ? splitOutput[2] : "0"))
258         }
259     }
260     $stderr.print "\r                                                                                \n"
261
262     Results.new(executionTime, peakMemory, memoryAtEnd)
263 end
264
265 def printResults(dylibs, results)
266     def printHeader(dylibs, fieldSize)
267         print
268         print lpad("", fieldSize)
269         print lpad(dylibs[0].name, fieldSize)
270         if dylibs[1]
271             print lpad(dylibs[1].name, fieldSize)
272             print lpad("Δ", fieldSize * 1.25)
273         end
274         print "\n"
275     end
276
277     def printMetric(name, results, compareFunction, suffix, fieldSize)
278         def printMean(name, results, meanFunction, compareFunction, suffix, fieldSize)
279             means = []
280
281             means.push(meanFunction.call(results.collect { | stats | stats[0].result }))
282             print rpad("    " + name, fieldSize)
283             print lpad("#{prettify(means[0].round, suffix)}", fieldSize)
284
285             if results[0][1]
286                 means.push(meanFunction.call(results.collect { | stats | stats[1].result }))
287                 print lpad("#{prettify(means[1].round, suffix)}", fieldSize)
288                 print lpad(compareFunction.call(means[0], means[1]), fieldSize * 1.25)
289             end
290
291             print "\n"
292         end
293
294         if results[0][0].result == 0
295             return
296         end
297
298         print name + ":\n"
299         results.each {
300             | stats |
301
302             print rpad("    " + stats[0].benchmark, fieldSize)
303             print lpad("#{prettify(stats[0].result, suffix)}", fieldSize)
304
305             if stats[1]
306                 print lpad("#{prettify(stats[1].result, suffix)}", fieldSize)
307                 print lpad(compareFunction.call(stats[0].result, stats[1].result), fieldSize * 1.25)
308             end
309
310             print "\n"
311         }
312
313         print "\n"
314
315         printMean("<geometric mean>", results, method(:computeGeometricMean), compareFunction, suffix, fieldSize)
316         printMean("<arithmetic mean>", results, method(:computeArithmeticMean), compareFunction, suffix, fieldSize)
317         printMean("<harmonic mean>", results, method(:computeHarmonicMean), compareFunction, suffix, fieldSize)
318
319         print "\n"
320     end
321
322     fieldSize = ($benchmarks + ["<arithmetic mean>"]).collect {
323         | benchmark |
324         benchmark.size
325     }.max + 4
326
327     printHeader(dylibs, fieldSize)
328     printMetric("Execution Time", results.executionTime, method(:lowerIsFaster), "ms", fieldSize)
329     printMetric("Peak Memory", results.peakMemory, method(:lowerIsSmaller), "kB", fieldSize)
330     printMetric("Memory at End", results.memoryAtEnd, method(:lowerIsSmaller), "kB", fieldSize)
331 end
332
333 def main
334     begin
335         dylibs = parseOptions()
336         results = runBenchmarks(dylibs)
337         printResults(dylibs, results)
338     rescue => exception
339         puts
340         puts
341         puts exception
342         puts exception.backtrace
343         puts
344     end
345 end
346
347 main()