Renamed XLarge* => Large*
[WebKit-https.git] / PerformanceTests / MallocBench / run-malloc-benchmarks
1 #!/usr/bin/env ruby
2
3 require 'getoptlong'
4 require 'pathname'
5
6 $binDir = "#{File.expand_path(File.dirname(__FILE__))}"
7 $productDir = `perl -e 'use lib \"#{$binDir}/../../Tools/Scripts\"; use webkitdirs; print productDir()'`
8
9 $benchmarks_all = [
10     "message_one",
11     "message_many",
12     "churn --parallel",
13     "list_allocate --parallel",
14     "tree_allocate --parallel",
15     "tree_churn --parallel",
16     "facebook --parallel",
17     "reddit --parallel",
18     "flickr --parallel",
19     "theverge --parallel",
20 ]
21
22 $benchmarks_memory = [
23     "facebook",
24     "reddit",
25     "flickr",
26     "theverge",
27     "nimlang"
28 ]
29
30 $benchmarks_memory_warning = [
31     "reddit_memory_warning --runs 0",
32     "flickr_memory_warning --runs 0",
33     "theverge_memory_warning --runs 0",
34 ]
35
36 $benchmarks = $benchmarks_all
37 $heap = 0
38
39 def usage
40         puts "run-malloc-benchmarks [options] <Name:/path/to/dylib> [<Name:/path/to/dylib>]"
41         puts
42         puts "    Runs a suite of memory allocation and access benchmarks."
43     puts
44     puts "    <Name:/path/to/dylib> is a symbolic name followed by a folder containing a libmbmalloc.dylib."
45     puts
46     puts "    Specify \"SystemMalloc\" to test the built-in libc malloc."
47     puts "    Specify \"NanoMalloc\" to test the built-in libc malloc using the NanoMalloc zone."
48     puts
49     puts "    Example usage:"
50     puts
51     puts "        run-malloc-benchmarks SystemMalloc NanoMalloc"
52     puts "        run-malloc-benchmarks FastMalloc:/path/to/FastMalloc/Build/Products/Release/"
53     puts "        run-malloc-benchmarks --benchmark churn SystemMalloc FastMalloc:/path/to/FastMalloc/Build/Products/Release/"
54     puts
55         puts "Options:"
56     puts
57     puts "    --benchmark <benchmark>      Select a single benchmark to run instead of the full suite."
58     puts "    --heap <heap>           Set a baseline heap size."
59     puts
60 end
61
62 class Dylib
63     attr_reader :name
64     attr_reader :path
65
66     def initialize(name, path)
67         @name = name
68         @path = File.join(path, "libmbmalloc.dylib")
69     end
70 end
71
72 class Results
73     attr_reader :executionTime
74     attr_reader :peakMemory
75     attr_reader :memoryAtEnd
76
77     def initialize(executionTime, peakMemory, memoryAtEnd)
78         @executionTime = executionTime
79         @peakMemory = peakMemory
80         @memoryAtEnd = memoryAtEnd
81     end
82 end
83
84 class Stat
85     attr_reader :benchmark
86     attr_reader :result
87
88     def initialize(benchmark, result)
89         @benchmark = benchmark
90         @result = result[/\d+/].to_i
91     end
92 end
93
94 class TimeStat < Stat
95     def to_s
96         @result + "ms"
97     end
98 end
99
100 class MemoryStat < Stat
101     def to_s
102         @result + "kB"
103     end
104 end
105
106 class PeakMemoryStat < Stat
107     def to_s
108         @result + "kB"
109     end
110 end
111
112 def lpad(str, chars)
113     if str.length > chars
114         str
115     else
116         "%#{chars}s"%(str)
117     end
118 end
119
120 def rpad(str, chars)
121     while str.length < chars
122         str += " "
123     end
124     str
125 end
126
127 def computeArithmeticMean(array)
128   sum = 0.0
129   array.each {
130     | value |
131     sum += value
132   }
133   (sum / array.length)
134 end
135
136 def computeGeometricMean(array)
137   mult = 1.0
138   array.each {
139     | value |
140     mult *= value ? value : 1.0
141   }
142   (mult ** (1.0 / array.length))
143 end
144
145 def computeHarmonicMean(array)
146   1.0 / computeArithmeticMean(array.collect{ | value | 1.0 / value })
147 end
148
149 def lowerIsBetter(a, b, better, worse)
150     if b < a
151         return "^ " + (a.to_f / b.to_f).round(2).to_s + "x " + better
152     end
153
154     if b == a
155         return ""
156     end
157
158     "! " + (b.to_f / a.to_f).round(2).to_s + "x " + worse
159 end
160
161
162 def lowerIsFaster(a, b)
163     lowerIsBetter(a, b, "faster", "slower")
164 end
165
166 def lowerIsSmaller(a, b)
167     lowerIsBetter(a, b, "smaller", "bigger")
168 end
169
170 def numberWithDelimiter(number)
171     number.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse
172 end
173
174 def prettify(number, suffix)
175     numberWithDelimiter(number) + suffix
176 end
177
178 def parseOptions
179     GetoptLong.new(
180         ['--benchmark', GetoptLong::REQUIRED_ARGUMENT],
181         ['--memory', GetoptLong::NO_ARGUMENT],
182         ['--memory_warning', GetoptLong::NO_ARGUMENT],
183         ['--heap', GetoptLong::REQUIRED_ARGUMENT],
184         ['--help', GetoptLong::NO_ARGUMENT],
185     ).each {
186         | opt, arg |
187         case opt
188         when '--benchmark'
189             $benchmarks = [ arg ]
190         when '--memory'
191             $benchmarks = $benchmarks_memory
192         when '--memory_warning'
193             $benchmarks = $benchmarks_memory_warning
194         when '--heap'
195             $heap = arg
196         when '--help'
197             usage
198             exit 1
199         else
200           raise "bad option: #{opt}"
201         end
202     }
203
204     if ARGV.length < 1
205         puts "Error: No dylib specified."
206         exit 1
207     end
208
209     dylibs = []
210     ARGV.each {
211         | arg |
212         if arg == "SystemMalloc"
213             dylib = Dylib.new("SystemMalloc", $productDir)
214         elsif arg == "NanoMalloc"
215             dylib = Dylib.new("NanoMalloc", $productDir)
216         else
217             name = arg.split(":")[0]
218             path = arg.split(":")[1]
219             if !name || name.length < 1 ||
220                 !path || path.length < 1
221                 puts "Invalid <Name:/path/to/dylib>: '#{arg}'."
222                 exit 1
223             end
224
225             dylib = Dylib.new(name, File.expand_path(path))
226         end
227
228         if !File.exists?(dylib.path)
229             puts "File not found: #{dylib.path}."
230             exit 1
231         end
232
233         dylibs.push(dylib)
234     }
235     dylibs
236 end
237
238 def runBenchmarks(dylibs)
239     executionTime = []
240     peakMemory = []
241     memoryAtEnd = []
242
243     $benchmarks.each {
244         | benchmark |
245
246         executionTime.push([])
247         peakMemory.push([])
248         memoryAtEnd.push([])
249
250         dylibs.each {
251             | dylib |
252
253             $stderr.print "\rRUNNING #{dylib.name}: #{benchmark}...                                "
254             env = "DYLD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' "
255             if dylib.name == "NanoMalloc"
256                 env += "MallocNanoZone=1 "
257             end
258             input = "cd '#{$productDir}'; #{env} '#{$productDir}/MallocBench' --benchmark #{benchmark} --heap #{$heap}}"
259             output =`#{input}`
260             splitOutput = output.split("\n")
261
262             executionTime[-1].push(TimeStat.new(benchmark, splitOutput[1]))
263             peakMemory[-1].push(PeakMemoryStat.new(benchmark, splitOutput.length > 3 ? splitOutput[2] : "0"))
264             memoryAtEnd[-1].push(MemoryStat.new(benchmark, splitOutput.length > 2 ? splitOutput[3] : "0"))
265         }
266     }
267     $stderr.print "\r                                                                                \n"
268
269     Results.new(executionTime, peakMemory, memoryAtEnd)
270 end
271
272 def printResults(dylibs, results)
273     def printHeader(dylibs, fieldSize)
274         print
275         print lpad("", fieldSize)
276         print lpad(dylibs[0].name, fieldSize)
277         if dylibs.length > 1
278             print lpad(dylibs[1].name, fieldSize)
279             print lpad("Δ", fieldSize)
280         end
281         print "\n"
282     end
283
284     def printMetric(name, results, compareFunction, suffix, fieldSize)
285         def printMean(name, results, meanFunction, compareFunction, suffix, fieldSize)
286             means = []
287
288             means.push(meanFunction.call(results.collect { | stats | stats[0].result }))
289             print rpad("    " + name, fieldSize)
290             print lpad("#{prettify(means[0].round, suffix)}", fieldSize)
291
292             if results[0][1]
293                 means.push(meanFunction.call(results.collect { | stats | stats[1].result }))
294                 print lpad("#{prettify(means[1].round, suffix)}", fieldSize)
295                 print lpad(compareFunction.call(means[0], means[1]), fieldSize)
296             end
297
298             print "\n"
299         end
300
301         if results[0][0].result == 0
302             return
303         end
304
305         print name + ":\n"
306         results.each {
307             | stats |
308
309             print rpad("    " + stats[0].benchmark, fieldSize)
310             print lpad("#{prettify(stats[0].result, suffix)}", fieldSize)
311
312             if stats[1]
313                 print lpad("#{prettify(stats[1].result, suffix)}", fieldSize)
314                 print lpad(compareFunction.call(stats[0].result, stats[1].result), fieldSize)
315             end
316
317             print "\n"
318         }
319
320         print "\n"
321
322         printMean("<geometric mean>", results, method(:computeGeometricMean), compareFunction, suffix, fieldSize)
323         printMean("<arithmetic mean>", results, method(:computeArithmeticMean), compareFunction, suffix, fieldSize)
324         printMean("<harmonic mean>", results, method(:computeHarmonicMean), compareFunction, suffix, fieldSize)
325
326         print "\n"
327     end
328
329     fieldSize = ($benchmarks + ["<arithmetic mean>"]).collect {
330         | benchmark |
331         benchmark.size
332     }.max + 4
333
334     printHeader(dylibs, fieldSize)
335     printMetric("Execution Time", results.executionTime, method(:lowerIsFaster), "ms", fieldSize)
336     printMetric("Peak Memory", results.peakMemory, method(:lowerIsSmaller), "kB", fieldSize)
337     printMetric("Memory at End", results.memoryAtEnd, method(:lowerIsSmaller), "kB", fieldSize)
338 end
339
340 def main
341     begin
342         dylibs = parseOptions()
343         results = runBenchmarks(dylibs)
344         printResults(dylibs, results)
345     rescue => exception
346         puts
347         puts
348         puts exception
349         puts exception.backtrace
350         puts
351     end
352 end
353
354 main()