[Flatpak SDK] Add perl-term-readkey
[WebKit.git] / PerformanceTests / MallocBench / run-malloc-benchmarks
1 #!/usr/bin/env ruby
2 # coding: utf-8
3
4 require 'getoptlong'
5 require 'pathname'
6
7 $benchmarks_all = [
8     # Single-threaded benchmarks.
9     "churn",
10     "list_allocate",
11     "tree_allocate",
12     "tree_churn",
13     "fragment",
14     "fragment_iterate",
15     "medium",
16     "big",
17
18     # Benchmarks based on browser recordings.
19     "facebook",
20     "reddit",
21     "flickr",
22     "theverge",
23     "nimlang",
24
25     # Multi-threaded benchmark variants.
26     "message_one",
27     "message_many",
28     "churn --parallel",
29     "list_allocate --parallel",
30     "tree_allocate --parallel",
31     "tree_churn --parallel",
32     "fragment --parallel",
33     "fragment_iterate --parallel",
34
35     # These tests often crash TCMalloc: <rdar://problem/13657137>.
36     "medium --parallel",
37     "big --parallel",
38
39     # Enable these tests to test memory footprint. The way they run is not
40     # really compatible with throughput testing.
41     # "reddit_memory_warning --runs 0",
42     # "flickr_memory_warning --runs 0",
43     # "theverge_memory_warning --runs 0",
44
45     # Enable this test to test shrinking back down from a large heap while a process remains active.
46     # The way it runs is not really compatible with throughput testing.
47     # "balloon"
48     "facebook --parallel",
49     "reddit --parallel",
50     "flickr --parallel",
51     "theverge --parallel",
52     # "nimlang --use-thread-id",
53 ]
54
55 $benchmarks_memory = [
56     "facebook",
57     "reddit",
58     "flickr",
59     "theverge",
60     "nimlang"
61 ]
62
63 $benchmarks_memory_warning = [
64     "reddit_memory_warning --runs 0",
65     "flickr_memory_warning --runs 0",
66     "theverge_memory_warning --runs 0",
67 ]
68
69 $benchmarks = $benchmarks_all
70 $heap = 0
71
72 def usage
73         puts "run-malloc-benchmarks [options] /path/to/MallocBench Name:/path/to/libmbmalloc.dylib [ Name:/path/to/libmbmalloc.dylib ]"
74         puts
75         puts "    Runs a suite of memory allocation and access benchmarks."
76     puts
77     puts "    <Name:/path/to/libmbmalloc.dylib> is a symbolic name followed by a path to libmbmalloc.dylib."
78     puts
79     puts "    Specify \"SystemMalloc\" to test the built-in libc malloc."
80     puts "    Specify \"NanoMalloc\" to test the built-in libc malloc using the NanoMalloc zone."
81     puts
82     puts "    Example usage:"
83     puts
84     puts "        run-malloc-benchmarks /BUILD/MallocBench SystemMalloc:/BUILD/libmbmalloc.dylib NanoMalloc:/BUILD/libmbmalloc.dylib"
85     puts "        run-malloc-benchmarks /BUILD/MallocBench FastMalloc:/BUILD/FastMalloc/libmbmalloc.dylib"
86     puts "        run-malloc-benchmarks --benchmark churn SystemMalloc:/BUILD/libmbmalloc.dylib FastMalloc:/BUILD/FastMalloc/libmbmalloc.dylib"
87     puts
88         puts "Options:"
89     puts
90     puts "    --benchmark <benchmark>      Select a single benchmark to run instead of the full suite."
91     puts "    --heap <heap>                Set a baseline heap size."
92     puts
93 end
94
95 class Dylib
96     attr_reader :name
97     attr_reader :path
98
99     def initialize(name, path)
100         @name = name
101         @path = path
102     end
103 end
104
105 class Results
106     attr_reader :executionTime
107     attr_reader :peakMemory
108     attr_reader :memoryAtEnd
109
110     def initialize(executionTime, peakMemory, memoryAtEnd)
111         @executionTime = executionTime
112         @peakMemory = peakMemory
113         @memoryAtEnd = memoryAtEnd
114     end
115 end
116
117 class Stat
118     attr_reader :benchmark
119     attr_reader :result
120
121     def initialize(benchmark, result)
122         @benchmark = benchmark
123         @result = result[/\d+/].to_i
124     end
125 end
126
127 class TimeStat < Stat
128     def to_s
129         @result + "ms"
130     end
131 end
132
133 class MemoryStat < Stat
134     def to_s
135         @result + "kB"
136     end
137 end
138
139 class PeakMemoryStat < Stat
140     def to_s
141         @result + "kB"
142     end
143 end
144
145 def lpad(str, chars)
146     if str.length > chars
147         str
148     else
149         "%#{chars}s"%(str)
150     end
151 end
152
153 def rpad(str, chars)
154     while str.length < chars
155         str += " "
156     end
157     str
158 end
159
160 def computeArithmeticMean(array)
161   sum = 0.0
162   array.each {
163     | value |
164     sum += value
165   }
166   (sum / array.length)
167 end
168
169 def computeGeometricMean(array)
170   mult = 1.0
171   array.each {
172     | value |
173     mult *= value ? value : 1.0
174   }
175   (mult ** (1.0 / array.length))
176 end
177
178 def computeHarmonicMean(array)
179   1.0 / computeArithmeticMean(array.collect{ | value | 1.0 / value })
180 end
181
182 def lowerIsBetter(a, b, better, worse)
183     if b < a
184         return "^ " + (a.to_f / b.to_f).round(2).to_s + "x " + better
185     end
186
187     if b == a
188         return ""
189     end
190
191     "! " + (b.to_f / a.to_f).round(2).to_s + "x " + worse
192 end
193
194
195 def lowerIsFaster(a, b)
196     lowerIsBetter(a, b, "faster", "slower")
197 end
198
199 def lowerIsSmaller(a, b)
200     lowerIsBetter(a, b, "smaller", "bigger")
201 end
202
203 def numberWithDelimiter(number)
204     number.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse
205 end
206
207 def prettify(number, suffix)
208     numberWithDelimiter(number) + suffix
209 end
210
211 def parseOptions
212     GetoptLong.new(
213         ['--benchmark', GetoptLong::REQUIRED_ARGUMENT],
214         ['--memory', GetoptLong::NO_ARGUMENT],
215         ['--memory_warning', GetoptLong::NO_ARGUMENT],
216         ['--heap', GetoptLong::REQUIRED_ARGUMENT],
217         ['--help', GetoptLong::NO_ARGUMENT],
218     ).each {
219         | opt, arg |
220         case opt
221         when '--benchmark'
222             $benchmarks = [ arg ]
223         when '--memory'
224             $benchmarks = $benchmarks_memory
225         when '--memory_warning'
226             $benchmarks = $benchmarks_memory_warning
227         when '--heap'
228             $heap = arg
229         when '--help'
230             usage
231             exit 1
232         else
233           raise "bad option: #{opt}"
234         end
235     }
236
237     if ARGV.length < 1
238         puts "Error: No MallocBench specified."
239         exit 1
240     end
241
242     if ARGV.length < 2
243         puts "Error: No dylib specified."
244         exit 1
245     end
246
247     $mallocBench = File.absolute_path(ARGV.shift)
248     if !File.exists?($mallocBench)
249         puts "File not found: #{$mallocBench}."
250         exit 1
251     end
252
253     $buildDir = Pathname.new($mallocBench).dirname
254
255     dylibs = []
256     ARGV.each {
257         | arg |
258         name, path = arg.split(":")
259         if !name || name.length < 1 ||
260             !path || path.length < 1
261             puts "Invalid <Name:/path/to/dylib>: '#{arg}'."
262             exit 1
263         end
264
265         dylib = Dylib.new(name, File.expand_path(path))
266
267         if !File.exists?(dylib.path)
268             puts "File not found: #{dylib.path}."
269             exit 1
270         end
271
272         dylibs.push(dylib)
273     }
274     dylibs
275 end
276
277 def runBenchmarks(dylibs)
278     executionTime = []
279     peakMemory = []
280     memoryAtEnd = []
281
282     $benchmarks.each {
283         | benchmark |
284
285         executionTime.push([])
286         peakMemory.push([])
287         memoryAtEnd.push([])
288
289         dylibs.each {
290             | dylib |
291
292             $stderr.print "\rRUNNING #{dylib.name}: #{benchmark}...                                "
293             env = "DYLD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' "
294             env += "LD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' "
295             if dylib.name == "NanoMalloc"
296                 env += "MallocNanoZone=1 "
297             elsif dylib.name == "SystemMalloc"
298                 env += "MallocNanoZone=0 "
299             end
300             input = "cd '#{$buildDir}'; #{env} '#{$mallocBench}' --benchmark #{benchmark} --heap #{$heap}}"
301             output =`#{input}`
302             splitOutput = output.split("\n")
303
304             executionTime[-1].push(TimeStat.new(benchmark, splitOutput[1]))
305             peakMemory[-1].push(PeakMemoryStat.new(benchmark, splitOutput.length > 3 ? splitOutput[2] : "0"))
306             memoryAtEnd[-1].push(MemoryStat.new(benchmark, splitOutput.length > 2 ? splitOutput[3] : "0"))
307         }
308     }
309     $stderr.print "\r                                                                                \n"
310
311     Results.new(executionTime, peakMemory, memoryAtEnd)
312 end
313
314 def printResults(dylibs, results)
315     def printHeader(dylibs, fieldSize)
316         print
317         print lpad("", fieldSize)
318         print lpad(dylibs[0].name, fieldSize)
319         if dylibs.length > 1
320             print lpad(dylibs[1].name, fieldSize)
321             print lpad("Δ", fieldSize)
322         end
323         print "\n"
324     end
325
326     def printMetric(name, results, compareFunction, suffix, fieldSize)
327         def printMean(name, results, meanFunction, compareFunction, suffix, fieldSize)
328             means = []
329
330             means.push(meanFunction.call(results.collect { | stats | stats[0].result }))
331             print rpad("    " + name, fieldSize)
332             print lpad("#{prettify(means[0].round, suffix)}", fieldSize)
333
334             if results[0][1]
335                 means.push(meanFunction.call(results.collect { | stats | stats[1].result }))
336                 print lpad("#{prettify(means[1].round, suffix)}", fieldSize)
337                 print lpad(compareFunction.call(means[0], means[1]), fieldSize)
338             end
339
340             print "\n"
341         end
342
343         if results[0][0].result == 0
344             return
345         end
346
347         print name + ":\n"
348         results.each {
349             | stats |
350
351             print rpad("    " + stats[0].benchmark, fieldSize)
352             print lpad("#{prettify(stats[0].result, suffix)}", fieldSize)
353
354             if stats[1]
355                 print lpad("#{prettify(stats[1].result, suffix)}", fieldSize)
356                 print lpad(compareFunction.call(stats[0].result, stats[1].result), fieldSize)
357             end
358
359             print "\n"
360         }
361
362         print "\n"
363
364         printMean("<geometric mean>", results, method(:computeGeometricMean), compareFunction, suffix, fieldSize)
365         printMean("<arithmetic mean>", results, method(:computeArithmeticMean), compareFunction, suffix, fieldSize)
366         printMean("<harmonic mean>", results, method(:computeHarmonicMean), compareFunction, suffix, fieldSize)
367
368         print "\n"
369     end
370
371     fieldSize = ($benchmarks + ["<arithmetic mean>"]).collect {
372         | benchmark |
373         benchmark.size
374     }.max + 4
375
376     printHeader(dylibs, fieldSize)
377     printMetric("Execution Time", results.executionTime, method(:lowerIsFaster), "ms", fieldSize)
378     printMetric("Peak Memory", results.peakMemory, method(:lowerIsSmaller), "kB", fieldSize)
379     printMetric("Memory at End", results.memoryAtEnd, method(:lowerIsSmaller), "kB", fieldSize)
380 end
381
382 def main
383     begin
384         dylibs = parseOptions()
385         results = runBenchmarks(dylibs)
386         printResults(dylibs, results)
387     rescue => exception
388         puts
389         puts
390         puts exception
391         puts exception.backtrace
392         puts
393     end
394 end
395
396 main()