Unreviewed change to revert extraneous changes made part of change set 196955.
[WebKit-https.git] / PerformanceTests / MallocBench / run-malloc-benchmarks
1 #!/usr/bin/env ruby
2
3 require 'getoptlong'
4 require 'pathname'
5
6 $binDir = "#{File.expand_path(File.dirname(__FILE__))}"
7 $productDir = `perl -e 'use lib \"#{$binDir}/../../Tools/Scripts\"; use webkitdirs; print productDir()'`
8
9 $benchmarks = [
10     # Single-threaded benchmarks.
11     "churn",
12     "list_allocate",
13     "tree_allocate",
14     "tree_churn",
15     "fragment",
16     "fragment_iterate",
17     "medium",
18     "big",
19
20     # Benchmarks based on browser recordings.
21     "facebook",
22     "reddit",
23     "flickr",
24     "theverge",
25     "nimlang",
26
27     # Multi-threaded benchmark variants.
28     "message_one",
29     "message_many",
30     "churn --parallel",
31     "list_allocate --parallel",
32     "tree_allocate --parallel",
33     "tree_churn --parallel",
34     # "facebook --parallel",
35     # "reddit --parallel",
36     # "flickr --parallel",
37     # "theverge --parallel",
38     # "nimlang --use-thread-id",
39     "fragment --parallel",
40     "fragment_iterate --parallel",
41
42     # These tests often crash TCMalloc: <rdar://problem/13657137>.
43     "medium --parallel",
44     "big --parallel",
45
46     # Enable these tests to test memory footprint. The way they run is not
47     # really compatible with throughput testing.
48     # "reddit_memory_warning --runs 0",
49     # "flickr_memory_warning --runs 0",
50     # "theverge_memory_warning --runs 0",
51
52     # Enable this test to test shrinking back down from a large heap while a process remains active.
53     # The way it runs is not really compatible with throughput testing.
54     # "balloon"
55 ]
56
57 $heap = 0
58
59 def usage
60         puts "run-malloc-benchmarks [options] <Name:/path/to/dylib> [<Name:/path/to/dylib>]"
61         puts
62         puts "    Runs a suite of memory allocation and access benchmarks."
63     puts
64     puts "    <Name:/path/to/dylib> is a symbolic name followed by a folder containing a libmbmalloc.dylib."
65     puts
66     puts "    Specify \"SystemMalloc\" to test the built-in libc malloc."
67     puts "    Specify \"NanoMalloc\" to test the built-in libc malloc using the NanoMalloc zone."
68     puts
69     puts "    Example usage:"
70     puts
71     puts "        run-malloc-benchmarks SystemMalloc NanoMalloc"
72     puts "        run-malloc-benchmarks FastMalloc:/path/to/FastMalloc/Build/Products/Release/"
73     puts "        run-malloc-benchmarks --benchmark churn SystemMalloc FastMalloc:/path/to/FastMalloc/Build/Products/Release/"
74     puts
75         puts "Options:"
76     puts
77     puts "    --benchmark <benchmark>      Select a single benchmark to run instead of the full suite."
78     puts "    --heap <heap>           Set a baseline heap size."
79     puts
80 end
81
82 class Dylib
83     attr_reader :name
84     attr_reader :path
85
86     def initialize(name, path)
87         @name = name
88         @path = File.join(path, "libmbmalloc.dylib")
89     end
90 end
91
92 class Results
93     attr_reader :executionTime
94     attr_reader :peakMemory
95     attr_reader :memoryAtEnd
96
97     def initialize(executionTime, peakMemory, memoryAtEnd)
98         @executionTime = executionTime
99         @peakMemory = peakMemory
100         @memoryAtEnd = memoryAtEnd
101     end
102 end
103
104 class Stat
105     attr_reader :benchmark
106     attr_reader :result
107
108     def initialize(benchmark, result)
109         @benchmark = benchmark
110         @result = result[/\d+/].to_i
111     end
112 end
113
114 class TimeStat < Stat
115     def to_s
116         @result + "ms"
117     end
118 end
119
120 class MemoryStat < Stat
121     def to_s
122         @result + "kB"
123     end
124 end
125
126 class PeakMemoryStat < Stat
127     def to_s
128         @result + "kB"
129     end
130 end
131
132 def lpad(str, chars)
133     if str.length > chars
134         str
135     else
136         "%#{chars}s"%(str)
137     end
138 end
139
140 def rpad(str, chars)
141     while str.length < chars
142         str += " "
143     end
144     str
145 end
146
147 def computeArithmeticMean(array)
148   sum = 0.0
149   array.each {
150     | value |
151     sum += value
152   }
153   (sum / array.length)
154 end
155
156 def computeGeometricMean(array)
157   mult = 1.0
158   array.each {
159     | value |
160     mult *= value ? value : 1.0
161   }
162   (mult ** (1.0 / array.length))
163 end
164
165 def computeHarmonicMean(array)
166   1.0 / computeArithmeticMean(array.collect{ | value | 1.0 / value })
167 end
168
169 def lowerIsBetter(a, b, better, worse)
170     if b < a
171         return "^ " + (a.to_f / b.to_f).round(2).to_s + "x " + better
172     end
173
174     if b == a
175         return ""
176     end
177
178     "! " + (b.to_f / a.to_f).round(2).to_s + "x " + worse
179 end
180
181
182 def lowerIsFaster(a, b)
183     lowerIsBetter(a, b, "faster", "slower")
184 end
185
186 def lowerIsSmaller(a, b)
187     lowerIsBetter(a, b, "smaller", "bigger")
188 end
189
190 def numberWithDelimiter(number)
191     number.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse
192 end
193
194 def prettify(number, suffix)
195     numberWithDelimiter(number) + suffix
196 end
197
198 def parseOptions
199     GetoptLong.new(
200         ['--benchmark', GetoptLong::REQUIRED_ARGUMENT],
201         ['--heap', GetoptLong::REQUIRED_ARGUMENT],
202         ['--help', GetoptLong::NO_ARGUMENT],
203     ).each {
204         | opt, arg |
205         case opt
206         when '--benchmark'
207             $benchmarks = [ arg ]
208         when '--heap'
209             $heap = arg
210         when '--help'
211             usage
212             exit 1
213         else
214           raise "bad option: #{opt}"
215         end
216     }
217
218     if ARGV.length < 1
219         puts "Error: No dylib specified."
220         exit 1
221     end
222
223     dylibs = []
224     ARGV.each {
225         | arg |
226         if arg == "SystemMalloc"
227             dylib = Dylib.new("SystemMalloc", $productDir)
228         elsif arg == "NanoMalloc"
229             dylib = Dylib.new("NanoMalloc", $productDir)
230         else
231             name = arg.split(":")[0]
232             path = arg.split(":")[1]
233             if !name || name.length < 1 ||
234                 !path || path.length < 1
235                 puts "Invalid <Name:/path/to/dylib>: '#{arg}'."
236                 exit 1
237             end
238
239             dylib = Dylib.new(name, File.expand_path(path))
240         end
241
242         if !File.exists?(dylib.path)
243             puts "File not found: #{dylib.path}."
244             exit 1
245         end
246
247         dylibs.push(dylib)
248     }
249     dylibs
250 end
251
252 def runBenchmarks(dylibs)
253     executionTime = []
254     peakMemory = []
255     memoryAtEnd = []
256
257     $benchmarks.each {
258         | benchmark |
259
260         executionTime.push([])
261         peakMemory.push([])
262         memoryAtEnd.push([])
263
264         dylibs.each {
265             | dylib |
266
267             $stderr.print "\rRUNNING #{dylib.name}: #{benchmark}...                                "
268             env = "DYLD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' "
269             if dylib.name == "NanoMalloc"
270                 env += "MallocNanoZone=1 "
271             end
272             input = "cd '#{$productDir}'; #{env} '#{$productDir}/MallocBench' --benchmark #{benchmark} --heap #{$heap}}"
273             output =`#{input}`
274             splitOutput = output.split("\n")
275
276             executionTime[-1].push(TimeStat.new(benchmark, splitOutput[1]))
277             peakMemory[-1].push(PeakMemoryStat.new(benchmark, splitOutput.length > 3 ? splitOutput[3] : "0"))
278             memoryAtEnd[-1].push(MemoryStat.new(benchmark, splitOutput.length > 2 ? splitOutput[2] : "0"))
279         }
280     }
281     $stderr.print "\r                                                                                \n"
282
283     Results.new(executionTime, peakMemory, memoryAtEnd)
284 end
285
286 def printResults(dylibs, results)
287     def printHeader(dylibs, fieldSize)
288         print
289         print lpad("", fieldSize)
290         print lpad(dylibs[0].name, fieldSize)
291         if dylibs.length > 1
292             print lpad(dylibs[1].name, fieldSize)
293             print lpad("Δ", fieldSize)
294         end
295         print "\n"
296     end
297
298     def printMetric(name, results, compareFunction, suffix, fieldSize)
299         def printMean(name, results, meanFunction, compareFunction, suffix, fieldSize)
300             means = []
301
302             means.push(meanFunction.call(results.collect { | stats | stats[0].result }))
303             print rpad("    " + name, fieldSize)
304             print lpad("#{prettify(means[0].round, suffix)}", fieldSize)
305
306             if results[0][1]
307                 means.push(meanFunction.call(results.collect { | stats | stats[1].result }))
308                 print lpad("#{prettify(means[1].round, suffix)}", fieldSize)
309                 print lpad(compareFunction.call(means[0], means[1]), fieldSize)
310             end
311
312             print "\n"
313         end
314
315         if results[0][0].result == 0
316             return
317         end
318
319         print name + ":\n"
320         results.each {
321             | stats |
322
323             print rpad("    " + stats[0].benchmark, fieldSize)
324             print lpad("#{prettify(stats[0].result, suffix)}", fieldSize)
325
326             if stats[1]
327                 print lpad("#{prettify(stats[1].result, suffix)}", fieldSize)
328                 print lpad(compareFunction.call(stats[0].result, stats[1].result), fieldSize)
329             end
330
331             print "\n"
332         }
333
334         print "\n"
335
336         printMean("<geometric mean>", results, method(:computeGeometricMean), compareFunction, suffix, fieldSize)
337         printMean("<arithmetic mean>", results, method(:computeArithmeticMean), compareFunction, suffix, fieldSize)
338         printMean("<harmonic mean>", results, method(:computeHarmonicMean), compareFunction, suffix, fieldSize)
339
340         print "\n"
341     end
342
343     fieldSize = ($benchmarks + ["<arithmetic mean>"]).collect {
344         | benchmark |
345         benchmark.size
346     }.max + 4
347
348     printHeader(dylibs, fieldSize)
349     printMetric("Execution Time", results.executionTime, method(:lowerIsFaster), "ms", fieldSize)
350     printMetric("Peak Memory", results.peakMemory, method(:lowerIsSmaller), "kB", fieldSize)
351     printMetric("Memory at End", results.memoryAtEnd, method(:lowerIsSmaller), "kB", fieldSize)
352 end
353
354 def main
355     begin
356         dylibs = parseOptions()
357         results = runBenchmarks(dylibs)
358         printResults(dylibs, results)
359     rescue => exception
360         puts
361         puts
362         puts exception
363         puts exception.backtrace
364         puts
365     end
366 end
367
368 main()