[Linux] Port MallocBench
[WebKit-https.git] / PerformanceTests / MallocBench / run-malloc-benchmarks
1 #!/usr/bin/env ruby
2 # coding: utf-8
3
4 require 'getoptlong'
5 require 'pathname'
6
7 $binDir = "#{File.expand_path(File.dirname(__FILE__))}"
8 $productDir = `perl -e 'use lib \"#{$binDir}/../../Tools/Scripts\"; use webkitdirs; print productDir()'`
9
10 def determineOS
11     case RbConfig::CONFIG["host_os"]
12     when /darwin/i
13         "darwin"
14     when /linux/i
15         "linux"
16     when /mswin|mingw|cygwin/
17         "windows"
18     else
19         $stderr.puts "Warning: unable to determine host operating system"
20         nil
21     end
22 end
23
24 $hostOS = determineOS unless $hostOS
25 $cmake = false
26
27 if $hostOS == 'darwin'
28     $libraryExtension = "dylib"
29 else
30     $libraryExtension = "so"
31 end
32
33 $benchmarks_all = [
34     # Single-threaded benchmarks.
35     "churn",
36     "list_allocate",
37     "tree_allocate",
38     "tree_churn",
39     "fragment",
40     "fragment_iterate",
41     "medium",
42     "big",
43
44     # Benchmarks based on browser recordings.
45     "facebook",
46     "reddit",
47     "flickr",
48     "theverge",
49     "nimlang",
50
51     # Multi-threaded benchmark variants.
52     "message_one",
53     "message_many",
54     "churn --parallel",
55     "list_allocate --parallel",
56     "tree_allocate --parallel",
57     "tree_churn --parallel",
58     "fragment --parallel",
59     "fragment_iterate --parallel",
60
61     # These tests often crash TCMalloc: <rdar://problem/13657137>.
62     "medium --parallel",
63     "big --parallel",
64
65     # Enable these tests to test memory footprint. The way they run is not
66     # really compatible with throughput testing.
67     # "reddit_memory_warning --runs 0",
68     # "flickr_memory_warning --runs 0",
69     # "theverge_memory_warning --runs 0",
70
71     # Enable this test to test shrinking back down from a large heap while a process remains active.
72     # The way it runs is not really compatible with throughput testing.
73     # "balloon"
74     "facebook --parallel",
75     "reddit --parallel",
76     "flickr --parallel",
77     "theverge --parallel",
78     # "nimlang --use-thread-id",
79 ]
80
81 $benchmarks_memory = [
82     "facebook",
83     "reddit",
84     "flickr",
85     "theverge",
86     "nimlang"
87 ]
88
89 $benchmarks_memory_warning = [
90     "reddit_memory_warning --runs 0",
91     "flickr_memory_warning --runs 0",
92     "theverge_memory_warning --runs 0",
93 ]
94
95 $benchmarks = $benchmarks_all
96 $heap = 0
97
98 def usage
99         puts "run-malloc-benchmarks [options] <Name:/path/to/dylib> [<Name:/path/to/dylib>]"
100         puts
101         puts "    Runs a suite of memory allocation and access benchmarks."
102     puts
103     puts "    <Name:/path/to/dylib> is a symbolic name followed by a folder containing a libmbmalloc.dylib."
104     puts
105     puts "    Specify \"SystemMalloc\" to test the built-in libc malloc."
106     puts "    Specify \"NanoMalloc\" to test the built-in libc malloc using the NanoMalloc zone."
107     puts
108     puts "    Example usage:"
109     puts
110     puts "        run-malloc-benchmarks SystemMalloc NanoMalloc"
111     puts "        run-malloc-benchmarks FastMalloc:/path/to/FastMalloc/Build/Products/Release/"
112     puts "        run-malloc-benchmarks --benchmark churn SystemMalloc FastMalloc:/path/to/FastMalloc/Build/Products/Release/"
113     puts
114         puts "Options:"
115     puts
116     puts "    --benchmark <benchmark>      Select a single benchmark to run instead of the full suite."
117     puts "    --heap <heap>                Set a baseline heap size."
118     puts "    --cmake                      Specify if build directory layout is for CMake."
119     puts
120 end
121
122 class Dylib
123     attr_reader :name
124     attr_reader :path
125
126     def initialize(name, path)
127         @name = name
128         @path = File.join(path, "libmbmalloc.#{$libraryExtension}")
129     end
130 end
131
132 class Results
133     attr_reader :executionTime
134     attr_reader :peakMemory
135     attr_reader :memoryAtEnd
136
137     def initialize(executionTime, peakMemory, memoryAtEnd)
138         @executionTime = executionTime
139         @peakMemory = peakMemory
140         @memoryAtEnd = memoryAtEnd
141     end
142 end
143
144 class Stat
145     attr_reader :benchmark
146     attr_reader :result
147
148     def initialize(benchmark, result)
149         @benchmark = benchmark
150         @result = result[/\d+/].to_i
151     end
152 end
153
154 class TimeStat < Stat
155     def to_s
156         @result + "ms"
157     end
158 end
159
160 class MemoryStat < Stat
161     def to_s
162         @result + "kB"
163     end
164 end
165
166 class PeakMemoryStat < Stat
167     def to_s
168         @result + "kB"
169     end
170 end
171
172 def lpad(str, chars)
173     if str.length > chars
174         str
175     else
176         "%#{chars}s"%(str)
177     end
178 end
179
180 def rpad(str, chars)
181     while str.length < chars
182         str += " "
183     end
184     str
185 end
186
187 def computeArithmeticMean(array)
188   sum = 0.0
189   array.each {
190     | value |
191     sum += value
192   }
193   (sum / array.length)
194 end
195
196 def computeGeometricMean(array)
197   mult = 1.0
198   array.each {
199     | value |
200     mult *= value ? value : 1.0
201   }
202   (mult ** (1.0 / array.length))
203 end
204
205 def computeHarmonicMean(array)
206   1.0 / computeArithmeticMean(array.collect{ | value | 1.0 / value })
207 end
208
209 def lowerIsBetter(a, b, better, worse)
210     if b < a
211         return "^ " + (a.to_f / b.to_f).round(2).to_s + "x " + better
212     end
213
214     if b == a
215         return ""
216     end
217
218     "! " + (b.to_f / a.to_f).round(2).to_s + "x " + worse
219 end
220
221
222 def lowerIsFaster(a, b)
223     lowerIsBetter(a, b, "faster", "slower")
224 end
225
226 def lowerIsSmaller(a, b)
227     lowerIsBetter(a, b, "smaller", "bigger")
228 end
229
230 def numberWithDelimiter(number)
231     number.to_s.reverse.gsub(/...(?=.)/,'\&,').reverse
232 end
233
234 def prettify(number, suffix)
235     numberWithDelimiter(number) + suffix
236 end
237
238 def parseOptions
239     GetoptLong.new(
240         ['--benchmark', GetoptLong::REQUIRED_ARGUMENT],
241         ['--cmake', GetoptLong::NO_ARGUMENT],
242         ['--memory', GetoptLong::NO_ARGUMENT],
243         ['--memory_warning', GetoptLong::NO_ARGUMENT],
244         ['--heap', GetoptLong::REQUIRED_ARGUMENT],
245         ['--help', GetoptLong::NO_ARGUMENT],
246     ).each {
247         | opt, arg |
248         case opt
249         when '--benchmark'
250             $benchmarks = [ arg ]
251         when '--memory'
252             $benchmarks = $benchmarks_memory
253         when '--memory_warning'
254             $benchmarks = $benchmarks_memory_warning
255         when '--cmake'
256             $cmake = true
257         when '--heap'
258             $heap = arg
259         when '--help'
260             usage
261             exit 1
262         else
263           raise "bad option: #{opt}"
264         end
265     }
266
267     if $cmake
268         $libraryDir = "#{$productDir}/lib"
269         $systemMallocLibraryDir = "#{$productDir}/lib/system"
270         $binaryDir = "#{$productDir}/bin"
271     else
272         $libraryDir = $productDir
273         $binaryDir = $productDir
274         $systemMallocLibraryDir = $productDir
275     end
276
277     if ARGV.length < 1
278         puts "Error: No dylib specified."
279         exit 1
280     end
281
282     dylibs = []
283     ARGV.each {
284         | arg |
285         if arg == "SystemMalloc"
286             dylib = Dylib.new("SystemMalloc", $systemMallocLibraryDir)
287         elsif arg == "NanoMalloc"
288             dylib = Dylib.new("NanoMalloc", $libraryDir)
289         else
290             name = arg.split(":")[0]
291             path = arg.split(":")[1]
292             if !name || name.length < 1 ||
293                 !path || path.length < 1
294                 puts "Invalid <Name:/path/to/dylib>: '#{arg}'."
295                 exit 1
296             end
297
298             dylib = Dylib.new(name, File.expand_path(path))
299         end
300
301         if !File.exists?(dylib.path)
302             puts "File not found: #{dylib.path}."
303             exit 1
304         end
305
306         dylibs.push(dylib)
307     }
308     dylibs
309 end
310
311 def runBenchmarks(dylibs)
312     executionTime = []
313     peakMemory = []
314     memoryAtEnd = []
315
316     $benchmarks.each {
317         | benchmark |
318
319         executionTime.push([])
320         peakMemory.push([])
321         memoryAtEnd.push([])
322
323         dylibs.each {
324             | dylib |
325
326             $stderr.print "\rRUNNING #{dylib.name}: #{benchmark}...                                "
327             env = "DYLD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' "
328             env += "LD_LIBRARY_PATH='#{Pathname.new(dylib.path).dirname}' "
329             if dylib.name == "NanoMalloc"
330                 env += "MallocNanoZone=1 "
331             end
332             input = "cd '#{$productDir}'; #{env} '#{$binaryDir}/MallocBench' --benchmark #{benchmark} --heap #{$heap}}"
333             output =`#{input}`
334             splitOutput = output.split("\n")
335
336             executionTime[-1].push(TimeStat.new(benchmark, splitOutput[1]))
337             peakMemory[-1].push(PeakMemoryStat.new(benchmark, splitOutput.length > 3 ? splitOutput[2] : "0"))
338             memoryAtEnd[-1].push(MemoryStat.new(benchmark, splitOutput.length > 2 ? splitOutput[3] : "0"))
339         }
340     }
341     $stderr.print "\r                                                                                \n"
342
343     Results.new(executionTime, peakMemory, memoryAtEnd)
344 end
345
346 def printResults(dylibs, results)
347     def printHeader(dylibs, fieldSize)
348         print
349         print lpad("", fieldSize)
350         print lpad(dylibs[0].name, fieldSize)
351         if dylibs.length > 1
352             print lpad(dylibs[1].name, fieldSize)
353             print lpad("Δ", fieldSize)
354         end
355         print "\n"
356     end
357
358     def printMetric(name, results, compareFunction, suffix, fieldSize)
359         def printMean(name, results, meanFunction, compareFunction, suffix, fieldSize)
360             means = []
361
362             means.push(meanFunction.call(results.collect { | stats | stats[0].result }))
363             print rpad("    " + name, fieldSize)
364             print lpad("#{prettify(means[0].round, suffix)}", fieldSize)
365
366             if results[0][1]
367                 means.push(meanFunction.call(results.collect { | stats | stats[1].result }))
368                 print lpad("#{prettify(means[1].round, suffix)}", fieldSize)
369                 print lpad(compareFunction.call(means[0], means[1]), fieldSize)
370             end
371
372             print "\n"
373         end
374
375         if results[0][0].result == 0
376             return
377         end
378
379         print name + ":\n"
380         results.each {
381             | stats |
382
383             print rpad("    " + stats[0].benchmark, fieldSize)
384             print lpad("#{prettify(stats[0].result, suffix)}", fieldSize)
385
386             if stats[1]
387                 print lpad("#{prettify(stats[1].result, suffix)}", fieldSize)
388                 print lpad(compareFunction.call(stats[0].result, stats[1].result), fieldSize)
389             end
390
391             print "\n"
392         }
393
394         print "\n"
395
396         printMean("<geometric mean>", results, method(:computeGeometricMean), compareFunction, suffix, fieldSize)
397         printMean("<arithmetic mean>", results, method(:computeArithmeticMean), compareFunction, suffix, fieldSize)
398         printMean("<harmonic mean>", results, method(:computeHarmonicMean), compareFunction, suffix, fieldSize)
399
400         print "\n"
401     end
402
403     fieldSize = ($benchmarks + ["<arithmetic mean>"]).collect {
404         | benchmark |
405         benchmark.size
406     }.max + 4
407
408     printHeader(dylibs, fieldSize)
409     printMetric("Execution Time", results.executionTime, method(:lowerIsFaster), "ms", fieldSize)
410     printMetric("Peak Memory", results.peakMemory, method(:lowerIsSmaller), "kB", fieldSize)
411     printMetric("Memory at End", results.memoryAtEnd, method(:lowerIsSmaller), "kB", fieldSize)
412 end
413
414 def main
415     begin
416         dylibs = parseOptions()
417         results = runBenchmarks(dylibs)
418         printResults(dylibs, results)
419     rescue => exception
420         puts
421         puts
422         puts exception
423         puts exception.backtrace
424         puts
425     end
426 end
427
428 main()