GC constraint solving should be parallel
[WebKit-https.git] / Tools / Scripts / run-jsc-benchmarks
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2011-2015 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 # 1. Redistributions of source code must retain the above copyright
9 #    notice, this list of conditions and the following disclaimer.
10 # 2. Redistributions in binary form must reproduce the above copyright
11 #    notice, this list of conditions and the following disclaimer in the
12 #    documentation and/or other materials provided with the distribution.
13 #
14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24 # THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'rubygems'
27
28 require 'getoptlong'
29 require 'pathname'
30 require 'shellwords'
31 require 'socket'
32
33 begin
34   require 'json'
35 rescue LoadError => e
36   $stderr.puts "It does not appear that you have the 'json' package installed.  Try running 'sudo gem install json'."
37   exit 1
38 end
39
40 SCRIPT_PATH = Pathname.new(__FILE__).realpath
41
42 raise unless SCRIPT_PATH.dirname.basename.to_s == "Scripts"
43 raise unless SCRIPT_PATH.dirname.dirname.basename.to_s == "Tools"
44
45 OPENSOURCE_PATH = SCRIPT_PATH.dirname.dirname.dirname
46 PERFORMANCETESTS_PATH = OPENSOURCE_PATH + "PerformanceTests"
47
48 SUNSPIDER_PATH = PERFORMANCETESTS_PATH + "SunSpider" + "tests" + "sunspider-1.0"
49 LONGSPIDER_PATH = PERFORMANCETESTS_PATH + "LongSpider"
50 V8_PATH = PERFORMANCETESTS_PATH + "SunSpider" + "tests" + "v8-v6"
51 TAILBENCH_PATH = PERFORMANCETESTS_PATH + "TailBench9000"
52 MICROBENCHMARKS_PATH = OPENSOURCE_PATH + "JSTests" + "microbenchmarks"
53 OPENSOURCE_OCTANE_PATH = PERFORMANCETESTS_PATH + "Octane"
54 OCTANE_WRAPPER_PATH = OPENSOURCE_OCTANE_PATH + "wrappers"
55 JSBENCH_PATH = PERFORMANCETESTS_PATH + "JSBench"
56 SIXSPEED_PATH = PERFORMANCETESTS_PATH + "SixSpeed" + "tests"
57 SIXSPEED_WRAPPER_PATH = PERFORMANCETESTS_PATH + "SixSpeed" + "wrappers"
58
59 TEMP_PATH = OPENSOURCE_PATH + "BenchmarkTemp"
60
61 if TEMP_PATH.exist?
62   raise unless TEMP_PATH.directory?
63 else
64   Dir.mkdir(TEMP_PATH)
65 end
66
67 BENCH_DATA_PATH = TEMP_PATH + "benchdata"
68
69 IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933, 
70             0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684, 
71             0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227, 
72             0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674, 
73             0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652, 
74             0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604, 
75             0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338, 
76             0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538, 
77             0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185, 
78             0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694, 
79             0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244, 
80             0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067, 
81             0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323, 
82             0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129, 
83             0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574, 
84             0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722, 
85             0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624, 
86             0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321, 
87             0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843, 
88             0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217, 
89             0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464, 
90             0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599, 
91             0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638, 
92             0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592, 
93             0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471, 
94             0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283, 
95             0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037, 
96             0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738, 
97             0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481, 
98             0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085, 
99             0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651, 
100             0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183, 
101             0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683, 
102             0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154, 
103             0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599, 
104             0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019, 
105             0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417, 
106             0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795, 
107             0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153, 
108             0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494, 
109             0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818, 
110             0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128, 
111             0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423, 
112             0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704, 
113             0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974, 
114             0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232, 
115             0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479, 
116             0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716, 
117             0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943, 
118             0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192, 
119             0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401, 
120             0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963, 
121             0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822, 
122             0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007, 
123             0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186, 
124             0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358, 
125             0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525, 
126             0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686, 
127             0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841, 
128             0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991, 
129             0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137, 
130             0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278, 
131             0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414, 
132             0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547, 
133             0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675, 
134             0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799, 
135             0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192, 
136             0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037, 
137             0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151, 
138             0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262, 
139             0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369, 
140             0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489, 
141             0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259, 
142             0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689, 
143             0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785, 
144             0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879, 
145             0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297, 
146             0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059, 
147             0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145, 
148             0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242, 
149             0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324, 
150             0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404, 
151             0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483, 
152             0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559, 
153             0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634, 
154             0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707, 
155             0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778, 
156             0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848, 
157             0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916, 
158             0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983, 
159             0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057, 
160             0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121, 
161             0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183, 
162             0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244, 
163             0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304, 
164             0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363, 
165             0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442, 
166             0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476, 
167             0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539, 
168             0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946, 
169             0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653, 
170             0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712, 
171             0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762, 
172             0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812, 
173             0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867, 
174             0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915, 
175             0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962, 
176             0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007, 
177             0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059, 
178             0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103, 
179             0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146, 
180             0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189, 
181             0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231, 
182             0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272, 
183             0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312, 
184             0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352, 
185             0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391, 
186             0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429, 
187             0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467, 
188             0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504, 
189             0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546, 
190             0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582, 
191             0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617, 
192             0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652, 
193             0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686, 
194             0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724, 
195             0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757, 
196             0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579, 
197             0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822, 
198             0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853, 
199             0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885, 
200             0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915, 
201             0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595, 
202             0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979, 
203             0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013, 
204             0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041, 
205             0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074, 
206             0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102, 
207             0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129, 
208             0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156, 
209             0.99616, 0.996164]
210
211 # Run-time configuration parameters (can be set with command-line options)
212
213 $rerun=1
214 $inner=1
215 $warmup=1
216 $outer=4
217 $quantum=1000
218 $includeSunSpider=true
219 $includeLongSpider=true
220 $includeV8=true
221 $includeKraken=true
222 $includeJSBench=true
223 $includeMicrobenchmarks=true
224 $includeAsmBench=true
225 $includeDSPJS=true
226 $includeBrowsermarkJS=false
227 $includeBrowsermarkDOM=false
228 $includeOctane=true
229 $includeCompressionBench = true
230 $includeSixSpeed = true
231 $includeTailBench = true
232 $measureGC=false
233 $benchmarkPattern=nil
234 $verbosity=0
235 $timeMode=:preciseTime
236 $forceVMKind=nil
237 $brief=false
238 $silent=false
239 $remoteHosts=[]
240 $alsoLocal=false
241 $sshOptions=[]
242 $vms = []
243 $environment = {}
244 $dependencies = []
245 $needToCopyVMs = false
246 $dontCopyVMs = false
247 $allDRT = true
248 $outputName = nil
249 $sunSpiderWarmup = true
250 $configPath = Pathname.new(ENV["HOME"]) + ".run-jsc-benchmarks"
251
252 $prepare = true
253 $run = true
254 $analyze = []
255
256 # Helpful functions and classes
257
258 def smallUsage
259   puts "Use the --help option to get basic usage information."
260   exit 1
261 end
262
263 def usage
264   puts "run-jsc-benchmarks [options] <vm1> [<vm2> ...]"
265   puts
266   puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
267   puts "benchmarks, and reports detailed statistics.  What makes run-jsc-benchmarks"
268   puts "special is that each benchmark/VM configuration is run in a single VM invocation,"
269   puts "and the invocations are run in random order.  This minimizes systematics due to"
270   puts "one benchmark polluting the running time of another.  The fine-grained"
271   puts "interleaving of VM invocations further minimizes systematics due to changes in"
272   puts "the performance or behavior of your machine."
273   puts 
274   puts "Run-jsc-benchmarks is highly configurable.  You can compare as many VMs as you"
275   puts "like.  You can change the amount of warm-up iterations, number of iterations"
276   puts "executed per VM invocation, and the number of VM invocations per benchmark."
277   puts
278   puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
279   puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
280   puts "the executable and <name> is the name that you would like to give the"
281   puts "configuration for the purposeof reporting.  If no name is given, a generic name"
282   puts "of the form Conf#<n> will be ascribed to the configuration automatically."
283   puts
284   puts "It's also possible to specify per-VM environment variables. For example, you"
285   puts "might specify a VM like Foo:JSC_useJIT=false:/path/to/jsc, in which case the"
286   puts "harness will set the JSC_useJIT environment variable to false just before running"
287   puts "the given VM. Note that the harness will not unset the environment variable, so"
288   puts "you must ensure that your other VMs will use the opposite setting"
289   puts "(JSC_useJIT=true in this case)."
290   puts
291   puts "Options:"
292   puts "--rerun <n>          Set the number of iterations of the benchmark that"
293   puts "                     contribute to the measured run time.  Default is #{$rerun}."
294   puts "--inner <n>          Set the number of inner (per-runtime-invocation)"
295   puts "                     iterations.  Default is #{$inner}."
296   puts "--outer <n>          Set the number of runtime invocations for each benchmark."
297   puts "                     Default is #{$outer}."
298   puts "--warmup <n>         Set the number of warm-up runs per invocation.  Default"
299   puts "                     is #{$warmup}. This has a different effect on different kinds"
300   puts "                     benchmarks. Some benchmarks have no notion of warm-up."
301   puts "--no-ss-warmup       Disable SunSpider-based warm-up runs."
302   puts "--quantum <n>        Set the duration in milliseconds for which an iteration of"
303   puts "                     a throughput benchmark should be run.  Default is #{$quantum}."
304   puts "--timing-mode        Set the way that time is measured.  Possible values"
305   puts "                     are 'preciseTime' and 'date'.  Default is 'preciseTime'."
306   puts "--force-vm-kind      Turn off auto-detection of VM kind, and assume that it is"
307   puts "                     the one specified.  Valid arguments are 'jsc', "
308   puts "                     'DumpRenderTree', or 'WebKitTestRunner'."
309   puts "--force-vm-copy      Force VM builds to be copied to the working directory."
310   puts "                     This may reduce pathologies resulting from path names."
311   puts "--dont-copy-vms      Don't copy VMs even when doing a remote benchmarking run;"
312   puts "                     instead assume that they are already there."
313   puts "--sunspider          Only run SunSpider."
314   puts "--v8-spider          Only run V8."
315   puts "--kraken             Only run Kraken."
316   puts "--js-bench           Only run JSBench."
317   puts "--microbenchmarks    Only run microbenchmarks."
318   puts "--dsp                Only run DSP."
319   puts "--asm-bench          Only run AsmBench."
320   puts "--browsermark-js     Only run browsermark-js."
321   puts "--browsermark-dom    Only run browsermark-dom."
322   puts "--octane             Only run Octane."
323   puts "--tail-bench         Only run TailBench"
324   puts "--compression-bench  Only run compression bench"
325   puts "                     The default is to run all benchmarks. The above options can"
326   puts "                     be combined to run any subset (so --sunspider --dsp will run"
327   puts "                     both SunSpider and DSP)."
328   puts "--six-speed          Only run SixSpeed."
329   puts "--benchmarks         Only run benchmarks matching the given regular expression."
330   puts "--measure-gc         Turn off manual calls to gc(), so that GC time is measured."
331   puts "                     Works best with large values of --inner.  You can also say"
332   puts "                     --measure-gc <conf>, which turns this on for one"
333   puts "                     configuration only."
334   puts "--verbose or -v      Print more stuff."
335   puts "--brief              Print only the final result for each VM."
336   puts "--silent             Don't print progress. This might slightly reduce some"
337   puts "                     performance perturbation."
338   puts "--remote <sshhosts>  Perform performance measurements remotely, on the given"
339   puts "                     SSH host(s). Easiest way to use this is to specify the SSH"
340   puts "                     user@host string. However, you can also supply a comma-"
341   puts "                     separated list of SSH hosts. Alternatively, you can use this"
342   puts "                     option multiple times to specify multiple hosts. This"
343   puts "                     automatically copies the WebKit release builds of the VMs"
344   puts "                     you specified to all of the hosts."
345   puts "--ssh-options        Pass additional options to SSH."
346   puts "--local              Also do a local benchmark run even when doing --remote."
347   puts "--vms                Use a JSON file to specify which VMs to run, as opposed to"
348   puts "                     specifying them on the command line."
349   puts "--prepare-only       Only prepare the runscript (a shell script that"
350   puts "                     invokes the VMs to run benchmarks) but don't run it."
351   puts "--analyze            Only read the output of the runscript but don't do anything"
352   puts "                     else. This requires passing the same arguments that you"
353   puts "                     passed when running --prepare-only."
354   puts "--output-name        Base of the filenames to put results into. Will write a file"
355   puts "                     called <base>_report.txt and <base>.json.  By default this"
356   puts "                     name is automatically synthesized from the machine name,"
357   puts "                     date, set of benchmarks run, and set of configurations."
358   puts "--environment        JSON file that specifies the environment variables that should"
359   puts "                     be used for particular VMs and benchmarks."
360   puts "--config <path>      Specify the path of the configuration file. Defaults to"
361   puts "                     ~/.run-jsc-benchmarks"
362   puts "--dependencies       Additional dependent library paths."
363   puts "--help or -h         Display this message."
364   puts
365   puts "Example:"
366   puts "run-jsc-benchmarks TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
367   exit 1
368 end
369
370 def fail(reason)
371   if reason.respond_to? :backtrace
372     puts "FAILED: #{reason.inspect}"
373     puts "Stack trace:"
374     puts reason.backtrace.join("\n")
375   else
376     puts "FAILED: #{reason.inspect}"
377   end
378   smallUsage
379 end
380
381 def quickFail(r1,r2)
382   $stderr.puts "#{$0}: #{r1}"
383   puts
384   fail(r2)
385 end
386
387 def intArg(argName,arg,min,max)
388   result=arg.to_i
389   unless result.to_s == arg
390     quickFail("Expected an integer value for #{argName}, but got #{arg}.",
391               "Invalid argument for command-line option")
392   end
393   if min and result<min
394     quickFail("Argument for #{argName} cannot be smaller than #{min}.",
395               "Invalid argument for command-line option")
396   end
397   if max and result>max
398     quickFail("Argument for #{argName} cannot be greater than #{max}.",
399               "Invalid argument for command-line option")
400   end
401   result
402 end
403
404 def computeMean(array)
405   sum=0.0
406   array.each {
407     | value |
408     sum += value
409   }
410   sum/array.length
411 end
412
413 def computeGeometricMean(array)
414   sum = 0.0
415   array.each {
416     | value |
417     sum += Math.log(value)
418   }
419   Math.exp(sum * (1.0/array.length))
420 end
421
422 def computeHarmonicMean(array)
423   1.0 / computeMean(array.collect{ | value | 1.0 / value })
424 end
425
426 def computeStdDev(array)
427   case array.length
428   when 0
429     0.0/0.0
430   when 1
431     0.0
432   else
433     begin
434       mean=computeMean(array)
435       sum=0.0
436       array.each {
437         | value |
438         sum += (value-mean)**2
439       }
440       Math.sqrt(sum/(array.length-1))
441     rescue
442       0.0/0.0
443     end
444   end
445 end
446
447 class Array
448   def shuffle!
449     size.downto(1) { |n| push delete_at(rand(n)) }
450     self
451   end
452 end
453
454 def inverseBetaRegularized(n)
455   IBR_LOOKUP[n-1]
456 end
457
458 def numToStr(num, decimalShift)
459   ("%." + (4 + decimalShift).to_s + "f") % (num.to_f)
460 end
461
462 class CantSay
463   def initialize
464   end
465   
466   def shortForm
467     " "
468   end
469   
470   def longForm
471     ""
472   end
473   
474   def to_s
475     ""
476   end
477 end
478   
479 class NoChange
480   attr_reader :amountFaster
481   
482   def initialize(amountFaster)
483     @amountFaster = amountFaster
484   end
485   
486   def shortForm
487     " "
488   end
489   
490   def longForm
491     "  might be #{numToStr(@amountFaster, 0)}x faster"
492   end
493   
494   def to_s
495     if @amountFaster < 1.01
496       ""
497     else
498       longForm
499     end
500   end
501 end
502
503 class Faster
504   attr_reader :amountFaster
505   
506   def initialize(amountFaster)
507     @amountFaster = amountFaster
508   end
509   
510   def shortForm
511     "^"
512   end
513   
514   def longForm
515     "^ definitely #{numToStr(@amountFaster, 0)}x faster"
516   end
517   
518   def to_s
519     longForm
520   end
521 end
522
523 class Slower
524   attr_reader :amountSlower
525   
526   def initialize(amountSlower)
527     @amountSlower = amountSlower
528   end
529   
530   def shortForm
531     "!"
532   end
533   
534   def longForm
535     "! definitely #{numToStr(@amountSlower, 0)}x slower"
536   end
537   
538   def to_s
539     longForm
540   end
541 end
542
543 class MayBeSlower
544   attr_reader :amountSlower
545   
546   def initialize(amountSlower)
547     @amountSlower = amountSlower
548   end
549   
550   def shortForm
551     "?"
552   end
553   
554   def longForm
555     "? might be #{numToStr(@amountSlower, 0)}x slower"
556   end
557   
558   def to_s
559     if @amountSlower < 1.01
560       "?"
561     else
562       longForm
563     end
564   end
565 end
566
567 def jsonSanitize(value)
568   if value.is_a? Fixnum
569     value
570   elsif value.is_a? Float
571     if value.nan? or value.infinite?
572       value.to_s
573     else
574       value
575     end
576   elsif value.is_a? Array
577     value.map{|v| jsonSanitize(v)}
578   elsif value.nil?
579     value
580   else
581     raise "Unrecognized value #{value.inspect}"
582   end
583 end
584
585 class Stats
586   def initialize
587     @array = []
588   end
589   
590   def add(value)
591     if not value or not @array
592       @array = nil
593     elsif value.is_a? Float
594       if value.nan? or value.infinite?
595         @array = nil
596       else
597         @array << value
598       end
599     elsif value.is_a? Stats
600       add(value.array)
601     elsif value.respond_to? :each
602       value.each {
603         | v |
604         add(v)
605       }
606     else
607       @array << value.to_f
608     end
609   end
610   
611   def status
612     if @array
613       :ok
614     else
615       :error
616     end
617   end
618   
619   def error?
620     # TODO: We're probably still not handling this case correctly. 
621     not @array or @array.empty?
622   end
623   
624   def ok?
625     not not @array
626   end
627     
628   def array
629     @array
630   end
631   
632   def sum
633     result=0
634     @array.each {
635       | value |
636       result += value
637     }
638     result
639   end
640   
641   def min
642     @array.min
643   end
644   
645   def max
646     @array.max
647   end
648   
649   def size
650     @array.length
651   end
652   
653   def mean
654     computeMean(array)
655   end
656   
657   def arithmeticMean
658     mean
659   end
660   
661   def stdDev
662     computeStdDev(array)
663   end
664
665   def stdErr
666     stdDev/Math.sqrt(size)
667   end
668   
669   # Computes a 95% Student's t distribution confidence interval
670   def confInt
671     if size < 2
672       0.0/0.0
673     else
674       raise if size > 1000
675       Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
676     end
677   end
678   
679   def lower
680     mean-confInt
681   end
682   
683   def upper
684     mean+confInt
685   end
686   
687   def geometricMean
688     computeGeometricMean(array)
689   end
690   
691   def harmonicMean
692     computeHarmonicMean(array)
693   end
694   
695   def compareTo(other)
696     return CantSay.new unless ok? and other.ok?
697     
698     if upper < other.lower
699       Faster.new(other.mean/mean)
700     elsif lower > other.upper
701       Slower.new(mean/other.mean)
702     elsif mean > other.mean
703       MayBeSlower.new(mean/other.mean)
704     else
705       NoChange.new(other.mean/mean)
706     end
707   end
708   
709   def to_s
710     "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
711   end
712   
713   def jsonMap
714     if ok?
715       {"data"=>jsonSanitize(@array), "mean"=>jsonSanitize(mean), "confInt"=>jsonSanitize(confInt)}
716     else
717       "ERROR"
718     end
719   end
720 end
721
722 def doublePuts(out1,out2,msg)
723   out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
724   out2.puts msg
725 end
726
727 class Benchfile < File
728   @@counter = 0
729   
730   attr_reader :filename, :basename
731   
732   def initialize(name)
733     @basename, @filename = Benchfile.uniqueFilename(name)
734     super(@filename, "w")
735   end
736   
737   def self.uniqueFilename(name)
738     if name.is_a? Array
739       basename = name[0] + @@counter.to_s + name[1]
740     else
741       basename = name + @@counter.to_s
742     end
743     filename = BENCH_DATA_PATH + basename
744     @@counter += 1
745     raise "Benchfile #{filename} already exists" if FileTest.exist?(filename)
746     [basename, filename]
747   end
748   
749   def self.create(name)
750     file = Benchfile.new(name)
751     yield file
752     file.close
753     file.basename
754   end
755 end
756
757 $dataFiles={}
758 def ensureFile(key, filename)
759   unless $dataFiles[key]
760     $dataFiles[key] = Benchfile.create(key) {
761       | outp |
762       doublePuts($stderr,outp,IO::read(filename))
763     }
764   end
765   $dataFiles[key]
766 end
767
768 # Helper for files that cannot be renamed.
769 $absoluteFiles={}
770 def ensureAbsoluteFile(filename, basedir=nil)
771   return if $absoluteFiles[filename]
772   filename = Pathname.new(filename)
773
774   directory = Pathname.new('')
775   if basedir and filename.dirname != basedir
776     remainingPath = filename.dirname
777     while remainingPath != basedir
778       directory = remainingPath.basename + directory
779       remainingPath = remainingPath.dirname
780     end
781     if not $absoluteFiles[directory]
782       cmd = "mkdir -p #{Shellwords.shellescape((BENCH_DATA_PATH + directory).to_s)}"
783       $stderr.puts ">> #{cmd}" if $verbosity >= 2
784       raise unless system(cmd)
785       intermediateDirectory = Pathname.new(directory)
786       while intermediateDirectory.basename.to_s != "."
787         $absoluteFiles[intermediateDirectory] = true
788         intermediateDirectory = intermediateDirectory.dirname
789       end
790     end
791   end
792   
793   cmd = "cp #{Shellwords.shellescape(filename.to_s)} #{Shellwords.shellescape((BENCH_DATA_PATH + directory + filename.basename).to_s)}"
794   $stderr.puts ">> #{cmd}" if $verbosity >= 2
795   raise unless system(cmd)
796   $absoluteFiles[filename] = true
797 end
798
799 # Helper for large benchmarks with lots of files and directories.
800 def ensureBenchmarkFiles(rootdir)
801     toProcess = [rootdir]
802     while not toProcess.empty?
803       currdir = toProcess.pop
804       Dir.foreach(currdir.to_s) {
805         | filename |
806         path = currdir + filename
807         next if filename.match(/^\./)
808         toProcess.push(path) if File.directory?(path.to_s)
809         ensureAbsoluteFile(path, rootdir) if File.file?(path.to_s)
810       }
811     end
812 end
813
814 class JSCommand
815   attr_reader :js, :html
816   def initialize(js, html)
817     @js = js
818     @html = html
819   end
820 end
821
822 def loadCommandForFile(key, filename)
823   file = ensureFile(key, filename)
824   JSCommand.new("load(#{file.inspect});", "<script src=#{file.inspect}></script>")
825 end
826
827 def simpleCommand(command)
828   JSCommand.new(command, "<script type=\"text/javascript\">#{command}</script>")
829 end
830
831 # Benchmark that consists of a single file and must be loaded in its own global object each
832 # time (i.e. run()).
833 class SingleFileTimedBenchmarkParameters
834   attr_reader :benchPath
835   
836   def initialize(benchPath)
837     @benchPath = benchPath
838   end
839   
840   def kind
841     :singleFileTimedBenchmark
842   end
843 end
844
845 # Benchmark that consists of a single file and must be loaded in its own global object each
846 # time (i.e. run()), but returns its result in a custom way.
847 class SingleFileCustomTimedBenchmarkParameters
848   attr_reader :benchPath
849   
850   def initialize(benchPath)
851     @benchPath = benchPath
852   end
853   
854   def kind
855     :singleFileCustomTimedBenchmark
856   end
857 end
858
859 # Benchmark that consists of one or more data files that should be loaded globally, followed
860 # by a command to run the benchmark.
861 class MultiFileTimedBenchmarkParameters
862   attr_reader :dataPaths, :command
863
864   def initialize(dataPaths, command)
865     @dataPaths = dataPaths
866     @command = command
867   end
868   
869   def kind
870     :multiFileTimedBenchmark
871   end
872 end
873
874 # Benchmark that consists of one or more data files that should be loaded globally, followed
875 # by a command to run a short tick of the benchmark. The benchmark should be run for as many
876 # ticks as possible, for one quantum (quantum is 1000ms by default).
877 class ThroughputBenchmarkParameters
878   attr_reader :dataPaths, :setUpCommand, :command, :tearDownCommand, :doWarmup, :deterministic, :minimumIterations
879
880   def initialize(dataPaths, setUpCommand, command, tearDownCommand, doWarmup, deterministic, minimumIterations)
881     @dataPaths = dataPaths
882     @setUpCommand = setUpCommand
883     @command = command
884     @tearDownCommand = tearDownCommand
885     @doWarmup = doWarmup
886     @deterministic = deterministic
887     @minimumIterations = minimumIterations
888   end
889   
890   def kind
891     :throughputBenchmark
892   end
893 end
894
895 # Benchmark that can only run in DumpRenderTree or WebKitTestRunner, that has its own callback for reporting
896 # results. Other than that it's just like SingleFileTimedBenchmark.
897 class SingleFileTimedCallbackBenchmarkParameters
898   attr_reader :callbackDecl, :benchPath
899   
900   def initialize(callbackDecl, benchPath)
901     @callbackDecl = callbackDecl
902     @benchPath = benchPath
903   end
904   
905   def kind
906     :singleFileTimedCallbackBenchmark
907   end
908 end
909
910 def emitTimerFunctionCode(file)
911   case $timeMode
912   when :preciseTime
913     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
914     doublePuts($stderr,file,"   return preciseTime()*1000")
915     doublePuts($stderr,file,"}")
916   when :date
917     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
918     doublePuts($stderr,file,"   return Date.now()")
919     doublePuts($stderr,file,"}")
920   else
921     raise
922   end
923 end
924
925 def emitBenchRunCodeFile(name, plan, benchParams)
926   case plan.vm.vmType
927   when :jsc
928     Benchfile.create("bencher") {
929       | file |
930       emitTimerFunctionCode(file)
931       
932       if benchParams.kind == :multiFileTimedBenchmark
933         benchParams.dataPaths.each {
934           | path |
935           doublePuts($stderr,file,"load(#{path.inspect});")
936         }
937         doublePuts($stderr,file,"gc();")
938         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
939         doublePuts($stderr,file,"   var __before = __bencher_curTimeMS();")
940         $rerun.times {
941           doublePuts($stderr,file,"   #{benchParams.command.js}")
942         }
943         doublePuts($stderr,file,"   var __after = __bencher_curTimeMS();")
944         doublePuts($stderr,file,"   if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(__after-__before));");
945         doublePuts($stderr,file,"   gc();") unless plan.vm.shouldMeasureGC
946         doublePuts($stderr,file,"}")
947       elsif benchParams.kind == :throughputBenchmark
948         emitTimerFunctionCode(file)
949         benchParams.dataPaths.each {
950           | path |
951           doublePuts($stderr,file,"load(#{path.inspect});")
952         }
953         doublePuts($stderr,file,"#{benchParams.setUpCommand.js}")
954         if benchParams.doWarmup
955           warmup = $warmup
956         else
957           warmup = 0
958         end
959         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{warmup + $inner}; __bencher_index++) {")
960         doublePuts($stderr,file,"    var __before = __bencher_curTimeMS();")
961         doublePuts($stderr,file,"    var __after = __before;")
962         doublePuts($stderr,file,"    var __runs = 0;")
963         doublePuts($stderr,file,"    var __expected = #{$quantum};")
964         doublePuts($stderr,file,"    while (true) {")
965         $rerun.times {
966           doublePuts($stderr,file,"       #{benchParams.command.js}")
967         }
968         doublePuts($stderr,file,"       __runs++;")
969         doublePuts($stderr,file,"       __after = __bencher_curTimeMS();")
970         if benchParams.deterministic
971           doublePuts($stderr,file,"       if (true) {")
972         else
973           doublePuts($stderr,file,"       if (__after - __before >= __expected) {")
974         end
975         doublePuts($stderr,file,"           if (__runs >= #{benchParams.minimumIterations} || __bencher_index < #{warmup})")
976         doublePuts($stderr,file,"               break;")
977         doublePuts($stderr,file,"           __expected += #{$quantum}")
978         doublePuts($stderr,file,"       }")
979         doublePuts($stderr,file,"    }")
980         doublePuts($stderr,file,"    if (__bencher_index >= #{warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{warmup}) + \": Time: \"+((__after-__before)/__runs));")
981         doublePuts($stderr,file,"}")
982         doublePuts($stderr,file,"#{benchParams.tearDownCommand.js}")
983       elsif benchParams.kind == :singleFileCustomTimedBenchmark
984         doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
985         doublePuts($stderr,file,"   var __bencher_result = {value: null};")
986         $rerun.times {
987           doublePuts($stderr,file,"   run(__bencher_what, __bencher_result);")
988         }
989         doublePuts($stderr,file,"   if (__bencher_result.value == null)")
990         doublePuts($stderr,file,"      throw new Error(\"Null result\");")
991         doublePuts($stderr,file,"   return __bencher_result.value;")
992         doublePuts($stderr,file,"}")
993         $warmup.times {
994           doublePuts($stderr,file,"__bencher_run(#{benchParams.benchPath.inspect})")
995           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
996         }
997         $inner.times {
998           | innerIndex |
999           doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchParams.benchPath.inspect}));")
1000           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
1001         }
1002       else
1003         raise unless benchParams.kind == :singleFileTimedBenchmark
1004         doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
1005         doublePuts($stderr,file,"   var __bencher_before = __bencher_curTimeMS();")
1006         $rerun.times {
1007           doublePuts($stderr,file,"   run(__bencher_what);")
1008         }
1009         doublePuts($stderr,file,"   var __bencher_after = __bencher_curTimeMS();")
1010         doublePuts($stderr,file,"   return __bencher_after - __bencher_before;")
1011         doublePuts($stderr,file,"}")
1012         $warmup.times {
1013           doublePuts($stderr,file,"__bencher_run(#{benchParams.benchPath.inspect})")
1014           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
1015         }
1016         $inner.times {
1017           | innerIndex |
1018           doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchParams.benchPath.inspect}));")
1019           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
1020         }
1021       end
1022     }
1023   when :dumpRenderTree, :webkitTestRunner
1024     case $timeMode
1025     when :preciseTime
1026       curTime = "(testRunner.preciseTime()*1000)"
1027     when :date
1028       curTime = "(Date.now())"
1029     else
1030       raise
1031     end
1032
1033     mainCode = Benchfile.create("bencher") {
1034       | file |
1035       doublePuts($stderr,file,"__bencher_count = 0;")
1036       doublePuts($stderr,file,"function __bencher_doNext(result) {")
1037       doublePuts($stderr,file,"    if (__bencher_count >= #{$warmup})")
1038       doublePuts($stderr,file,"        debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);")
1039       doublePuts($stderr,file,"    __bencher_count++;")
1040       doublePuts($stderr,file,"    if (__bencher_count < #{$inner+$warmup})")
1041       doublePuts($stderr,file,"        __bencher_runImpl(__bencher_doNext);")
1042       doublePuts($stderr,file,"    else")
1043       doublePuts($stderr,file,"        quit();")
1044       doublePuts($stderr,file,"}")
1045       doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);")
1046     }
1047     
1048     cssCode = Benchfile.create("bencher-css") {
1049       | file |
1050       doublePuts($stderr,file,".pass {\n    font-weight: bold;\n    color: green;\n}\n.fail {\n    font-weight: bold;\n    color: red;\n}\n\#console {\n    white-space: pre-wrap;\n    font-family: monospace;\n}")
1051     }
1052     
1053     preCode = Benchfile.create("bencher-pre") {
1054       | file |
1055       doublePuts($stderr,file,"if (window.testRunner) {")
1056       doublePuts($stderr,file,"    testRunner.dumpAsText(window.enablePixelTesting);")
1057       doublePuts($stderr,file,"    testRunner.waitUntilDone();")
1058       doublePuts($stderr,file,"}")
1059       doublePuts($stderr,file,"")
1060       doublePuts($stderr,file,"function debug(msg)")
1061       doublePuts($stderr,file,"{")
1062       doublePuts($stderr,file,"    var span = document.createElement(\"span\");")
1063       doublePuts($stderr,file,"    document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace")
1064       doublePuts($stderr,file,"    span.innerHTML = msg + '<br />';")
1065       doublePuts($stderr,file,"}")
1066       doublePuts($stderr,file,"")
1067       doublePuts($stderr,file,"function quit() {")
1068       doublePuts($stderr,file,"    testRunner.notifyDone();")
1069       doublePuts($stderr,file,"}")
1070       doublePuts($stderr,file,"")
1071       doublePuts($stderr,file,"__bencher_continuation=null;")
1072       doublePuts($stderr,file,"")
1073       doublePuts($stderr,file,"function reportResult(result) {")
1074       doublePuts($stderr,file,"    __bencher_continuation(result);")
1075       doublePuts($stderr,file,"}")
1076       doublePuts($stderr,file,"")
1077       doublePuts($stderr,file,"function currentTimeInMS(msg)")
1078       doublePuts($stderr,file,"{")
1079       doublePuts($stderr,file,"    return #{curTime};")
1080       doublePuts($stderr,file,"}")
1081       if benchParams.kind == :singleFileTimedCallbackBenchmark
1082         doublePuts($stderr,file,"")
1083         doublePuts($stderr,file,benchParams.callbackDecl)
1084       end
1085       doublePuts($stderr,file,"")
1086       doublePuts($stderr,file,"function __bencher_runImpl(continuation) {")
1087       doublePuts($stderr,file,"    function doit() {")
1088       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"\";")
1089       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";")
1090       doublePuts($stderr,file,"        var testFrame = document.getElementById(\"testframe\");")
1091       doublePuts($stderr,file,"        testFrame.contentDocument.open();")
1092       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");")
1093       if benchParams.kind == :throughputBenchmark or benchParams.kind == :multiFileTimedBenchmark
1094         benchParams.dataPaths.each {
1095           | path |
1096           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{path.inspect.inspect[1..-2]}></script>\");")
1097         }
1098       end
1099       if benchParams.kind == :throughputBenchmark
1100         if benchParams.doWarmup
1101           warmup = $warmup
1102         else
1103           warmup = 0
1104         end
1105         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">\");")
1106         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.setUpCommand.js}\");")
1107         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_before = #{curTime};\");")
1108         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_after = __bencher_before;\");")
1109         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_expected = #{$quantum};\");")
1110         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_runs = 0;\");")
1111         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"while (true) {\");")
1112         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    #{benchParams.command.js}\");")
1113         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_runs++;\");")
1114         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_after = #{curTime};\");")
1115         if benchParams.deterministic
1116           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (true) {\");")
1117         else
1118           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (__bencher_after - __bencher_before >= __bencher_expected) {\");")
1119         end
1120         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        if (__bencher_runs >= #{benchParams.minimumIterations} || window.parent.__bencher_count < #{warmup})\");")
1121         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"            break;\");")
1122         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        __bencher_expected += #{$quantum}\");")
1123         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    }\");")
1124         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"}\");")
1125         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.tearDownCommand.js}\");")
1126         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"window.parent.reportResult((__bencher_after - __bencher_before) / __bencher_runs);\");")
1127         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</script>\");")
1128       else
1129         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">var __bencher_before = #{curTime};</script>\");")
1130         if benchParams.kind == :multiFileTimedBenchmark
1131           doublePuts($stderr,file,"        testFrame.contentDocument.write(#{benchParams.command.html.inspect});")
1132         else
1133           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{benchParams.benchPath.inspect.inspect[1..-2]}></script>\");")
1134         end
1135         unless benchParams.kind == :singleFileTimedCallbackBenchmark
1136           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">window.parent.reportResult(#{curTime} - __bencher_before);</script>\");")
1137         end
1138       end
1139       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</body></html>\");")
1140       doublePuts($stderr,file,"        testFrame.contentDocument.close();")
1141       doublePuts($stderr,file,"    }")
1142       doublePuts($stderr,file,"    __bencher_continuation = continuation;")
1143       doublePuts($stderr,file,"    window.setTimeout(doit, 10);")
1144       doublePuts($stderr,file,"}")
1145     }
1146
1147     Benchfile.create(["bencher-htmldoc",".html"]) {
1148       | file |
1149       doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>")
1150     }
1151   else
1152     raise
1153   end
1154 end
1155
1156 def emitBenchRunCode(name, plan, benchParams)
1157   plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchParams), plan)
1158 end
1159
1160 class FileCreator
1161   def initialize(filename)
1162     @filename = filename
1163     @state = :empty
1164   end
1165   
1166   def puts(text)
1167     $script.print "echo #{Shellwords.shellescape(text)}"
1168     if @state == :empty
1169       $script.print " > "
1170       @state = :nonEmpty
1171     else
1172       $script.print " >> "
1173     end
1174     $script.puts "#{Shellwords.shellescape(@filename)}"
1175   end
1176   
1177   def close
1178     if @state == :empty
1179       $script.puts "rm -f #{Shellwords.shellescape(text)}"
1180       $script.puts "touch #{Shellwords.shellescape(text)}"
1181     end
1182   end
1183   
1184   def self.open(filename)
1185     outp = FileCreator.new(filename)
1186     yield outp
1187     outp.close
1188   end
1189 end
1190
1191 def emitSelfContainedBenchRunCode(name, plan, targetFile, configFile, benchmark)
1192   FileCreator.open(configFile) {
1193     | outp |
1194     outp.puts "__bencher_message = \"#{name}: #{plan.vm}: #{plan.iteration}: \";"
1195     outp.puts "__bencher_warmup = #{$warmup};"
1196     outp.puts "__bencher_inner = #{$inner};"
1197     outp.puts "__bencher_benchmark = #{benchmark.to_json};"
1198     case $timeMode
1199     when :preciseTime
1200       outp.puts "__bencher_curTime = (function(){ return testRunner.preciseTime() * 1000; });"
1201     when :date
1202       outp.puts "__bencher_curTime = (function(){ return Date.now(); });"
1203     else
1204       raise
1205     end
1206   }
1207   
1208   plan.vm.emitRunCode(targetFile, plan)
1209 end
1210
1211 def planForDescription(string, plans, benchFullname, vmName, iteration)
1212   raise "Unexpected benchmark full name: #{benchFullname.inspect}, string: #{string.inspect}" unless benchFullname =~ /\//
1213   suiteName = $~.pre_match
1214   return nil if suiteName == "WARMUP"
1215   benchName = $~.post_match
1216   result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration}
1217   raise "Unexpected result dimensions: #{result.inspect}, string: #{string.inspect}" unless result.size == 1
1218   result[0]
1219 end
1220
1221 class ParsedResult
1222   attr_reader :plan, :innerIndex, :time, :result
1223   
1224   def initialize(plan, innerIndex, time)
1225     @plan = plan
1226     @innerIndex = innerIndex
1227     if time == :crashed
1228       @result = :error
1229     else
1230       @time = time
1231       @result = :success
1232     end
1233     
1234     raise unless @plan.is_a? BenchPlan
1235     raise unless @innerIndex.is_a? Integer
1236     raise unless @time.is_a? Numeric or @result == :error
1237   end
1238   
1239   def benchmark
1240     plan.benchmark
1241   end
1242   
1243   def suite
1244     plan.suite
1245   end
1246   
1247   def vm
1248     plan.vm
1249   end
1250   
1251   def outerIndex
1252     plan.iteration
1253   end
1254   
1255   def self.create(plan, innerIndex, time)
1256     if plan
1257       ParsedResult.new(plan, innerIndex, time)
1258     else
1259       nil
1260     end
1261   end
1262   
1263   def self.parse(plans, string)
1264     if string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): Time: /
1265       benchFullname = $1
1266       vmName = $2
1267       outerIndex = $3.to_i
1268       innerIndex = $4.to_i
1269       time = $~.post_match.to_f
1270       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, time)
1271     elsif string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): CRASHED/
1272       benchFullname = $1
1273       vmName = $2
1274       outerIndex = $3.to_i
1275       innerIndex = $4.to_i
1276       time = $~.post_match.to_f
1277       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, :crashed)
1278     else
1279       nil
1280     end
1281   end
1282 end
1283
1284 class VM
1285   @@extraEnvSet = {}
1286   
1287   def initialize(origPath, name, nameKind, svnRevision)
1288     @origPath = origPath.to_s
1289     @path = origPath.to_s
1290     @name = name
1291     @nameKind = nameKind
1292     @extraEnv = {}
1293     
1294     if $forceVMKind
1295       @vmType = $forceVMKind
1296     else
1297       if @origPath =~ /DumpRenderTree$/
1298         @vmType = :dumpRenderTree
1299       elsif @origPath =~ /WebKitTestRunner$/
1300         @vmType = :webkitTestRunner
1301       else
1302         @vmType = :jsc
1303       end
1304     end
1305     
1306     @svnRevision = svnRevision
1307     
1308     # Try to detect information about the VM.
1309     if path =~ /\/WebKitBuild\/(Release|Debug)+\/([a-zA-Z]+)$/
1310       @checkoutPath = $~.pre_match
1311       # FIXME: Use some variant of this: 
1312       # <bdash>   def retrieve_revision
1313       # <bdash>     `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i
1314       # <bdash>   end
1315       unless @svnRevision
1316         begin
1317           Dir.chdir(@checkoutPath) {
1318             $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
1319             IO.popen("svn info", "r") {
1320               | inp |
1321               inp.each_line {
1322                 | line |
1323                 if line =~ /Revision: ([0-9]+)/
1324                   @svnRevision = $1
1325                 end
1326               }
1327             }
1328           }
1329           unless @svnRevision
1330             $stderr.puts "Warning: running svn info for #{name} silently failed."
1331           end
1332         rescue => e
1333           # Failed to detect svn revision.
1334           $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
1335         end
1336       end
1337     else
1338       $stderr.puts "Warning: could not identify checkout location for #{name}"
1339     end
1340     
1341     if @path =~ /\/Release\/([a-zA-Z]+)$/
1342       @libPath, @relativeBinPath = [$~.pre_match+"/Release"], "./#{$1}"
1343     elsif @path =~ /\/Debug\/([a-zA-Z]+)$/
1344       @libPath, @relativeBinPath = [$~.pre_match+"/Debug"], "./#{$1}"
1345     elsif @path =~ /\/Release\/bin\/([a-zA-Z]+)$/
1346       @libPath, @relativeBinPath = [$~.pre_match+"/Release/lib"], "./#{$1}"
1347     elsif @path =~ /\/Debug\/bin\/([a-zA-Z]+)$/
1348       @libPath, @relativeBinPath = [$~.pre_match+"/Debug/lib"], "./#{$1}"
1349     elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/
1350       @libPath = [$~.pre_match + "/Contents/Resources", $~.pre_match + "/Contents/Frameworks"]
1351     elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/
1352       @libPath, @relativeBinPath = [$~.pre_match], $&[1..-1]
1353     elsif @path =~ /(DumpRenderTree|webkitTestRunner|jsc)$/
1354       @libPath, @relativeBinPath = [$~.pre_match+"/"], "./#{$1}"
1355     end
1356     @libPath += $dependencies
1357   end
1358   
1359   def canCopyIntoBenchPath
1360     if @libPath and @relativeBinPath
1361       true
1362     else
1363       false
1364     end
1365   end
1366   
1367   def addExtraEnv(key, val)
1368     @extraEnv[key] = val
1369     @@extraEnvSet[key] = true
1370   end
1371   
1372   def copyIntoBenchPath
1373     raise unless canCopyIntoBenchPath
1374     basename, filename = Benchfile.uniqueFilename("vm")
1375     raise unless Dir.mkdir(filename)
1376     @libPath.each {
1377       | libPathPart |
1378       cmd = "cp -a #{Shellwords.shellescape(libPathPart)}/* #{Shellwords.shellescape(filename.to_s)}"
1379       $stderr.puts ">> #{cmd}" if $verbosity>=2
1380       raise unless system(cmd)
1381     }
1382     @path = "#{basename}/#{@relativeBinPath}"
1383     @libPath = [basename]
1384   end
1385   
1386   def to_s
1387     @name
1388   end
1389   
1390   def name
1391     @name
1392   end
1393   
1394   def shouldMeasureGC
1395     $measureGC == true or ($measureGC == name)
1396   end
1397   
1398   def origPath
1399     @origPath
1400   end
1401   
1402   def path
1403     @path
1404   end
1405   
1406   def nameKind
1407     @nameKind
1408   end
1409   
1410   def vmType
1411     @vmType
1412   end
1413   
1414   def checkoutPath
1415     @checkoutPath
1416   end
1417   
1418   def svnRevision
1419     @svnRevision
1420   end
1421   
1422   def extraEnv
1423     @extraEnv
1424   end
1425   
1426   def printFunction
1427     case @vmType
1428     when :jsc
1429       "print"
1430     when :dumpRenderTree, :webkitTestRunner
1431       "debug"
1432     else
1433       raise @vmType
1434     end
1435   end
1436   
1437   def emitRunCode(fileToRun, plan)
1438     myLibPath = @libPath
1439     myLibPath = [] unless myLibPath
1440     @@extraEnvSet.keys.each {
1441       | key |
1442       $script.puts "unset #{Shellwords.shellescape(key)}"
1443     }
1444     $script.puts "export JSC_useDollarVM=true" if @vmType == :jsc
1445     $script.puts "export DYLD_LIBRARY_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1446     $script.puts "export DYLD_FRAMEWORK_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1447     $script.puts "export LD_LIBRARY_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1448
1449     unless myLibPath.empty?
1450         primaryLibPath = myLibPath.first
1451         $script.puts "export TEST_RUNNER_TEST_PLUGIN_PATH=#{Shellwords.shellescape(Pathname.new(primaryLibPath).join('plugins').to_s)}"
1452         $script.puts "export TEST_RUNNER_INJECTED_BUNDLE_FILENAME=#{Shellwords.shellescape(Pathname.new(primaryLibPath).join('libTestRunnerInjectedBundle.so').to_s)}"
1453     end
1454
1455     @extraEnv.each_pair {
1456       | key, val |
1457       $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1458     }
1459     plan.environment.each_pair {
1460         | key, val |
1461         $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1462     }
1463     $script.puts "#{path} #{fileToRun} 2>&1 || {"
1464     $script.puts "    echo " + Shellwords.shellescape("#{name} failed to run!") + " 1>&2"
1465     $inner.times {
1466       | iteration |
1467       $script.puts "    echo " + Shellwords.shellescape("#{plan.prefix}: #{iteration}: CRASHED")
1468     }
1469     $script.puts "}"
1470     plan.environment.keys.each {
1471         | key |
1472         $script.puts "export #{Shellwords.shellescape(key)}="
1473     }
1474   end
1475 end
1476
1477 class StatsAccumulator
1478   def initialize
1479     @stats = []
1480     ($outer*$inner).times {
1481       @stats << Stats.new
1482     }
1483   end
1484   
1485   def statsForIteration(outerIteration, innerIteration)
1486     @stats[outerIteration*$inner + innerIteration]
1487   end
1488   
1489   def stats
1490     result = Stats.new
1491     @stats.each {
1492       | stat |
1493       if stat.ok?
1494         result.add(yield stat)
1495       else
1496         result.add(nil)
1497       end
1498     }
1499     result
1500   end
1501   
1502   def geometricMeanStats
1503     stats {
1504       | stat |
1505       stat.geometricMean
1506     }
1507   end
1508   
1509   def arithmeticMeanStats
1510     stats {
1511       | stat |
1512       stat.arithmeticMean
1513     }
1514   end
1515 end
1516
1517 module Benchmark
1518   attr_accessor :benchmarkSuite
1519   attr_reader :name
1520   
1521   def fullname
1522     benchmarkSuite.name + "/" + name
1523   end
1524   
1525   def to_s
1526     fullname
1527   end
1528   
1529   def weight
1530     1
1531   end
1532   
1533   def weightString
1534     raise unless weight.is_a? Fixnum
1535     raise unless weight >= 1
1536     if weight == 1
1537       ""
1538     else
1539       "x#{weight} "
1540     end
1541   end
1542 end
1543
1544 class SunSpiderBenchmark
1545   include Benchmark
1546   
1547   def initialize(name)
1548     @name = name
1549   end
1550   
1551   def emitRunCode(plan)
1552     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js")))
1553   end
1554 end
1555
1556 class LongSpiderBenchmark
1557   include Benchmark
1558   
1559   def initialize(name)
1560     @name = name
1561   end
1562   
1563   def emitRunCode(plan)
1564     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("LongSpider-#{@name}", "#{LONGSPIDER_PATH}/#{@name}.js")))
1565   end
1566 end
1567
1568 class V8Benchmark
1569   include Benchmark
1570   
1571   def initialize(name)
1572     @name = name
1573   end
1574   
1575   def emitRunCode(plan)
1576     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js")))
1577   end
1578 end
1579
1580 class V8RealBenchmark
1581   include Benchmark
1582   
1583   attr_reader :v8SuiteName
1584   
1585   def initialize(v8SuiteName, name, weight, minimumIterations)
1586     @v8SuiteName = v8SuiteName
1587     @name = name
1588     @weight = weight
1589     @minimumIterations = minimumIterations
1590   end
1591   
1592   def weight
1593     @weight
1594   end
1595   
1596   def emitRunCode(plan)
1597     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(["base", @v8SuiteName, "jsc-#{@name}"].collect{|v| ensureFile("V8Real-#{v}", "#{V8_REAL_PATH}/#{v}.js")}, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, false, @minimumIterations))
1598   end
1599 end
1600
1601 class OctaneBenchmark
1602   include Benchmark
1603   
1604   def initialize(files, name, weight, doWarmup, deterministic, minimumIterations)
1605     @files = files
1606     @name = name
1607     @weight = weight
1608     @doWarmup = doWarmup
1609     @deterministic = deterministic
1610     @minimumIterations = minimumIterations
1611   end
1612   
1613   def weight
1614     @weight
1615   end
1616   
1617   def emitRunCode(plan)
1618     files = []
1619     files += (["base"] + @files).collect {
1620       | v |
1621       ensureFile("Octane-#{v}", "#{OCTANE_PATH}/#{v}.js")
1622     }
1623     files += ["jsc-#{@name}"].collect {
1624       | v |
1625       ensureFile("Octane-#{v}", "#{OCTANE_WRAPPER_PATH}/#{v}.js")
1626     }
1627     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(files, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1628   end
1629 end
1630
1631 class CustomTimedBenchmark
1632   include Benchmark
1633   
1634   def initialize(name, fullPath)
1635     @name = name
1636     @fullPath = fullPath
1637   end
1638   
1639   def emitRunCode(plan)
1640     emitBenchRunCode(fullname, plan, SingleFileCustomTimedBenchmarkParameters.new(ensureFile("CustomTimed-#{@name}", @fullPath)))
1641   end
1642 end
1643
1644 class KrakenBenchmark
1645   include Benchmark
1646   
1647   def initialize(name)
1648     @name = name
1649   end
1650   
1651   def emitRunCode(plan)
1652     emitBenchRunCode(fullname, plan, MultiFileTimedBenchmarkParameters.new([ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js")], loadCommandForFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js")))
1653   end
1654 end
1655
1656 class JSBenchBenchmark
1657   include Benchmark
1658   
1659   attr_reader :jsBenchMode
1660   
1661   def initialize(name, jsBenchMode)
1662     @name = name
1663     @jsBenchMode = jsBenchMode
1664   end
1665   
1666   def emitRunCode(plan)
1667     callbackDecl  = "function JSBNG_handleResult(result) {\n"
1668     callbackDecl += "    if (result.error) {\n"
1669     callbackDecl += "        console.log(\"Did not run benchmark correctly!\");\n"
1670     callbackDecl += "        quit();\n"
1671     callbackDecl += "    }\n"
1672     callbackDecl += "    reportResult(result.time);\n"
1673     callbackDecl += "}\n"
1674     emitBenchRunCode(fullname, plan, SingleFileTimedCallbackBenchmarkParameters.new(callbackDecl, ensureFile("JSBench-#{@name}", "#{JSBENCH_PATH}/#{@name}/#{@jsBenchMode}.js")))
1675   end
1676 end
1677
1678 class TailBenchBenchmark
1679   include Benchmark
1680
1681   def initialize(name)
1682     @name = name
1683   end
1684
1685   def emitRunCode(plan)
1686     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("TailBench-#{@name}", "#{TAILBENCH_PATH}/#{@name}.js")))
1687   end
1688 end
1689
1690 class MicrobenchmarksBenchmark
1691   include Benchmark
1692   
1693   def initialize(name)
1694     @name = name
1695   end
1696   
1697   def emitRunCode(plan)
1698     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("Microbenchmarks-#{@name}", "#{MICROBENCHMARKS_PATH}/#{@name}.js")))
1699   end
1700 end
1701
1702 class AsmBenchBenchmark
1703   include Benchmark
1704   
1705   def initialize(name)
1706     @name = name
1707   end
1708   
1709   def emitRunCode(plan)
1710     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("AsmBench-#{@name}", "#{ASMBENCH_PATH}/#{@name}.js")))
1711   end
1712 end
1713
1714 class SixSpeedBenchmark
1715   include Benchmark
1716
1717   def initialize(name, path, iterations)
1718     @name = name
1719     @path = path
1720     @iterations = iterations
1721   end
1722
1723   def emitRunCode(plan)
1724     emitBenchRunCode(fullname, plan, MultiFileTimedBenchmarkParameters.new([ensureFile("SixSpeed-#{@name}-wrapper", "#{SIXSPEED_WRAPPER_PATH}/wrapper.js"), ensureFile("SixSpeed-#{@name}", "#{SIXSPEED_PATH}/#{@path}")], simpleCommand("jscRun(#{@iterations});")))
1725   end
1726 end
1727
1728
1729 class CompressionBenchBenchmark
1730   include Benchmark
1731   
1732   def initialize(files, name, model)
1733     @files = files
1734     @name = name;
1735     @name = name + "-" + model if !model.empty?
1736     @name = @name.gsub(" ", "-").downcase
1737     @scriptName = name
1738     @weight = 1
1739     @doWarmup = true
1740     @deterministic = true
1741     @minimumIterations = 1
1742     @model = model
1743   end
1744   
1745   def weight
1746     @weight
1747   end
1748   
1749   def emitRunCode(plan)
1750     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new((["base"] + @files + ["jsc-#{@scriptName}"]).collect{|v| ensureFile("Compression-#{v}", "#{COMPRESSIONBENCH_PATH}/#{v}.js")}, simpleCommand("jscSetUp('#{@model}');"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1751   end
1752 end
1753
1754 class DSPJSFiltrrBenchmark
1755   include Benchmark
1756   
1757   def initialize(name, filterKey)
1758     @name = name
1759     @filterKey = filterKey
1760   end
1761   
1762   def emitRunCode(plan)
1763     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr.js")
1764     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr_back.jpg")
1765     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-jquery.min.js")
1766     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-bencher.html")
1767     emitSelfContainedBenchRunCode(fullname, plan, "filtrr-bencher.html", "bencher-config.js", @filterKey)
1768   end
1769 end
1770
1771 class DSPJSVP8Benchmark
1772   include Benchmark
1773   
1774   def initialize
1775     @name = "route9-vp8"
1776   end
1777   
1778   def weight
1779     5
1780   end
1781   
1782   def emitRunCode(plan)
1783     ensureBenchmarkFiles(DSPJS_ROUTE9_PATH)
1784     emitSelfContainedBenchRunCode(fullname, plan, "route9-bencher.html", "bencher-config.js", "")
1785   end
1786 end
1787
1788 class DSPStarfieldBenchmark
1789   include Benchmark
1790
1791   def initialize
1792     @name = "starfield"
1793   end
1794   
1795   def weight
1796     5
1797   end
1798
1799   def emitRunCode(plan)
1800     ensureBenchmarkFiles(DSPJS_STARFIELD_PATH)
1801     emitSelfContainedBenchRunCode(fullname, plan, "starfield-bencher.html", "bencher-config.js", "")
1802   end
1803 end
1804
1805 class DSPJSJSLinuxBenchmark
1806   include Benchmark
1807   def initialize
1808     @name = "bellard-jslinux"
1809   end
1810
1811   def weight
1812     5
1813   end
1814
1815   def emitRunCode(plan)
1816     ensureBenchmarkFiles(DSPJS_JSLINUX_PATH)
1817     emitSelfContainedBenchRunCode(fullname, plan, "jslinux-bencher.html", "bencher-config.js", "")
1818   end
1819 end
1820
1821 class DSPJSQuake3Benchmark
1822   include Benchmark
1823
1824   def initialize
1825     @name = "zynaps-quake3"
1826   end
1827
1828   def weight
1829     5
1830   end
1831
1832   def emitRunCode(plan)
1833     ensureBenchmarkFiles(DSPJS_QUAKE3_PATH)
1834     emitSelfContainedBenchRunCode(fullname, plan, "quake-bencher.html", "bencher-config.js", "")
1835   end
1836 end
1837
1838 class DSPJSMandelbrotBenchmark
1839   include Benchmark
1840
1841   def initialize
1842     @name = "zynaps-mandelbrot"
1843   end
1844
1845   def weight
1846     5
1847   end
1848
1849   def emitRunCode(plan)
1850     ensureBenchmarkFiles(DSPJS_MANDELBROT_PATH)
1851     emitSelfContainedBenchRunCode(fullname, plan, "mandelbrot-bencher.html", "bencher-config.js", "")
1852   end
1853 end
1854
1855 class DSPJSAmmoJSASMBenchmark
1856   include Benchmark
1857
1858   def initialize
1859     @name = "ammojs-asm-js"
1860   end
1861
1862   def weight
1863     5
1864   end
1865
1866   def emitRunCode(plan)
1867     ensureBenchmarkFiles(DSPJS_AMMOJS_ASMJS_PATH)
1868     emitSelfContainedBenchRunCode(fullname, plan, "ammo-asmjs-bencher.html", "bencher-config.js", "")
1869   end
1870 end
1871
1872 class DSPJSAmmoJSRegularBenchmark
1873   include Benchmark
1874
1875   def initialize
1876     @name = "ammojs-regular-js"
1877   end
1878
1879   def weight
1880     5
1881   end
1882
1883   def emitRunCode(plan)
1884     ensureBenchmarkFiles(DSPJS_AMMOJS_REGULAR_PATH)
1885     emitSelfContainedBenchRunCode(fullname, plan, "ammo-regular-bencher.html", "bencher-config.js", "")
1886   end
1887 end
1888
1889 class BrowsermarkJSBenchmark
1890   include Benchmark
1891     
1892   def initialize(name)
1893     @name = name
1894   end
1895   
1896   def emitRunCode(plan)
1897     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new([ensureFile(name, "#{BROWSERMARK_JS_PATH}/#{name}/test.js"), ensureFile("browsermark-bencher", "#{BROWSERMARK_JS_PATH}/browsermark-bencher.js")], simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, 32))
1898   end
1899 end
1900
1901 class BrowsermarkDOMBenchmark
1902   include Benchmark
1903     
1904   def initialize(name)
1905     @name = name
1906   end
1907   
1908   def emitRunCode(plan)
1909     ensureBenchmarkFiles(BROWSERMARK_PATH)
1910     emitSelfContainedBenchRunCode(fullname, plan, "tests/benchmarks/dom/#{name}/index.html", "bencher-config.js", name)
1911   end
1912 end
1913
1914 class BenchmarkSuite
1915   def initialize(name, preferredMean, decimalShift)
1916     @name = name
1917     @preferredMean = preferredMean
1918     @benchmarks = []
1919     @subSuites = []
1920     @decimalShift = decimalShift
1921   end
1922   
1923   def name
1924     @name
1925   end
1926   
1927   def to_s
1928     @name
1929   end
1930   
1931   def decimalShift
1932     @decimalShift
1933   end
1934   
1935   def addIgnoringPattern(benchmark)
1936     benchmark.benchmarkSuite = self
1937     @benchmarks << benchmark
1938   end
1939   
1940   def add(benchmark)
1941     if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern
1942         addIgnoringPattern(benchmark)
1943     end
1944   end
1945   
1946   def addSubSuite(subSuite)
1947     @subSuites << subSuite
1948   end
1949   
1950   def benchmarks
1951     @benchmarks
1952   end
1953   
1954   def benchmarkForName(name)
1955     result = @benchmarks.select{|v| v.name == name}
1956     raise unless result.length == 1
1957     result[0]
1958   end
1959   
1960   def hasBenchmark(benchmark)
1961     array = @benchmarks.select{|v| v == benchmark}
1962     raise unless array.length == 1 or array.length == 0
1963     array.length == 1
1964   end
1965   
1966   def subSuites
1967     @subSuites
1968   end
1969   
1970   def suites
1971     [self] + @subSuites
1972   end
1973   
1974   def suitesWithBenchmark(benchmark)
1975     result = [self]
1976     @subSuites.each {
1977       | subSuite |
1978       if subSuite.hasBenchmark(benchmark)
1979         result << subSuite
1980       end
1981     }
1982     result
1983   end
1984   
1985   def empty?
1986     @benchmarks.empty?
1987   end
1988   
1989   def retain_if
1990     @benchmarks.delete_if {
1991       | benchmark |
1992       not yield benchmark
1993     }
1994   end
1995   
1996   def preferredMean
1997     @preferredMean
1998   end
1999   
2000   def computeMean(stat)
2001     if stat.ok?
2002       (stat.send @preferredMean) * (10 ** decimalShift)
2003     else
2004       nil
2005     end
2006   end
2007 end
2008
2009 class BenchRunPlan
2010   def initialize(benchmark, vm, iteration)
2011     @benchmark = benchmark
2012     @vm = vm
2013     @iteration = iteration
2014     @environment = {}
2015     if $environment.has_key?(vm.name)
2016       if $environment[vm.name].has_key?(benchmark.benchmarkSuite.name)
2017         if $environment[vm.name][benchmark.benchmarkSuite.name].has_key?(benchmark.name)
2018           @environment = $environment[vm.name][benchmark.benchmarkSuite.name][benchmark.name]
2019         end
2020       end
2021     end
2022   end
2023   
2024   def benchmark
2025     @benchmark
2026   end
2027   
2028   def suite
2029     @benchmark.benchmarkSuite
2030   end
2031   
2032   def vm
2033     @vm
2034   end
2035   
2036   def iteration
2037     @iteration
2038   end
2039  
2040   def environment
2041     @environment
2042   end
2043  
2044   def prefix
2045     "#{@benchmark.fullname}: #{vm.name}: #{iteration}"
2046   end
2047   
2048   def emitRunCode
2049     @benchmark.emitRunCode(self)
2050   end
2051   
2052   def to_s
2053     benchmark.to_s + "/" + vm.to_s
2054   end
2055 end
2056
2057 class BenchmarkOnVM
2058   def initialize(benchmark, suiteOnVM, subSuitesOnVM)
2059     @benchmark = benchmark
2060     @suiteOnVM = suiteOnVM
2061     @subSuitesOnVM = subSuitesOnVM
2062     @stats = Stats.new
2063   end
2064   
2065   def to_s
2066     "#{@benchmark} on #{@suiteOnVM.vm}"
2067   end
2068   
2069   def benchmark
2070     @benchmark
2071   end
2072   
2073   def vm
2074     @suiteOnVM.vm
2075   end
2076   
2077   def vmStats
2078     @suiteOnVM.vmStats
2079   end
2080   
2081   def suite
2082     @benchmark.benchmarkSuite
2083   end
2084   
2085   def suiteOnVM
2086     @suiteOnVM
2087   end
2088   
2089   def subSuitesOnVM
2090     @subSuitesOnVM
2091   end
2092   
2093   def stats
2094     @stats
2095   end
2096   
2097   def parseResult(result)
2098     raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm
2099     raise unless result.benchmark == @benchmark
2100     @stats.add(result.time)
2101   end
2102 end
2103
2104 class NamedStatsAccumulator < StatsAccumulator
2105   def initialize(name)
2106     super()
2107     @name = name
2108   end
2109   
2110   def reportingName
2111     @name
2112   end
2113 end
2114
2115 class SuiteOnVM < StatsAccumulator
2116   def initialize(vm, vmStats, suite)
2117     super()
2118     @vm = vm
2119     @vmStats = vmStats
2120     @suite = suite
2121     
2122     raise unless @vm.is_a? VM
2123     raise unless @vmStats.is_a? StatsAccumulator
2124     raise unless @suite.is_a? BenchmarkSuite
2125   end
2126   
2127   def to_s
2128     "#{@suite} on #{@vm}"
2129   end
2130   
2131   def suite
2132     @suite
2133   end
2134   
2135   def vm
2136     @vm
2137   end
2138
2139   def reportingName
2140     @vm.name
2141   end
2142   
2143   def vmStats
2144     raise unless @vmStats
2145     @vmStats
2146   end
2147 end
2148
2149 class SubSuiteOnVM < StatsAccumulator
2150   def initialize(vm, suite)
2151     super()
2152     @vm = vm
2153     @suite = suite
2154     raise unless @vm.is_a? VM
2155     raise unless @suite.is_a? BenchmarkSuite
2156   end
2157   
2158   def to_s
2159     "#{@suite} on #{@vm}"
2160   end
2161   
2162   def suite
2163     @suite
2164   end
2165   
2166   def vm
2167     @vm
2168   end
2169   
2170   def reportingName
2171     @vm.name
2172   end
2173 end
2174
2175 class BenchPlan
2176   def initialize(benchmarkOnVM, iteration)
2177     @benchmarkOnVM = benchmarkOnVM
2178     @iteration = iteration
2179   end
2180   
2181   def to_s
2182     "#{@benchmarkOnVM} \##{@iteration+1}"
2183   end
2184   
2185   def benchmarkOnVM
2186     @benchmarkOnVM
2187   end
2188   
2189   def benchmark
2190     @benchmarkOnVM.benchmark
2191   end
2192   
2193   def suite
2194     @benchmarkOnVM.suite
2195   end
2196   
2197   def vm
2198     @benchmarkOnVM.vm
2199   end
2200   
2201   def iteration
2202     @iteration
2203   end
2204   
2205   def parseResult(result)
2206     raise unless result.plan == self
2207     @benchmarkOnVM.parseResult(result)
2208     benchmark.weight.times {
2209       @benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time)
2210       @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time)
2211       @benchmarkOnVM.subSuitesOnVM.each {
2212         | subSuite |
2213         subSuite.statsForIteration(@iteration, result.innerIndex).add(result.time)
2214       }
2215     }
2216   end
2217 end
2218
2219 def lpad(str,chars)
2220   if str.length>chars
2221     str
2222   else
2223     "%#{chars}s"%(str)
2224   end
2225 end
2226
2227 def rpad(str,chars)
2228   while str.length < chars
2229     str+=" "
2230   end
2231   str
2232 end
2233
2234 def center(str,chars)
2235   while str.length<chars
2236     str+=" "
2237     if str.length<chars
2238       str=" "+str
2239     end
2240   end
2241   str
2242 end
2243
2244 def statsToStr(stats, decimalShift)
2245   if stats.error?
2246     lpad(center("ERROR", 10+10+2), 12+10+2)
2247   elsif $inner*$outer == 1
2248     string = numToStr(stats.mean, decimalShift)
2249     raise unless string =~ /\./
2250     left = $~.pre_match
2251     right = $~.post_match
2252     lpad(left, 13 - decimalShift) + "." + rpad(right, 10 + decimalShift)
2253   else
2254     lpad(numToStr(stats.mean, decimalShift), 12) + "+-" + rpad(numToStr(stats.confInt, decimalShift), 10)
2255   end
2256 end
2257
2258 def plural(num)
2259   if num == 1
2260     ""
2261   else
2262     "s"
2263   end
2264 end
2265
2266 def wrap(str, columns)
2267   array = str.split
2268   result = ""
2269   curLine = array.shift
2270   array.each {
2271     | curStr |
2272     if (curLine + " " + curStr).size > columns
2273       result += curLine + "\n"
2274       curLine = curStr
2275     else
2276       curLine += " " + curStr
2277     end
2278   }
2279   result + curLine + "\n"
2280 end
2281   
2282 def runAndGetResults
2283   results = nil
2284   Dir.chdir(BENCH_DATA_PATH) {
2285     $stderr.puts ">> sh ./runscript" if $verbosity >= 2
2286     raise "Script did not complete correctly: #{$?}" unless system("sh ./runscript > runlog")
2287     results = IO::read("runlog")
2288   }
2289   raise unless results
2290   results
2291 end
2292
2293 def parseAndDisplayResults(results)
2294   vmStatses = []
2295   $vms.each {
2296     | vm |
2297     vmStatses << NamedStatsAccumulator.new(vm.name)
2298   }
2299   
2300   suitesOnVMs = []
2301   suitesOnVMsForSuite = {}
2302   subSuitesOnVMsForSubSuite = {}
2303   $suites.each {
2304     | suite |
2305     suitesOnVMsForSuite[suite] = []
2306     suite.subSuites.each {
2307       | subSuite |
2308       subSuitesOnVMsForSubSuite[subSuite] = []
2309     }
2310   }
2311   suitesOnVMsForVM = {}
2312   $vms.each {
2313     | vm |
2314     suitesOnVMsForVM[vm] = []
2315   }
2316   
2317   benchmarksOnVMs = []
2318   benchmarksOnVMsForBenchmark = {}
2319   $benchmarks.each {
2320     | benchmark |
2321     benchmarksOnVMsForBenchmark[benchmark] = []
2322   }
2323   
2324   $vms.each_with_index {
2325     | vm, vmIndex |
2326     vmStats = vmStatses[vmIndex]
2327     $suites.each {
2328       | suite |
2329       suiteOnVM = SuiteOnVM.new(vm, vmStats, suite)
2330       subSuitesOnVM = suite.subSuites.map {
2331         | subSuite |
2332         result = SubSuiteOnVM.new(vm, subSuite)
2333         subSuitesOnVMsForSubSuite[subSuite] << result
2334         result
2335       }
2336       suitesOnVMs << suiteOnVM
2337       suitesOnVMsForSuite[suite] << suiteOnVM
2338       suitesOnVMsForVM[vm] << suiteOnVM
2339       suite.benchmarks.each {
2340         | benchmark |
2341         subSuitesOnVMForThisBenchmark = []
2342         subSuitesOnVM.each {
2343           | subSuiteOnVM |
2344           if subSuiteOnVM.suite.hasBenchmark(benchmark)
2345             subSuitesOnVMForThisBenchmark << subSuiteOnVM
2346           end
2347         }
2348         benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM, subSuitesOnVMForThisBenchmark)
2349         benchmarksOnVMs << benchmarkOnVM
2350         benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
2351       }
2352     }
2353   }
2354   
2355   plans = []
2356   benchmarksOnVMs.each {
2357     | benchmarkOnVM |
2358     $outer.times {
2359       | iteration |
2360       plans << BenchPlan.new(benchmarkOnVM, iteration)
2361     }
2362   }
2363
2364   hostname = nil
2365   hwmodel = nil
2366   results.each_line {
2367     | line |
2368     line.chomp!
2369     if line =~ /HOSTNAME:([^.]+)/
2370       hostname = $1
2371     elsif line =~ /HARDWARE:hw\.model: /
2372       hwmodel = $~.post_match.chomp
2373     else
2374       result = ParsedResult.parse(plans, line)
2375       if result
2376         result.plan.parseResult(result)
2377       end
2378     end
2379   }
2380   
2381   # Compute the geomean of the preferred means of results on a SuiteOnVM
2382   overallResults = []
2383   $vms.each {
2384     | vm |
2385     result = Stats.new
2386     $outer.times {
2387       | outerIndex |
2388       $inner.times {
2389         | innerIndex |
2390         curResult = Stats.new
2391         suitesOnVMsForVM[vm].each {
2392           | suiteOnVM |
2393           # For a given iteration, suite, and VM, compute the suite's preferred mean
2394           # over the data collected for all benchmarks in that suite. We'll have one
2395           # sample per benchmark. For example on V8 this will be the geomean of 1
2396           # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
2397           # splay.
2398           curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
2399         }
2400         
2401         # curResult now holds 1 sample for each of the means computed in the above
2402         # loop. Compute the geomean over this, and store it.
2403         if curResult.ok?
2404           result.add(curResult.geometricMean)
2405         else
2406           result.add(nil)
2407         end
2408       }
2409     }
2410
2411     # $overallResults will have a Stats for each VM. That Stats object will hold
2412     # $inner*$outer geomeans, allowing us to compute the arithmetic mean and
2413     # confidence interval of the geomeans of preferred means. Convoluted, but
2414     # useful and probably sound.
2415     overallResults << result
2416   }
2417   
2418   if $verbosity >= 2
2419     benchmarksOnVMs.each {
2420       | benchmarkOnVM |
2421       $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
2422     }
2423     
2424     $vms.each_with_index {
2425       | vm, vmIndex |
2426       vmStats = vmStatses[vmIndex]
2427       $stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}"
2428       $stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}"
2429     }
2430   end
2431
2432   if $outputName
2433     reportName = $outputName
2434   else
2435     reportName =
2436       (if ($vms.collect {
2437              | vm |
2438              vm.nameKind
2439            }.index :auto)
2440          ""
2441        else
2442          text = $vms.collect {
2443            | vm |
2444            vm.to_s
2445          }.join("_") + "_"
2446          if text.size >= 40
2447            ""
2448          else
2449            text
2450          end
2451        end) +
2452       ($suites.collect {
2453          | suite |
2454          suite.to_s
2455        }.join("")) + "_" +
2456       (if hostname
2457          hostname + "_"
2458        else
2459          ""
2460        end)+
2461       (begin
2462          time = Time.now
2463          "%04d%02d%02d_%02d%02d" %
2464            [ time.year, time.month, time.day,
2465              time.hour, time.min ]
2466        end)
2467   end
2468
2469   unless $brief
2470     puts "Generating benchmark report at #{Dir.pwd}/#{reportName}_report.txt"
2471     puts "And raw data at #{Dir.pwd}/#{reportName}.json"
2472   end
2473   
2474   outp = $stdout
2475   json = {}
2476   begin
2477     outp = File.open(reportName + "_report.txt","w")
2478   rescue => e
2479     $stderr.puts "Error: could not save report to #{reportName}_report.txt: #{e}"
2480     $stderr.puts
2481   end
2482   
2483   def createVMsString
2484     result = ""
2485     result += "   " if $allSuites.size > 1
2486     result += rpad("", $benchpad + $weightpad)
2487     result += " "
2488     $vms.size.times {
2489       | index |
2490       if index != 0
2491         result += " "+NoChange.new(0).shortForm
2492       end
2493       result += lpad(center($vms[index].name, 10+10+2), 12+10+2)
2494     }
2495     result += "    "
2496     if $vms.size >= 3
2497       result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
2498     elsif $vms.size >= 2
2499       result += " "*26
2500     end
2501     result
2502   end
2503   
2504   def andJoin(list)
2505     if list.size == 1
2506       list[0].to_s
2507     elsif list.size == 2
2508       "#{list[0]} and #{list[1]}"
2509     else
2510       "#{list[0..-2].join(', ')}, and #{list[-1]}"
2511     end
2512   end
2513   
2514   json["vms"] = $vms.collect{|v| v.name}
2515   json["suites"] = {}
2516   json["runlog"] = results
2517   
2518   columns = [createVMsString.size, 78].max
2519   
2520   outp.print "Benchmark report for "
2521   outp.print andJoin($suites)
2522   if hostname
2523     outp.print " on #{hostname}"
2524   end
2525   if hwmodel
2526     outp.print " (#{hwmodel})"
2527   end
2528   outp.puts "."
2529   outp.puts
2530   
2531   outp.puts "VMs tested:"
2532   $vms.each {
2533     | vm |
2534     outp.print "\"#{vm.name}\" at #{vm.origPath}"
2535     if vm.svnRevision
2536       outp.print " (r#{vm.svnRevision})"
2537     end
2538     outp.puts
2539     vm.extraEnv.each_pair {
2540       | key, val |
2541       outp.puts "    export #{key}=#{val}"
2542     }
2543   }
2544   
2545   outp.puts
2546   
2547   outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
2548                  "with #{$outer} VM invocation#{plural($outer)} per benchmark."+
2549                  (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
2550                                       "total time of those iterations, for each sample.")
2551                   else "" end)+
2552                  (if $measureGC == true then (" No manual garbage collection invocations were "+
2553                                               "emitted.")
2554                   elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
2555                                          "all VMs except #{$measureGC}.")
2556                   else (" Emitted a call to gc() between sample measurements.") end)+
2557                  (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
2558                                         "began with the very first iteration.")
2559                   else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
2560                         "invocation for warm-up.") end)+
2561                  (case $timeMode
2562                   when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
2563                                           "microsecond-level timing.")
2564                   when :date then (" Used the portable Date.now() method to get millisecond-"+
2565                                    "level timing.")
2566                   else raise end)+
2567                  " Reporting benchmark execution times with 95% confidence "+
2568                  "intervals in milliseconds.",
2569                  columns)
2570   
2571   outp.puts
2572   
2573   def printVMs(outp)
2574     outp.puts createVMsString
2575   end
2576   
2577   def summaryStats(outp, json, accumulators, name, decimalShift, &proc)
2578     resultingJson = {}
2579     outp.print "   " if $allSuites.size > 1
2580     outp.print rpad(name, $benchpad + $weightpad)
2581     outp.print " "
2582     accumulators.size.times {
2583       | index |
2584       if index != 0
2585         outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
2586       end
2587       outp.print statsToStr(accumulators[index].stats(&proc), decimalShift)
2588       resultingJson[accumulators[index].reportingName] = accumulators[index].stats(&proc).jsonMap
2589     }
2590     if accumulators.size>=2
2591       outp.print("    "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm)
2592     end
2593     outp.puts
2594     json[name] = resultingJson
2595   end
2596
2597   def allSummaryStats(outp, json, accumulators, preferredMean, decimalShift)
2598     meanLabel = '<' + preferredMean.to_s.sub(/Mean$/, '') + '>'
2599     summaryStats(outp, json, accumulators, meanLabel, decimalShift) {
2600       | stat |
2601       stat.send(preferredMean)
2602     }
2603   end
2604   
2605   $suites.each {
2606     | suite |
2607     suiteJson = {}
2608     subSuiteJsons = {}
2609     suite.subSuites.each {
2610       | subSuite |
2611       subSuiteJsons[subSuite] = {}
2612     }
2613     
2614     printVMs(outp)
2615     if $allSuites.size > 1
2616       outp.puts(andJoin(suite.suites.map{|v| v.name}) + ":")
2617     else
2618       outp.puts
2619     end
2620     suite.benchmarks.each {
2621       | benchmark |
2622       benchmarkJson = {}
2623       outp.print "   " if $allSuites.size > 1
2624       outp.print rpad(benchmark.name, $benchpad) + rpad(benchmark.weightString, $weightpad)
2625       if benchmark.name.size > $benchNameClip
2626         outp.puts
2627         outp.print "   " if $allSuites.size > 1
2628         outp.print((" " * $benchpad) + (" " * $weightpad))
2629       end
2630       outp.print " "
2631       myConfigs = benchmarksOnVMsForBenchmark[benchmark]
2632       myConfigs.size.times {
2633         | index |
2634         if index != 0
2635           outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
2636         end
2637         outp.print statsToStr(myConfigs[index].stats, suite.decimalShift)
2638         benchmarkJson[myConfigs[index].vm.name] = myConfigs[index].stats.jsonMap
2639       }
2640       if $vms.size>=2
2641         outp.print("    "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
2642       end
2643       outp.puts
2644       suiteJson[benchmark.name] = benchmarkJson
2645       suite.subSuites.each {
2646         | subSuite |
2647         if subSuite.hasBenchmark(benchmark)
2648           subSuiteJsons[subSuite][benchmark.name] = benchmarkJson
2649         end
2650       }
2651     }
2652     outp.puts
2653     unless suite.subSuites.empty?
2654       suite.subSuites.each {
2655         | subSuite |
2656         outp.puts "#{subSuite.name}:"
2657         allSummaryStats(outp, subSuiteJsons[subSuite], subSuitesOnVMsForSubSuite[subSuite], subSuite.preferredMean, subSuite.decimalShift)
2658         outp.puts
2659       }
2660       outp.puts "#{suite.name} including #{andJoin(suite.subSuites.map{|v| v.name})}:"
2661     end
2662     allSummaryStats(outp, suiteJson, suitesOnVMsForSuite[suite], suite.preferredMean, suite.decimalShift)
2663     outp.puts if $allSuites.size > 1
2664     
2665     json["suites"][suite.name] = suiteJson
2666     suite.subSuites.each {
2667       | subSuite |
2668       json["suites"][subSuite.name] = subSuiteJsons[subSuite]
2669     }
2670   }
2671   
2672   if $suites.size > 1
2673     scaledResultJson = {}
2674     printVMs(outp)
2675     outp.puts "Geomean of preferred means:"
2676     outp.print "   "
2677     outp.print rpad("<scaled-result>", $benchpad + $weightpad)
2678     outp.print " "
2679     $vms.size.times {
2680       | index |
2681       if index != 0
2682         outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm
2683       end
2684       outp.print statsToStr(overallResults[index], 0)
2685       scaledResultJson[$vms[index].name] = overallResults[index].jsonMap
2686     }
2687     if overallResults.size>=2
2688       outp.print("    "+overallResults[-1].compareTo(overallResults[0]).longForm)
2689     end
2690     outp.puts
2691     
2692     json["<scaled-result>"] = scaledResultJson
2693   end
2694   outp.puts
2695   
2696   if outp != $stdout
2697     outp.close
2698   end
2699   
2700   if outp != $stdout and not $brief
2701     puts
2702     File.open(reportName + "_report.txt") {
2703       | inp |
2704       puts inp.read
2705     }
2706   end
2707   
2708   if $brief
2709     puts(overallResults.collect{|stats| stats.mean}.join("\t"))
2710     puts(overallResults.collect{|stats| stats.confInt}.join("\t"))
2711   end
2712   
2713   File.open(reportName + ".json", "w") {
2714     | outp |
2715     outp.puts json.to_json
2716   }
2717 end
2718
2719 begin
2720   $sawBenchOptions = false
2721   
2722   def resetBenchOptionsIfNecessary
2723     unless $sawBenchOptions
2724       $includeSunSpider = false
2725       $includeLongSpider = false
2726       $includeV8 = false
2727       $includeKraken = false
2728       $includeJSBench = false
2729       $includeMicrobenchmarks = false
2730       $includeAsmBench = false
2731       $includeDSPJS = false
2732       $includeBrowsermarkJS = false
2733       $includeBrowsermarkDOM = false
2734       $includeOctane = false
2735       $includeCompressionBench = false
2736       $includeSixSpeed = false
2737       $includeTailBench = false;
2738       $sawBenchOptions = true
2739     end
2740   end
2741   
2742   GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
2743                  ['--inner', GetoptLong::REQUIRED_ARGUMENT],
2744                  ['--outer', GetoptLong::REQUIRED_ARGUMENT],
2745                  ['--warmup', GetoptLong::REQUIRED_ARGUMENT],
2746                  ['--no-ss-warmup', GetoptLong::NO_ARGUMENT],
2747                  ['--quantum', GetoptLong::REQUIRED_ARGUMENT],
2748                  ['--minimum', GetoptLong::REQUIRED_ARGUMENT],
2749                  ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
2750                  ['--sunspider', GetoptLong::NO_ARGUMENT],
2751                  ['--longspider', GetoptLong::NO_ARGUMENT],
2752                  ['--v8-spider', GetoptLong::NO_ARGUMENT],
2753                  ['--kraken', GetoptLong::NO_ARGUMENT],
2754                  ['--js-bench', GetoptLong::NO_ARGUMENT],
2755                  ['--microbenchmarks', GetoptLong::NO_ARGUMENT],
2756                  ['--asm-bench', GetoptLong::NO_ARGUMENT],
2757                  ['--dsp', GetoptLong::NO_ARGUMENT],
2758                  ['--browsermark-js', GetoptLong::NO_ARGUMENT],
2759                  ['--browsermark-dom', GetoptLong::NO_ARGUMENT],
2760                  ['--octane', GetoptLong::NO_ARGUMENT],
2761                  ['--compression-bench', GetoptLong::NO_ARGUMENT],
2762                  ['--six-speed', GetoptLong::NO_ARGUMENT],
2763                  ['--tail-bench', GetoptLong::NO_ARGUMENT],
2764                  ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
2765                  ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
2766                  ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
2767                  ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
2768                  ['--dont-copy-vms', GetoptLong::NO_ARGUMENT],
2769                  ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
2770                  ['--brief', GetoptLong::NO_ARGUMENT],
2771                  ['--silent', GetoptLong::NO_ARGUMENT],
2772                  ['--remote', GetoptLong::REQUIRED_ARGUMENT],
2773                  ['--local', GetoptLong::NO_ARGUMENT],
2774                  ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
2775                  ['--slave', GetoptLong::NO_ARGUMENT],
2776                  ['--prepare-only', GetoptLong::NO_ARGUMENT],
2777                  ['--analyze', GetoptLong::REQUIRED_ARGUMENT],
2778                  ['--vms', GetoptLong::REQUIRED_ARGUMENT],
2779                  ['--output-name', GetoptLong::REQUIRED_ARGUMENT],
2780                  ['--environment', GetoptLong::REQUIRED_ARGUMENT],
2781                  ['--dependencies', GetoptLong::REQUIRED_ARGUMENT],
2782                  ['--config', GetoptLong::REQUIRED_ARGUMENT],
2783                  ['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
2784     | opt, arg |
2785     case opt
2786     when '--rerun'
2787       $rerun = intArg(opt,arg,1,nil)
2788     when '--inner'
2789       $inner = intArg(opt,arg,1,nil)
2790     when '--outer'
2791       $outer = intArg(opt,arg,1,nil)
2792     when '--warmup'
2793       $warmup = intArg(opt,arg,0,nil)
2794     when '--no-ss-warmup'
2795       $sunSpiderWarmup = false
2796     when '--quantum'
2797       $quantum = intArg(opt,arg,1,nil)
2798     when '--minimum'
2799       $minimum = intArg(opt,arg,1,nil)
2800     when '--timing-mode'
2801       if arg.upcase == "PRECISETIME"
2802         $timeMode = :preciseTime
2803       elsif arg.upcase == "DATE"
2804         $timeMode = :date
2805       elsif arg.upcase == "AUTO"
2806         $timeMode = :auto
2807       else
2808         quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
2809                   "Invalid argument for command-line option")
2810       end
2811     when '--force-vm-kind'
2812       if arg.upcase == "JSC"
2813         $forceVMKind = :jsc
2814       elsif arg.upcase == "DUMPRENDERTREE"
2815         $forceVMKind = :dumpRenderTree
2816       elsif arg.upcase == "WEBKITTESTRUNNER"
2817         $forceVMKind = :webkitTestRunner
2818       elsif arg.upcase == "AUTO"
2819         $forceVMKind = nil
2820       else
2821         quickFail("Expected 'jsc', 'DumpRenderTree', or 'WebKitTestRunner' for --force-vm-kind, but got '#{arg}'.",
2822                   "Invalid argument for command-line option")
2823       end
2824     when '--force-vm-copy'
2825       $needToCopyVMs = true
2826     when '--dont-copy-vms'
2827       $dontCopyVMs = true
2828     when '--sunspider'
2829       resetBenchOptionsIfNecessary
2830       $includeSunSpider = true
2831     when '--longspider'
2832       resetBenchOptionsIfNecessary
2833       $includeLongSpider = true
2834     when '--v8-spider'
2835       resetBenchOptionsIfNecessary
2836       $includeV8 = true
2837     when '--kraken'
2838       resetBenchOptionsIfNecessary
2839       $includeKraken = true
2840     when '--js-bench'
2841       resetBenchOptionsIfNecessary
2842       $includeJSBench = true
2843     when '--tail-bench'
2844       resetBenchOptionsIfNecessary
2845       $includeTailBench = true
2846     when '--microbenchmarks'
2847       resetBenchOptionsIfNecessary
2848       $includeMicrobenchmarks = true
2849     when '--asm-bench'
2850       resetBenchOptionsIfNecessary
2851       $includeAsmBench = true
2852     when '--dsp'
2853       resetBenchOptionsIfNecessary
2854       $includeDSPJS = true
2855     when '--browsermark-js'
2856       resetBenchOptionsIfNecessary
2857       $includeBrowsermarkJS = true
2858     when '--browsermark-dom'
2859       resetBenchOptionsIfNecessary
2860       $includeBrowsermarkDOM = true
2861     when '--octane'
2862       resetBenchOptionsIfNecessary
2863       $includeOctane = true
2864     when '--compression-bench'
2865       resetBenchOptionsIfNecessary
2866       $includeCompressionBench = true
2867     when '--six-speed'
2868       resetBenchOptionsIfNecessary
2869       $includeSixSpeed = true
2870     when '--benchmarks'
2871       $benchmarkPattern = Regexp.new(arg)
2872     when '--measure-gc'
2873       if arg == ''
2874         $measureGC = true
2875       else
2876         $measureGC = arg
2877       end
2878     when '--verbose'
2879       $verbosity += 1
2880     when '--brief'
2881       $brief = true
2882     when '--silent'
2883       $silent = true
2884     when '--remote'
2885       $remoteHosts += arg.split(',')
2886       $needToCopyVMs = true
2887     when '--ssh-options'
2888       $sshOptions << arg
2889     when '--local'
2890       $alsoLocal = true
2891     when '--prepare-only'
2892       $run = false
2893     when '--analyze'
2894       $prepare = false
2895       $run = false
2896       $analyze << arg
2897     when '--output-name'
2898       $outputName = arg
2899     when '--vms'
2900       JSON::parse(IO::read(arg)).each {
2901         | vmDescription |
2902         path = Pathname.new(vmDescription["path"]).realpath
2903         if vmDescription["name"]
2904           name = vmDescription["name"]
2905           nameKind = :given
2906         else
2907           name = "Conf\##{$vms.length+1}"
2908           nameKind = :auto
2909         end
2910         vm = VM.new(path, name, nameKind, nil)
2911         if vmDescription["env"]
2912           vmDescription["env"].each_pair {
2913             | key, val |
2914             vm.addExtraEnv(key, val)
2915           }
2916         end
2917         $vms << vm
2918       }
2919     when '--environment'
2920       $environment = JSON::parse(IO::read(arg))
2921     when '--dependencies'
2922       $dependencies.push(Pathname.new(arg).realpath)
2923     when '--config'
2924       $configPath = Pathname.new(arg)
2925     when '--help'
2926       usage
2927     else
2928       raise "bad option: #{opt}"
2929     end
2930   }
2931   
2932   # Figure out the configuration
2933   if $configPath.file?
2934     config = JSON::parse(IO::read($configPath.to_s))
2935   else
2936     config = {}
2937   end
2938
2939   def pathname_if_exist config
2940     if config
2941       Pathname.new config
2942     else
2943       config
2944     end
2945   end
2946
2947   OCTANE_PATH = pathname_if_exist(config["OctanePath"])
2948   BROWSERMARK_PATH = pathname_if_exist(config["BrowserMarkPath"])
2949   BROWSERMARK_JS_PATH = pathname_if_exist(config["BrowserMarkJSPath"])
2950   BROWSERMARK_DOM_PATH = pathname_if_exist(config["BrowserMarkDOMPath"])
2951   ASMBENCH_PATH = pathname_if_exist(config["AsmBenchPath"])
2952   COMPRESSIONBENCH_PATH = pathname_if_exist(config["CompressionBenchPath"])
2953   DSPJS_FILTRR_PATH = pathname_if_exist(config["DSPJSFiltrrPath"])
2954   DSPJS_ROUTE9_PATH = pathname_if_exist(config["DSPJSRoute9Path"])
2955   DSPJS_STARFIELD_PATH = pathname_if_exist(config["DSPJSStarfieldPath"])
2956   DSPJS_QUAKE3_PATH = pathname_if_exist(config["DSPJSQuake3Path"])
2957   DSPJS_MANDELBROT_PATH = pathname_if_exist(config["DSPJSMandelbrotPath"])
2958   DSPJS_JSLINUX_PATH = pathname_if_exist(config["DSPJSLinuxPath"])
2959   DSPJS_AMMOJS_ASMJS_PATH = pathname_if_exist(config["DSPJSAmmoJSAsmJSPath"])
2960   DSPJS_AMMOJS_REGULAR_PATH = pathname_if_exist(config["DSPJSAmmoJSRegularPath"])
2961   KRAKEN_PATH = pathname_if_exist(config["KrakenPath"])
2962   
2963   # If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option.
2964   if $dontCopyVMs
2965     $needToCopyVMs = false
2966   end
2967   
2968   ARGV.each {
2969     | vm |
2970     if vm =~ /([a-zA-Z0-9_ .]+):/
2971       name = $1
2972       nameKind = :given
2973       vm = $~.post_match
2974     else
2975       name = "Conf\##{$vms.length+1}"
2976       nameKind = :auto
2977     end
2978     envs = []
2979     while vm =~ /([a-zA-Z0-9_]+)=([a-zA-Z0-9_:.\/-]+):/
2980       envs << [$1, $2]
2981       vm = $~.post_match
2982     end
2983     $stderr.puts "#{name}: #{vm}" if $verbosity >= 1
2984     vm = VM.new(Pathname.new(vm).realpath, name, nameKind, nil)
2985     envs.each {
2986       | pair |
2987       vm.addExtraEnv(pair[0], pair[1])
2988     }
2989     $vms << vm
2990   }
2991   
2992   if $vms.empty?
2993     quickFail("Please specify at least on configuraiton on the command line.",
2994               "Insufficient arguments")
2995   end
2996   
2997   $vms.each {
2998     | vm |
2999     if vm.vmType == :jsc
3000       $allDRT = false
3001     end
3002   }
3003   
3004   SUNSPIDER = BenchmarkSuite.new("SunSpider", :arithmeticMean, 0)
3005   WARMUP = BenchmarkSuite.new("WARMUP", :arithmeticMean, 0)
3006   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
3007    "access-fannkuch", "access-nbody", "access-nsieve",
3008    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
3009    "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
3010    "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
3011    "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
3012    "string-base64", "string-fasta", "string-tagcloud",
3013    "string-unpack-code", "string-validate-input"].each {
3014     | name |
3015     SUNSPIDER.add SunSpiderBenchmark.new(name)
3016     WARMUP.addIgnoringPattern SunSpiderBenchmark.new(name)
3017   }
3018
3019   LONGSPIDER = BenchmarkSuite.new("LongSpider", :geometricMean, 0)
3020   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
3021    "access-fannkuch", "access-nbody", "access-nsieve",
3022    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-nsieve-bits",
3023    "controlflow-recursive", "crypto-aes", "crypto-md5", "crypto-sha1",
3024    "date-format-tofte", "date-format-xparb", "hash-map", "math-cordic",
3025    "math-partial-sums", "math-spectral-norm", "string-base64",
3026    "string-fasta", "string-tagcloud"].each {
3027     | name |
3028     LONGSPIDER.add LongSpiderBenchmark.new(name)
3029   }
3030
3031   V8 = BenchmarkSuite.new("V8Spider", :geometricMean, 0)
3032   ["crypto", "deltablue", "earley-boyer", "raytrace",
3033    "regexp", "richards", "splay"].each {
3034     | name |
3035     V8.add V8Benchmark.new(name)
3036   }
3037
3038   OCTANE = BenchmarkSuite.new("Octane", :geometricMean, 1)
3039   [[["crypto"], "encrypt", 1, true, false, 32],
3040    [["crypto"], "decrypt", 1, true, false, 32],
3041    [["deltablue"], "deltablue", 2, true, false, 32],
3042    [["earley-boyer"], "earley", 1, true, false, 32],
3043    [["earley-boyer"], "boyer", 1, true, false, 32],
3044    [["navier-stokes"], "navier-stokes", 2, true, false, 16],
3045    [["raytrace"], "raytrace", 2, true, false, 32],
3046    [["richards"], "richards", 2, true, false, 32],
3047    [["splay"], "splay", 2, true, false, 32],
3048    [["regexp"], "regexp", 2, true, false, 16],
3049    [["pdfjs"], "pdfjs", 2, false, false, 4],
3050    [["mandreel"], "mandreel", 2, false, false, 4],
3051    [["gbemu-part1", "gbemu-part2"], "gbemu", 2, false, false, 4],
3052    [["code-load"], "closure", 1, false, false, 16],
3053    [["code-load"], "jquery", 1, false, false, 16],
3054    [["box2d"], "box2d", 2, false, false, 8],
3055    [["zlib", "zlib-data"], "zlib", 2, false, true, 3],
3056    [["typescript", "typescript-input", "typescript-compiler"], "typescript", 2, false, true, 1]].each {
3057     | args |
3058     OCTANE.add OctaneBenchmark.new(*args)
3059   }
3060   OCTANE.add CustomTimedBenchmark.new("splay-latency", (OPENSOURCE_OCTANE_PATH + "splay.js").to_s)
3061
3062   KRAKEN = BenchmarkSuite.new("Kraken", :arithmeticMean, -1)
3063   ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
3064    "audio-oscillator", "imaging-darkroom", "imaging-desaturate",
3065    "imaging-gaussian-blur", "json-parse-financial",
3066    "json-stringify-tinderbox", "stanford-crypto-aes",
3067    "stanford-crypto-ccm", "stanford-crypto-pbkdf2",
3068    "stanford-crypto-sha256-iterative"].each {
3069     | name |
3070     KRAKEN.add KrakenBenchmark.new(name)
3071   }
3072   
3073   JSBENCH = BenchmarkSuite.new("JSBench", :geometricMean, 0)
3074   [["amazon-chrome", "urem"], ["amazon-chrome-win", "urem"], ["amazon-firefox", "urm"],
3075    ["amazon-firefox-win", "urm"], ["amazon-safari", "urem"], ["facebook-chrome", "urem"],
3076    ["facebook-chrome-win", "urem"], ["facebook-firefox", "urem"],
3077    ["facebook-firefox-win", "urem"], ["facebook-safari", "urem"],
3078    ["google-chrome", "urem"], ["google-chrome-win", "urem"], ["google-firefox", "uem"],
3079    ["google-firefox-win", "urem"], ["google-safari", "urem"], ["twitter-chrome", "urem"],
3080    ["twitter-chrome-win", "rem"], ["twitter-firefox", "urem"],
3081    ["twitter-firefox-win", "urem"], ["twitter-safari", "urem"], ["yahoo-chrome", "urem"],
3082    ["yahoo-chrome-win", "urem"], ["yahoo-firefox", "urem"], ["yahoo-firefox-win", "urem"],
3083    ["yahoo-safari", "urem"]].each {
3084     | nameAndMode |
3085     JSBENCH.add JSBenchBenchmark.new(*nameAndMode)
3086   }
3087
3088   TAILBENCH = BenchmarkSuite.new("TailBench", :geometricMean, 0)
3089   ["n-body", "merge-sort", "merge-sort-cps", "bf-interpreter"].each {
3090     | name |
3091     TAILBENCH.add TailBenchBenchmark.new(name);
3092   }
3093
3094   MICROBENCHMARKS = BenchmarkSuite.new("Microbenchmarks", :geometricMean, 0)
3095   Dir.foreach(MICROBENCHMARKS_PATH) {
3096     | filename |
3097     if filename =~ /\.js$/
3098       name = $~.pre_match
3099       MICROBENCHMARKS.add MicrobenchmarksBenchmark.new(name)
3100     end
3101   }
3102   
3103   ASMBENCH = BenchmarkSuite.new("AsmBench", :geometricMean, 0)
3104   if ASMBENCH_PATH
3105     Dir.foreach(ASMBENCH_PATH) {
3106       | filename |
3107       if filename =~ /\.js$/
3108         name = $~.pre_match
3109         ASMBENCH.add AsmBenchBenchmark.new(name)
3110       end
3111     }
3112   end
3113
3114   COMPRESSIONBENCH = BenchmarkSuite.new("CompressionBench", :geometricMean, 0)
3115   [[["huffman", "compression-data"], "huffman", ""],
3116    [["arithmetic", "compression-data"], "arithmetic", "Simple"],
3117    [["arithmetic", "compression-data"], "arithmetic", "Precise"],
3118    [["arithmetic", "compression-data"], "arithmetic", "Complex Precise"],
3119    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 0"],
3120    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 1"],
3121    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 2"],
3122    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 1"],
3123    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 2"],
3124    [["lz-string", "compression-data"], "lz-string", ""]
3125   ].each {
3126     | args |
3127     COMPRESSIONBENCH.add CompressionBenchBenchmark.new(*args)
3128   }
3129
3130   SIXSPEED = BenchmarkSuite.new("SixSpeed", :geometricMean, 0)
3131   [[ "template_string", 200000000 ],
3132    [ "defaults", 100000000 ],
3133    [ "map-set-lookup", 200000 ],
3134    [ "spread", 1000000 ],
3135    [ "object-assign", 600000 ],
3136    [ "spread-literal", 1000000 ],
3137    [ "map-set", 10000 ],
3138    [ "destructuring-simple", 20000000 ],
3139    [ "super", 3000000 ],
3140    [ "for-of-object", 1000000 ],
3141    [ "rest", 500000 ],
3142    [ "regex-u", 1000000 ],
3143    [ "arrow", 20000000 ],
3144    [ "bindings-compound", 20000000 ],
3145    [ "classes", 10000000 ],
3146    [ "template_string_tag", 2000000 ],
3147    [ "map-string", 30000000 ],
3148    [ "arrow-declare", 30000000 ],
3149    [ "spread-generator", 1000000 ],
3150    [ "object-literal-ext", 1000000 ],
3151    [ "generator", 3000000 ],
3152    [ "arrow-args", 20000000 ],
3153    [ "for-of-array", 5000000 ],
3154    [ "bindings", 20000000 ],
3155    [ "destructuring", 20000000 ],
3156    [ "map-set-object", 5000 ],
3157   ].each {
3158     | name, iterations |
3159     SIXSPEED.add SixSpeedBenchmark.new("#{name}.es5", "#{name}/#{name}.es5", iterations)
3160     SIXSPEED.add SixSpeedBenchmark.new("#{name}.es6", "#{name}/#{name}.es6", iterations)
3161   }
3162
3163   DSPJS = BenchmarkSuite.new("DSP", :geometricMean, 0)
3164   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-posterize-tint", "e2")
3165   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-contrast-sat-bright", "e5")
3166   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-sat-adj-contr-mult", "e7")
3167   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-blur-overlay-sat-contr", "e8")
3168   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sat-blur-mult-sharpen-contr", "e9")
3169   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sepia-bias", "e10")
3170   DSPJS.add DSPJSVP8Benchmark.new
3171   DSPJS.add DSPStarfieldBenchmark.new
3172   DSPJS.add DSPJSJSLinuxBenchmark.new
3173   DSPJS.add DSPJSQuake3Benchmark.new
3174   DSPJS.add DSPJSMandelbrotBenchmark.new
3175   DSPJS.add DSPJSAmmoJSASMBenchmark.new
3176   DSPJS.add DSPJSAmmoJSRegularBenchmark.new
3177
3178   BROWSERMARK_JS = BenchmarkSuite.new("BrowsermarkJS", :geometricMean, 1)
3179   ["array_blur", "array_weighted", "string_chat", "string_filter", "string_weighted"].each {
3180     | name |
3181       BROWSERMARK_JS.add BrowsermarkJSBenchmark.new(name)
3182   }
3183   
3184   BROWSERMARK_DOM = BenchmarkSuite.new("BrowsermarkDOM", :geometricMean, 1)
3185   ["advanced_search", "create_source", "dynamic_create", "search"].each {
3186     | name |
3187       BROWSERMARK_DOM.add BrowsermarkDOMBenchmark.new(name)
3188   }
3189
3190   $suites = []
3191   
3192   if $includeSunSpider and not SUNSPIDER.empty?
3193     $suites << SUNSPIDER
3194   end
3195   
3196   if $includeLongSpider and not LONGSPIDER.empty?
3197     $suites << LONGSPIDER
3198   end
3199   
3200   if $includeV8 and not V8.empty?
3201     $suites << V8
3202   end
3203   
3204   if $includeOctane and not OCTANE.empty?
3205     if OCTANE_PATH
3206       $suites << OCTANE
3207     else
3208       $stderr.puts "Warning: refusing to run Octane because \"OctanePath\" isn't set in #{$configPath}."
3209     end
3210   end
3211   
3212   if $includeKraken and not KRAKEN.empty?
3213     if KRAKEN_PATH
3214       $suites << KRAKEN
3215     else
3216       $stderr.puts "Warning: refusing to run Kraken because \"KrakenPath\" isn't set in #{$configPath}."
3217     end
3218   end
3219   
3220   if $includeJSBench and not JSBENCH.empty?
3221     if $allDRT
3222       if JSBENCH_PATH
3223         $suites << JSBENCH
3224       else
3225         $stderr.puts "Warning: refusing to run JSBench because \"JSBenchPath\" isn't set in #{$configPath}"
3226       end
3227     else
3228       $stderr.puts "Warning: refusing to run JSBench because not all VMs are DumpRenderTree or WebKitTestRunner."
3229     end
3230   end
3231   
3232   if $includeTailBench and not TAILBENCH.empty?
3233     $suites << TAILBENCH
3234   end
3235
3236   if $includeMicrobenchmarks and not MICROBENCHMARKS.empty?
3237     $suites << MICROBENCHMARKS
3238   end
3239
3240   if $includeAsmBench and not ASMBENCH.empty?
3241     if ASMBENCH_PATH
3242       $suites << ASMBENCH
3243     else
3244       $stderr.puts "Warning: refusing to run AsmBench because \"AsmBenchPath\" isn't set in #{$configPath}."
3245     end
3246   end
3247   
3248   if $includeDSPJS and not DSPJS.empty?
3249     if $allDRT
3250       if DSPJS_FILTRR_PATH and DSPJS_ROUTE9_PATH and DSPJS_STARFIELD_PATH and DSPJS_QUAKE3_PATH and DSPJS_MANDELBROT_PATH and DSPJS_JSLINUX_PATH and DSPJS_AMMOJS_ASMJS_PATH and DSPJS_AMMOJS_REGULAR_PATH
3251         $suites << DSPJS
3252       else
3253         $stderr.puts "Warning: refusing to run DSPJS because one of the following isn't set in #{$configPath}: \"DSPJSFiltrrPath\", \"DSPJSRoute9Path\", \"DSPJSStarfieldPath\", \"DSPJSQuake3Path\", \"DSPJSMandelbrotPath\", \"DSPJSLinuxPath\", \"DSPJSAmmoJSAsmJSPath\", \"DSPJSAmmoJSRegularPath\"."
3254       end
3255     else
3256       $stderr.puts "Warning: refusing to run DSPJS because not all VMs are DumpRenderTree or WebKitTestRunner."
3257     end
3258   end
3259   
3260   if $includeBrowsermarkJS and not BROWSERMARK_JS.empty?
3261     if BROWSERMARK_PATH and BROWSERMARK_JS_PATH
3262       $suites << BROWSERMARK_JS
3263     else
3264       $stderr.puts "Warning: refusing to run Browsermark-JS because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\" or \"BrowserMarkJSPath\"."
3265     end
3266   end
3267
3268   if $includeBrowsermarkDOM and not BROWSERMARK_DOM.empty?
3269     if $allDRT
3270       if BROWSERMARK_PATH and BROWSERMARK_JS_PATH and BROWSERMARK_DOM_PATH
3271         $suites << BROWSERMARK_DOM
3272       else
3273         $stderr.puts "Warning: refusing to run Browsermark-DOM because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\", \"BrowserMarkJSPath\", or \"BrowserMarkDOMPath\"."
3274       end
3275     else
3276       $stderr.puts "Warning: refusing to run Browsermark-DOM because not all VMs are DumpRenderTree or WebKitTestRunner."
3277     end
3278   end
3279
3280   if $includeCompressionBench and not COMPRESSIONBENCH.empty?
3281     if COMPRESSIONBENCH_PATH
3282       $suites << COMPRESSIONBENCH
3283     else
3284       $stderr.puts "Warning: refusing to run CompressionBench because \"CompressionBenchPath\" isn't set in #{$configPath}"
3285     end
3286   end
3287
3288   if $includeSixSpeed and not SIXSPEED.empty?
3289     if SIXSPEED_PATH
3290       $suites << SIXSPEED
3291     else
3292       $stderr.puts "Warning: refusing to run SixSpeed because \"SixSpeedPath\" isn't set in #{$configPath}."
3293     end
3294   end
3295
3296
3297   $allSuites = $suites.map{|v| v.suites}.flatten(1)
3298   
3299   $benchmarks = []
3300   $suites.each {
3301     | suite |
3302     $benchmarks += suite.benchmarks
3303   }
3304   
3305   if $suites.empty? or $benchmarks.empty?
3306     $stderr.puts "No benchmarks found.  Bailing out."
3307     exit 1
3308   end
3309   
3310   if $outer*$inner == 1
3311     $stderr.puts "Warning: will only collect one sample per benchmark/VM.  Confidence interval calculation will fail."
3312   end
3313   
3314   $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
3315   
3316   $runPlans = []
3317   $vms.each {
3318     | vm |
3319     $benchmarks.each {
3320       | benchmark |
3321       $outer.times {
3322         | iteration |
3323         $runPlans << BenchRunPlan.new(benchmark, vm, iteration)
3324       }
3325     }
3326   }
3327   
3328   $runPlans.shuffle!
3329   
3330   if $sunSpiderWarmup
3331     warmupPlans = []
3332     $vms.each {
3333       | vm |
3334       WARMUP.benchmarks.each {
3335         | benchmark |
3336         warmupPlans << BenchRunPlan.new(benchmark, vm, 0)
3337       }
3338     }
3339     
3340     $runPlans = warmupPlans.shuffle + $runPlans
3341   end
3342   
3343   $suitepad = $suites.collect {
3344     | suite |
3345     suite.to_s.size
3346   }.max + 1
3347   
3348   $planpad = $runPlans.collect {
3349     | plan |
3350     plan.to_s.size
3351   }.max + 1
3352   
3353   maxBenchNameLength =
3354     ($benchmarks + ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
3355     | benchmark |
3356     if benchmark.respond_to? :name
3357       benchmark.name.size
3358     else
3359       benchmark.size
3360     end
3361   }.max
3362   $benchNameClip = 40
3363   $benchpad = [maxBenchNameLength, $benchNameClip].min + 1
3364   
3365   $weightpad = $benchmarks.collect {
3366     | benchmark |
3367     benchmark.weightString.size
3368   }.max
3369
3370   $vmpad = $vms.collect {
3371     | vm |
3372     vm.to_s.size
3373   }.max + 1
3374   
3375   $analyze.each_with_index {
3376     | filename, index |
3377     if index >= 1
3378       puts
3379     end
3380     parseAndDisplayResults(IO::read(filename))
3381   }
3382   
3383   if not $prepare and not $run
3384     exit 0
3385   end
3386   
3387   if FileTest.exist? BENCH_DATA_PATH
3388     cmd = "rm -rf #{BENCH_DATA_PATH}"
3389     $stderr.puts ">> #{cmd}" if $verbosity >= 2
3390     raise unless system cmd
3391   end
3392   
3393   Dir.mkdir BENCH_DATA_PATH
3394   
3395   if $needToCopyVMs
3396     canCopyIntoBenchPath = true
3397     $vms.each {
3398       | vm |
3399       canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath
3400     }
3401     
3402     if canCopyIntoBenchPath
3403       $vms.each {
3404         | vm |
3405         $stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..."
3406         vm.copyIntoBenchPath
3407       }
3408       $stderr.puts "All VMs are in place."
3409     else
3410       $stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it."
3411     end
3412   end
3413   
3414   if $measureGC and $measureGC != true
3415     found = false
3416     $vms.each {
3417       | vm |
3418       if vm.name == $measureGC
3419         found = true
3420       end
3421     }
3422     unless found
3423       $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
3424     end
3425   end
3426   
3427   if $prepare
3428     File.open("#{BENCH_DATA_PATH}/runscript", "w") {
3429       | file |
3430       file.puts "echo -e \"HOSTNAME:\\c\""
3431       file.puts "hostname"
3432       file.puts "echo"
3433       file.puts "echo -e \"HARDWARE:\\c\""
3434       file.puts "/usr/sbin/sysctl hw.model"
3435       file.puts "echo"
3436       file.puts "set -e"
3437       $script = file
3438       $runPlans.each_with_index {
3439         | plan, idx |
3440         if $verbosity == 0 and not $silent
3441           text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s
3442           text2 = plan.to_s
3443           file.puts("echo -e " + Shellwords.shellescape("\r#{text1} #{rpad(text2,$planpad)}") + "\"\\c\" 1>&2")
3444           file.puts("echo -e " + Shellwords.shellescape("\r#{text1} #{text2}") + "\"\\c\" 1>&2")
3445         end
3446         plan.emitRunCode
3447       }
3448       if $verbosity == 0 and not $silent
3449         file.puts("echo -e " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}") + "\"\\c\" 1>&2")
3450         file.puts("echo -e " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size}") + " 1>&2")
3451       end
3452     }
3453   end
3454   
3455   if $run
3456     unless $remoteHosts.empty?
3457       $stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0
3458       Dir.chdir(TEMP_PATH) {
3459         cmd = "tar -czf payload.tar.gz benchdata"
3460         $stderr.puts ">> #{cmd}" if $verbosity>=2
3461         raise unless system(cmd)
3462       }
3463       
3464       def grokHost(host)
3465         if host =~ /:([0-9]+)$/
3466           "-p " + $1 + " " + Shellwords.shellescape($~.pre_match)
3467         else
3468           Shellwords.shellescape(host)
3469         end
3470       end
3471       
3472       def sshRead(host, command)
3473         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3474         $stderr.puts ">> #{cmd}" if $verbosity>=2
3475         result = ""
3476         IO.popen(cmd, "r") {
3477           | inp |
3478           inp.each_line {
3479             | line |
3480             $stderr.puts "#{host}: #{line}" if $verbosity>=2
3481             result += line
3482           }
3483         }
3484         raise "#{$?}" unless $?.success?
3485         result
3486       end
3487       
3488       def sshWrite(host, command, data)
3489         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3490         $stderr.puts ">> #{cmd}" if $verbosity>=2
3491         IO.popen(cmd, "w") {
3492           | outp |
3493           outp.write(data)
3494         }
3495         raise "#{$?}" unless $?.success?
3496       end
3497       
3498       $remoteHosts.each {
3499         | host |
3500         $stderr.puts "Sending benchmark payload to #{host}..." if $verbosity==0
3501         
3502         remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"]
3503         raise unless remoteTempPath
3504         
3505         sshWrite(host, "cd #{Shellwords.shellescape(remoteTempPath)} && rm -rf benchdata && tar -xz", IO::read("#{TEMP_PATH}/payload.tar.gz"))
3506         
3507         $stderr.puts "Running on #{host}..." if $verbosity==0
3508         
3509         parseAndDisplayResults(sshRead(host, "cd #{Shellwords.shellescape(remoteTempPath + '/benchdata')} && sh runscript"))
3510       }
3511     end
3512     
3513     if not $remoteHosts.empty? and $alsoLocal
3514       $stderr.puts "Running locally..."
3515     end
3516     
3517     if $remoteHosts.empty? or $alsoLocal
3518       parseAndDisplayResults(runAndGetResults)
3519     end
3520   end
3521   
3522   if $prepare and not $run and $analyze.empty?
3523     puts wrap("Benchmarking script and data are in #{BENCH_DATA_PATH}. You can run "+
3524               "the benchmarks and get the results by doing:", 78)
3525     puts
3526     puts "cd #{BENCH_DATA_PATH}"
3527     puts "sh runscript > results.txt"
3528     puts
3529     puts wrap("Then you can analyze the results by running bencher with the same arguments "+
3530               "as now, but replacing --prepare-only with --analyze results.txt.", 78)
3531   end
3532 rescue => e
3533   fail(e)
3534 end
3535