4da1c6c6d018d08a93acb7368b973d3daf757a63
[WebKit-https.git] / Tools / Scripts / run-jsc-benchmarks
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2011-2018 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 # 1. Redistributions of source code must retain the above copyright
9 #    notice, this list of conditions and the following disclaimer.
10 # 2. Redistributions in binary form must reproduce the above copyright
11 #    notice, this list of conditions and the following disclaimer in the
12 #    documentation and/or other materials provided with the distribution.
13 #
14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24 # THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'rubygems'
27
28 require 'getoptlong'
29 require 'pathname'
30 require 'shellwords'
31 require 'socket'
32
33 begin
34   require 'json'
35 rescue LoadError => e
36   $stderr.puts "It does not appear that you have the 'json' package installed.  Try running 'sudo gem install json'."
37   exit 1
38 end
39
40 SCRIPT_PATH = Pathname.new(__FILE__).realpath
41
42 raise unless SCRIPT_PATH.dirname.basename.to_s == "Scripts"
43 raise unless SCRIPT_PATH.dirname.dirname.basename.to_s == "Tools"
44
45 OPENSOURCE_PATH = SCRIPT_PATH.dirname.dirname.dirname
46 PERFORMANCETESTS_PATH = OPENSOURCE_PATH + "PerformanceTests"
47
48 SUNSPIDER_PATH = PERFORMANCETESTS_PATH + "SunSpider" + "tests" + "sunspider-1.0"
49 LONGSPIDER_PATH = PERFORMANCETESTS_PATH + "LongSpider"
50 V8_PATH = PERFORMANCETESTS_PATH + "SunSpider" + "tests" + "v8-v6"
51 TAILBENCH_PATH = PERFORMANCETESTS_PATH + "TailBench9000"
52 MICROBENCHMARKS_PATH = OPENSOURCE_PATH + "JSTests" + "microbenchmarks"
53 OPENSOURCE_OCTANE_PATH = PERFORMANCETESTS_PATH + "Octane"
54 OCTANE_WRAPPER_PATH = OPENSOURCE_OCTANE_PATH + "wrappers"
55 JSBENCH_PATH = PERFORMANCETESTS_PATH + "JSBench"
56 SIXSPEED_PATH = PERFORMANCETESTS_PATH + "SixSpeed" + "tests"
57 SIXSPEED_WRAPPER_PATH = PERFORMANCETESTS_PATH + "SixSpeed" + "wrappers"
58
59 TEMP_PATH = OPENSOURCE_PATH + "BenchmarkTemp"
60
61 if TEMP_PATH.exist?
62   raise unless TEMP_PATH.directory?
63 else
64   Dir.mkdir(TEMP_PATH)
65 end
66
67 BENCH_DATA_PATH = TEMP_PATH + "benchdata"
68
69 IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933, 
70             0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684, 
71             0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227, 
72             0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674, 
73             0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652, 
74             0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604, 
75             0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338, 
76             0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538, 
77             0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185, 
78             0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694, 
79             0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244, 
80             0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067, 
81             0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323, 
82             0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129, 
83             0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574, 
84             0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722, 
85             0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624, 
86             0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321, 
87             0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843, 
88             0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217, 
89             0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464, 
90             0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599, 
91             0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638, 
92             0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592, 
93             0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471, 
94             0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283, 
95             0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037, 
96             0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738, 
97             0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481, 
98             0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085, 
99             0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651, 
100             0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183, 
101             0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683, 
102             0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154, 
103             0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599, 
104             0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019, 
105             0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417, 
106             0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795, 
107             0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153, 
108             0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494, 
109             0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818, 
110             0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128, 
111             0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423, 
112             0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704, 
113             0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974, 
114             0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232, 
115             0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479, 
116             0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716, 
117             0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943, 
118             0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192, 
119             0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401, 
120             0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963, 
121             0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822, 
122             0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007, 
123             0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186, 
124             0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358, 
125             0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525, 
126             0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686, 
127             0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841, 
128             0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991, 
129             0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137, 
130             0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278, 
131             0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414, 
132             0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547, 
133             0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675, 
134             0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799, 
135             0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192, 
136             0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037, 
137             0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151, 
138             0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262, 
139             0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369, 
140             0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489, 
141             0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259, 
142             0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689, 
143             0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785, 
144             0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879, 
145             0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297, 
146             0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059, 
147             0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145, 
148             0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242, 
149             0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324, 
150             0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404, 
151             0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483, 
152             0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559, 
153             0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634, 
154             0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707, 
155             0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778, 
156             0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848, 
157             0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916, 
158             0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983, 
159             0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057, 
160             0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121, 
161             0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183, 
162             0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244, 
163             0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304, 
164             0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363, 
165             0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442, 
166             0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476, 
167             0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539, 
168             0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946, 
169             0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653, 
170             0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712, 
171             0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762, 
172             0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812, 
173             0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867, 
174             0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915, 
175             0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962, 
176             0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007, 
177             0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059, 
178             0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103, 
179             0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146, 
180             0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189, 
181             0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231, 
182             0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272, 
183             0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312, 
184             0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352, 
185             0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391, 
186             0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429, 
187             0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467, 
188             0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504, 
189             0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546, 
190             0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582, 
191             0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617, 
192             0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652, 
193             0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686, 
194             0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724, 
195             0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757, 
196             0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579, 
197             0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822, 
198             0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853, 
199             0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885, 
200             0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915, 
201             0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595, 
202             0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979, 
203             0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013, 
204             0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041, 
205             0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074, 
206             0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102, 
207             0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129, 
208             0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156, 
209             0.99616, 0.996164]
210
211 # Run-time configuration parameters (can be set with command-line options)
212
213 $rerun=1
214 $inner=1
215 $warmup=1
216 $outer=4
217 $quantum=1000
218 $includeSunSpider=true
219 $includeSunSpiderCompileTime=true
220 $includeLongSpider=false
221 $includeV8=true
222 $includeV8CompileTime=true
223 $includeKraken=true
224 $includeJSBench=true
225 $includeMicrobenchmarks=true
226 $includeAsmBench=true
227 $includeDSPJS=true
228 $includeBrowsermarkJS=false
229 $includeBrowsermarkDOM=false
230 $includeOctane=true
231 $includeCompressionBench = false
232 $includeSixSpeed = false
233 $includeTailBench = true
234 $measureGC=false
235 $benchmarkPattern=nil
236 $verbosity=0
237 $timeMode=:preciseTime
238 $forceVMKind=nil
239 $brief=false
240 $silent=false
241 $remoteHosts=[]
242 $alsoLocal=false
243 $sshOptions=[]
244 $vms = []
245 $environment = {}
246 $dependencies = []
247 $needToCopyVMs = false
248 $dontCopyVMs = false
249 $allDRT = true
250 $outputName = nil
251 $sunSpiderWarmup = true
252 $configPath = Pathname.new(ENV["HOME"]) + ".run-jsc-benchmarks"
253
254 $prepare = true
255 $run = true
256 $analyze = []
257
258 # Helpful functions and classes
259
260 def smallUsage
261   puts "Use the --help option to get basic usage information."
262   exit 1
263 end
264
265 def usage
266   puts "run-jsc-benchmarks [options] <vm1> [<vm2> ...]"
267   puts
268   puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
269   puts "benchmarks, and reports detailed statistics.  What makes run-jsc-benchmarks"
270   puts "special is that each benchmark/VM configuration is run in a single VM invocation,"
271   puts "and the invocations are run in random order.  This minimizes systematics due to"
272   puts "one benchmark polluting the running time of another.  The fine-grained"
273   puts "interleaving of VM invocations further minimizes systematics due to changes in"
274   puts "the performance or behavior of your machine."
275   puts 
276   puts "Run-jsc-benchmarks is highly configurable.  You can compare as many VMs as you"
277   puts "like.  You can change the amount of warm-up iterations, number of iterations"
278   puts "executed per VM invocation, and the number of VM invocations per benchmark."
279   puts
280   puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
281   puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
282   puts "the executable and <name> is the name that you would like to give the"
283   puts "configuration for the purposeof reporting.  If no name is given, a generic name"
284   puts "of the form Conf#<n> will be ascribed to the configuration automatically."
285   puts
286   puts "It's also possible to specify per-VM environment variables. For example, you"
287   puts "might specify a VM like Foo:JSC_useJIT=false:/path/to/jsc, in which case the"
288   puts "harness will set the JSC_useJIT environment variable to false just before running"
289   puts "the given VM. Note that the harness will not unset the environment variable, so"
290   puts "you must ensure that your other VMs will use the opposite setting"
291   puts "(JSC_useJIT=true in this case)."
292   puts
293   puts "Options:"
294   puts "--rerun <n>          Set the number of iterations of the benchmark that"
295   puts "                     contribute to the measured run time.  Default is #{$rerun}."
296   puts "--inner <n>          Set the number of inner (per-runtime-invocation)"
297   puts "                     iterations.  Default is #{$inner}."
298   puts "--outer <n>          Set the number of runtime invocations for each benchmark."
299   puts "                     Default is #{$outer}."
300   puts "--warmup <n>         Set the number of warm-up runs per invocation.  Default"
301   puts "                     is #{$warmup}. This has a different effect on different kinds"
302   puts "                     benchmarks. Some benchmarks have no notion of warm-up."
303   puts "--no-ss-warmup       Disable SunSpider-based warm-up runs."
304   puts "--quantum <n>        Set the duration in milliseconds for which an iteration of"
305   puts "                     a throughput benchmark should be run.  Default is #{$quantum}."
306   puts "--timing-mode        Set the way that time is measured.  Possible values"
307   puts "                     are 'preciseTime' and 'date'.  Default is 'preciseTime'."
308   puts "--force-vm-kind      Turn off auto-detection of VM kind, and assume that it is"
309   puts "                     the one specified.  Valid arguments are 'jsc', "
310   puts "                     'DumpRenderTree', or 'WebKitTestRunner'."
311   puts "--force-vm-copy      Force VM builds to be copied to the working directory."
312   puts "                     This may reduce pathologies resulting from path names."
313   puts "--dont-copy-vms      Don't copy VMs even when doing a remote benchmarking run;"
314   puts "                     instead assume that they are already there."
315   puts "--sunspider          Only run SunSpider."
316   puts "--sunspider-compile-time"
317   puts "                     Only run the SunSpider compile time benchmark."
318   puts "--v8-spider          Only run SunSpider-style V8."
319   puts "--v8-spider-compile-time"
320   puts "                     Only run the SunSpider-style V8 compile time benchmark."
321   puts "--kraken             Only run Kraken."
322   puts "--js-bench           Only run JSBench."
323   puts "--microbenchmarks    Only run microbenchmarks."
324   puts "--dsp                Only run DSP."
325   puts "--asm-bench          Only run AsmBench."
326   puts "--browsermark-js     Only run browsermark-js."
327   puts "--browsermark-dom    Only run browsermark-dom."
328   puts "--octane             Only run Octane."
329   puts "--tail-bench         Only run TailBench"
330   puts "--compression-bench  Only run compression bench"
331   puts "                     The default is to run all benchmarks. The above options can"
332   puts "                     be combined to run any subset (so --sunspider --dsp will run"
333   puts "                     both SunSpider and DSP)."
334   puts "--six-speed          Only run SixSpeed."
335   puts "--benchmarks         Only run benchmarks matching the given regular expression."
336   puts "--measure-gc         Turn off manual calls to gc(), so that GC time is measured."
337   puts "                     Works best with large values of --inner.  You can also say"
338   puts "                     --measure-gc <conf>, which turns this on for one"
339   puts "                     configuration only."
340   puts "--verbose or -v      Print more stuff."
341   puts "--brief              Print only the final result for each VM."
342   puts "--silent             Don't print progress. This might slightly reduce some"
343   puts "                     performance perturbation."
344   puts "--remote <sshhosts>  Perform performance measurements remotely, on the given"
345   puts "                     SSH host(s). Easiest way to use this is to specify the SSH"
346   puts "                     user@host string. However, you can also supply a comma-"
347   puts "                     separated list of SSH hosts. Alternatively, you can use this"
348   puts "                     option multiple times to specify multiple hosts. This"
349   puts "                     automatically copies the WebKit release builds of the VMs"
350   puts "                     you specified to all of the hosts."
351   puts "--ssh-options        Pass additional options to SSH."
352   puts "--local              Also do a local benchmark run even when doing --remote."
353   puts "--vms                Use a JSON file to specify which VMs to run, as opposed to"
354   puts "                     specifying them on the command line."
355   puts "--prepare-only       Only prepare the runscript (a shell script that"
356   puts "                     invokes the VMs to run benchmarks) but don't run it."
357   puts "--analyze            Only read the output of the runscript but don't do anything"
358   puts "                     else. This requires passing the same arguments that you"
359   puts "                     passed when running --prepare-only."
360   puts "--output-name        Base of the filenames to put results into. Will write a file"
361   puts "                     called <base>_report.txt and <base>.json.  By default this"
362   puts "                     name is automatically synthesized from the machine name,"
363   puts "                     date, set of benchmarks run, and set of configurations."
364   puts "--environment        JSON file that specifies the environment variables that should"
365   puts "                     be used for particular VMs and benchmarks."
366   puts "--config <path>      Specify the path of the configuration file. Defaults to"
367   puts "                     ~/.run-jsc-benchmarks"
368   puts "--dependencies       Additional dependent library paths."
369   puts "--help or -h         Display this message."
370   puts
371   puts "Example:"
372   puts "run-jsc-benchmarks TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
373   exit 1
374 end
375
376 def fail(reason)
377   if reason.respond_to? :backtrace
378     puts "FAILED: #{reason.inspect}"
379     puts "Stack trace:"
380     puts reason.backtrace.join("\n")
381   else
382     puts "FAILED: #{reason.inspect}"
383   end
384   smallUsage
385 end
386
387 def quickFail(r1,r2)
388   $stderr.puts "#{$0}: #{r1}"
389   puts
390   fail(r2)
391 end
392
393 def intArg(argName,arg,min,max)
394   result=arg.to_i
395   unless result.to_s == arg
396     quickFail("Expected an integer value for #{argName}, but got #{arg}.",
397               "Invalid argument for command-line option")
398   end
399   if min and result<min
400     quickFail("Argument for #{argName} cannot be smaller than #{min}.",
401               "Invalid argument for command-line option")
402   end
403   if max and result>max
404     quickFail("Argument for #{argName} cannot be greater than #{max}.",
405               "Invalid argument for command-line option")
406   end
407   result
408 end
409
410 def computeMean(array)
411   sum=0.0
412   array.each {
413     | value |
414     sum += value
415   }
416   sum/array.length
417 end
418
419 def computeGeometricMean(array)
420   sum = 0.0
421   array.each {
422     | value |
423     sum += Math.log(value)
424   }
425   Math.exp(sum * (1.0/array.length))
426 end
427
428 def computeHarmonicMean(array)
429   1.0 / computeMean(array.collect{ | value | 1.0 / value })
430 end
431
432 def computeStdDev(array)
433   case array.length
434   when 0
435     0.0/0.0
436   when 1
437     0.0
438   else
439     begin
440       mean=computeMean(array)
441       sum=0.0
442       array.each {
443         | value |
444         sum += (value-mean)**2
445       }
446       Math.sqrt(sum/(array.length-1))
447     rescue
448       0.0/0.0
449     end
450   end
451 end
452
453 class Array
454   def shuffle!
455     size.downto(1) { |n| push delete_at(rand(n)) }
456     self
457   end
458 end
459
460 def inverseBetaRegularized(n)
461   IBR_LOOKUP[n-1]
462 end
463
464 def numToStr(num, decimalShift)
465   ("%." + (4 + decimalShift).to_s + "f") % (num.to_f)
466 end
467
468 class CantSay
469   def initialize
470   end
471   
472   def shortForm
473     " "
474   end
475   
476   def longForm
477     ""
478   end
479   
480   def to_s
481     ""
482   end
483 end
484   
485 class NoChange
486   attr_reader :amountFaster
487   
488   def initialize(amountFaster)
489     @amountFaster = amountFaster
490   end
491   
492   def shortForm
493     " "
494   end
495   
496   def longForm
497     "  might be #{numToStr(@amountFaster, 0)}x faster"
498   end
499   
500   def to_s
501     if @amountFaster < 1.01
502       ""
503     else
504       longForm
505     end
506   end
507 end
508
509 class Faster
510   attr_reader :amountFaster
511   
512   def initialize(amountFaster)
513     @amountFaster = amountFaster
514   end
515   
516   def shortForm
517     "^"
518   end
519   
520   def longForm
521     "^ definitely #{numToStr(@amountFaster, 0)}x faster"
522   end
523   
524   def to_s
525     longForm
526   end
527 end
528
529 class Slower
530   attr_reader :amountSlower
531   
532   def initialize(amountSlower)
533     @amountSlower = amountSlower
534   end
535   
536   def shortForm
537     "!"
538   end
539   
540   def longForm
541     "! definitely #{numToStr(@amountSlower, 0)}x slower"
542   end
543   
544   def to_s
545     longForm
546   end
547 end
548
549 class MayBeSlower
550   attr_reader :amountSlower
551   
552   def initialize(amountSlower)
553     @amountSlower = amountSlower
554   end
555   
556   def shortForm
557     "?"
558   end
559   
560   def longForm
561     "? might be #{numToStr(@amountSlower, 0)}x slower"
562   end
563   
564   def to_s
565     if @amountSlower < 1.01
566       "?"
567     else
568       longForm
569     end
570   end
571 end
572
573 def jsonSanitize(value)
574   if value.is_a? Fixnum
575     value
576   elsif value.is_a? Float
577     if value.nan? or value.infinite?
578       value.to_s
579     else
580       value
581     end
582   elsif value.is_a? Array
583     value.map{|v| jsonSanitize(v)}
584   elsif value.nil?
585     value
586   else
587     raise "Unrecognized value #{value.inspect}"
588   end
589 end
590
591 class Stats
592   def initialize
593     @array = []
594   end
595   
596   def add(value)
597     if not value or not @array
598       @array = nil
599     elsif value.is_a? Float
600       if value.nan? or value.infinite?
601         @array = nil
602       else
603         @array << value
604       end
605     elsif value.is_a? Stats
606       add(value.array)
607     elsif value.respond_to? :each
608       value.each {
609         | v |
610         add(v)
611       }
612     else
613       @array << value.to_f
614     end
615   end
616   
617   def status
618     if @array
619       :ok
620     else
621       :error
622     end
623   end
624   
625   def error?
626     # TODO: We're probably still not handling this case correctly. 
627     not @array or @array.empty?
628   end
629   
630   def ok?
631     not not @array
632   end
633     
634   def array
635     @array
636   end
637   
638   def sum
639     result=0
640     @array.each {
641       | value |
642       result += value
643     }
644     result
645   end
646   
647   def min
648     @array.min
649   end
650   
651   def max
652     @array.max
653   end
654   
655   def size
656     @array.length
657   end
658   
659   def mean
660     computeMean(array)
661   end
662   
663   def arithmeticMean
664     mean
665   end
666   
667   def stdDev
668     computeStdDev(array)
669   end
670
671   def stdErr
672     stdDev/Math.sqrt(size)
673   end
674   
675   # Computes a 95% Student's t distribution confidence interval
676   def confInt
677     if size < 2
678       0.0/0.0
679     else
680       raise if size > 1000
681       Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
682     end
683   end
684   
685   def lower
686     mean-confInt
687   end
688   
689   def upper
690     mean+confInt
691   end
692   
693   def geometricMean
694     computeGeometricMean(array)
695   end
696   
697   def harmonicMean
698     computeHarmonicMean(array)
699   end
700   
701   def compareTo(other)
702     return CantSay.new unless ok? and other.ok?
703     
704     if upper < other.lower
705       Faster.new(other.mean/mean)
706     elsif lower > other.upper
707       Slower.new(mean/other.mean)
708     elsif mean > other.mean
709       MayBeSlower.new(mean/other.mean)
710     else
711       NoChange.new(other.mean/mean)
712     end
713   end
714   
715   def to_s
716     "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
717   end
718   
719   def jsonMap
720     if ok?
721       {"data"=>jsonSanitize(@array), "mean"=>jsonSanitize(mean), "confInt"=>jsonSanitize(confInt)}
722     else
723       "ERROR"
724     end
725   end
726 end
727
728 def doublePuts(out1,out2,msg)
729   out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
730   out2.puts msg
731 end
732
733 class Benchfile < File
734   @@counter = 0
735   
736   attr_reader :filename, :basename
737   
738   def initialize(name)
739     @basename, @filename = Benchfile.uniqueFilename(name)
740     super(@filename, "w")
741   end
742   
743   def self.uniqueFilename(name)
744     if name.is_a? Array
745       basename = name[0] + @@counter.to_s + name[1]
746     else
747       basename = name + @@counter.to_s
748     end
749     filename = BENCH_DATA_PATH + basename
750     @@counter += 1
751     raise "Benchfile #{filename} already exists" if FileTest.exist?(filename)
752     [basename, filename]
753   end
754   
755   def self.create(name)
756     file = Benchfile.new(name)
757     yield file
758     file.close
759     file.basename
760   end
761 end
762
763 $dataFiles={}
764 def ensureFile(key, filename)
765   unless $dataFiles[key]
766     $dataFiles[key] = Benchfile.create(key) {
767       | outp |
768       doublePuts($stderr,outp,IO::read(filename))
769     }
770   end
771   $dataFiles[key]
772 end
773
774 # Helper for files that cannot be renamed.
775 $absoluteFiles={}
776 def ensureAbsoluteFile(filename, basedir=nil)
777   return if $absoluteFiles[filename]
778   filename = Pathname.new(filename)
779
780   directory = Pathname.new('')
781   if basedir and filename.dirname != basedir
782     remainingPath = filename.dirname
783     while remainingPath != basedir
784       directory = remainingPath.basename + directory
785       remainingPath = remainingPath.dirname
786     end
787     if not $absoluteFiles[directory]
788       cmd = "mkdir -p #{Shellwords.shellescape((BENCH_DATA_PATH + directory).to_s)}"
789       $stderr.puts ">> #{cmd}" if $verbosity >= 2
790       raise unless system(cmd)
791       intermediateDirectory = Pathname.new(directory)
792       while intermediateDirectory.basename.to_s != "."
793         $absoluteFiles[intermediateDirectory] = true
794         intermediateDirectory = intermediateDirectory.dirname
795       end
796     end
797   end
798   
799   cmd = "cp #{Shellwords.shellescape(filename.to_s)} #{Shellwords.shellescape((BENCH_DATA_PATH + directory + filename.basename).to_s)}"
800   $stderr.puts ">> #{cmd}" if $verbosity >= 2
801   raise unless system(cmd)
802   $absoluteFiles[filename] = true
803 end
804
805 # Helper for large benchmarks with lots of files and directories.
806 def ensureBenchmarkFiles(rootdir)
807     toProcess = [rootdir]
808     while not toProcess.empty?
809       currdir = toProcess.pop
810       Dir.foreach(currdir.to_s) {
811         | filename |
812         path = currdir + filename
813         next if filename.match(/^\./)
814         toProcess.push(path) if File.directory?(path.to_s)
815         ensureAbsoluteFile(path, rootdir) if File.file?(path.to_s)
816       }
817     end
818 end
819
820 class JSCommand
821   attr_reader :js, :html
822   def initialize(js, html)
823     @js = js
824     @html = html
825   end
826 end
827
828 def loadCommandForFile(key, filename)
829   file = ensureFile(key, filename)
830   JSCommand.new("load(#{file.inspect});", "<script src=#{file.inspect}></script>")
831 end
832
833 def simpleCommand(command)
834   JSCommand.new(command, "<script type=\"text/javascript\">#{command}</script>")
835 end
836
837 # Benchmark that consists of a single file and must be loaded in its own global object each
838 # time (i.e. run()).
839 class SingleFileTimedBenchmarkParameters
840   attr_reader :benchPath
841   
842   def initialize(benchPath)
843     @benchPath = benchPath
844   end
845   
846   def kind
847     :singleFileTimedBenchmark
848   end
849 end
850
851 # Benchmark that consists of a single file and must be loaded in its own global object each
852 # time (i.e. run()), but returns its result in a custom way.
853 class SingleFileCustomTimedBenchmarkParameters
854   attr_reader :benchPath
855   
856   def initialize(benchPath)
857     @benchPath = benchPath
858   end
859   
860   def kind
861     :singleFileCustomTimedBenchmark
862   end
863 end
864
865 # Benchmark that consists of a single file and must be loaded in its own global object each
866 # time (i.e. run()), and returns the time spent compiling
867 class SingleFileCompileTimeBenchmarkParameters
868   attr_reader :benchPath
869   
870   def initialize(benchPath)
871     @benchPath = benchPath
872   end
873   
874   def kind
875     :singleFileCompileTimeBenchmark
876   end
877 end
878
879 # Benchmark that consists of one or more data files that should be loaded globally, followed
880 # by a command to run the benchmark.
881 class MultiFileTimedBenchmarkParameters
882   attr_reader :dataPaths, :command
883
884   def initialize(dataPaths, command)
885     @dataPaths = dataPaths
886     @command = command
887   end
888   
889   def kind
890     :multiFileTimedBenchmark
891   end
892 end
893
894 # Benchmark that consists of one or more data files that should be loaded globally, followed
895 # by a command to run a short tick of the benchmark. The benchmark should be run for as many
896 # ticks as possible, for one quantum (quantum is 1000ms by default).
897 class ThroughputBenchmarkParameters
898   attr_reader :dataPaths, :setUpCommand, :command, :tearDownCommand, :doWarmup, :deterministic, :minimumIterations
899
900   def initialize(dataPaths, setUpCommand, command, tearDownCommand, doWarmup, deterministic, minimumIterations)
901     @dataPaths = dataPaths
902     @setUpCommand = setUpCommand
903     @command = command
904     @tearDownCommand = tearDownCommand
905     @doWarmup = doWarmup
906     @deterministic = deterministic
907     @minimumIterations = minimumIterations
908   end
909   
910   def kind
911     :throughputBenchmark
912   end
913 end
914
915 # Benchmark that can only run in DumpRenderTree or WebKitTestRunner, that has its own callback for reporting
916 # results. Other than that it's just like SingleFileTimedBenchmark.
917 class SingleFileTimedCallbackBenchmarkParameters
918   attr_reader :callbackDecl, :benchPath
919   
920   def initialize(callbackDecl, benchPath)
921     @callbackDecl = callbackDecl
922     @benchPath = benchPath
923   end
924   
925   def kind
926     :singleFileTimedCallbackBenchmark
927   end
928 end
929
930 def emitTimerFunctionCode(file)
931   case $timeMode
932   when :preciseTime
933     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
934     doublePuts($stderr,file,"   return preciseTime()*1000")
935     doublePuts($stderr,file,"}")
936   when :date
937     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
938     doublePuts($stderr,file,"   return Date.now()")
939     doublePuts($stderr,file,"}")
940   else
941     raise
942   end
943 end
944
945 def emitBenchRunCodeFile(name, plan, benchParams)
946   case plan.vm.vmType
947   when :jsc
948     Benchfile.create("bencher") {
949       | file |
950       emitTimerFunctionCode(file)
951       
952       if benchParams.kind == :multiFileTimedBenchmark
953         benchParams.dataPaths.each {
954           | path |
955           doublePuts($stderr,file,"load(#{path.inspect});")
956         }
957         doublePuts($stderr,file,"gc();")
958         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
959         doublePuts($stderr,file,"   var __before = __bencher_curTimeMS();")
960         $rerun.times {
961           doublePuts($stderr,file,"   #{benchParams.command.js}")
962         }
963         doublePuts($stderr,file,"   var __after = __bencher_curTimeMS();")
964         doublePuts($stderr,file,"   if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(__after-__before));");
965         doublePuts($stderr,file,"   gc();") unless plan.vm.shouldMeasureGC
966         doublePuts($stderr,file,"}")
967       elsif benchParams.kind == :throughputBenchmark
968         emitTimerFunctionCode(file)
969         benchParams.dataPaths.each {
970           | path |
971           doublePuts($stderr,file,"load(#{path.inspect});")
972         }
973         doublePuts($stderr,file,"#{benchParams.setUpCommand.js}")
974         if benchParams.doWarmup
975           warmup = $warmup
976         else
977           warmup = 0
978         end
979         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{warmup + $inner}; __bencher_index++) {")
980         doublePuts($stderr,file,"    var __before = __bencher_curTimeMS();")
981         doublePuts($stderr,file,"    var __after = __before;")
982         doublePuts($stderr,file,"    var __runs = 0;")
983         doublePuts($stderr,file,"    var __expected = #{$quantum};")
984         doublePuts($stderr,file,"    while (true) {")
985         $rerun.times {
986           doublePuts($stderr,file,"       #{benchParams.command.js}")
987         }
988         doublePuts($stderr,file,"       __runs++;")
989         doublePuts($stderr,file,"       __after = __bencher_curTimeMS();")
990         if benchParams.deterministic
991           doublePuts($stderr,file,"       if (true) {")
992         else
993           doublePuts($stderr,file,"       if (__after - __before >= __expected) {")
994         end
995         doublePuts($stderr,file,"           if (__runs >= #{benchParams.minimumIterations} || __bencher_index < #{warmup})")
996         doublePuts($stderr,file,"               break;")
997         doublePuts($stderr,file,"           __expected += #{$quantum}")
998         doublePuts($stderr,file,"       }")
999         doublePuts($stderr,file,"    }")
1000         doublePuts($stderr,file,"    if (__bencher_index >= #{warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{warmup}) + \": Time: \"+((__after-__before)/__runs));")
1001         doublePuts($stderr,file,"}")
1002         doublePuts($stderr,file,"#{benchParams.tearDownCommand.js}")
1003       elsif benchParams.kind == :singleFileCustomTimedBenchmark
1004         doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
1005         doublePuts($stderr,file,"   var __bencher_result = {value: null};")
1006         $rerun.times {
1007           doublePuts($stderr,file,"   run(__bencher_what, __bencher_result);")
1008         }
1009         doublePuts($stderr,file,"   if (__bencher_result.value == null)")
1010         doublePuts($stderr,file,"      throw new Error(\"Null result\");")
1011         doublePuts($stderr,file,"   return __bencher_result.value;")
1012         doublePuts($stderr,file,"}")
1013         $warmup.times {
1014           doublePuts($stderr,file,"__bencher_run(#{benchParams.benchPath.inspect})")
1015           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
1016         }
1017         $inner.times {
1018           | innerIndex |
1019           doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchParams.benchPath.inspect}));")
1020           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
1021         }
1022       elsif benchParams.kind == :singleFileCompileTimeBenchmark
1023         doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
1024         doublePuts($stderr,file,"   var __compileTimeBefore = totalCompileTime();")
1025         $rerun.times {
1026           doublePuts($stderr,file,"   run(__bencher_what);")
1027         }
1028         doublePuts($stderr,file,"   return totalCompileTime() - __compileTimeBefore;")
1029         doublePuts($stderr,file,"}")
1030         $warmup.times {
1031           doublePuts($stderr,file,"__bencher_run(#{benchParams.benchPath.inspect})")
1032           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
1033         }
1034         $inner.times {
1035           | innerIndex |
1036           doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchParams.benchPath.inspect}));")
1037           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
1038         }
1039       else
1040         raise unless benchParams.kind == :singleFileTimedBenchmark
1041         doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
1042         doublePuts($stderr,file,"   var __bencher_before = __bencher_curTimeMS();")
1043         $rerun.times {
1044           doublePuts($stderr,file,"   run(__bencher_what);")
1045         }
1046         doublePuts($stderr,file,"   var __bencher_after = __bencher_curTimeMS();")
1047         doublePuts($stderr,file,"   return __bencher_after - __bencher_before;")
1048         doublePuts($stderr,file,"}")
1049         $warmup.times {
1050           doublePuts($stderr,file,"__bencher_run(#{benchParams.benchPath.inspect})")
1051           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
1052         }
1053         $inner.times {
1054           | innerIndex |
1055           doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchParams.benchPath.inspect}));")
1056           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
1057         }
1058       end
1059     }
1060   when :dumpRenderTree, :webkitTestRunner
1061     case $timeMode
1062     when :preciseTime
1063       curTime = "(testRunner.preciseTime()*1000)"
1064     when :date
1065       curTime = "(Date.now())"
1066     else
1067       raise
1068     end
1069
1070     mainCode = Benchfile.create("bencher") {
1071       | file |
1072       doublePuts($stderr,file,"__bencher_count = 0;")
1073       doublePuts($stderr,file,"function __bencher_doNext(result) {")
1074       doublePuts($stderr,file,"    if (__bencher_count >= #{$warmup})")
1075       doublePuts($stderr,file,"        debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);")
1076       doublePuts($stderr,file,"    __bencher_count++;")
1077       doublePuts($stderr,file,"    if (__bencher_count < #{$inner+$warmup})")
1078       doublePuts($stderr,file,"        __bencher_runImpl(__bencher_doNext);")
1079       doublePuts($stderr,file,"    else")
1080       doublePuts($stderr,file,"        quit();")
1081       doublePuts($stderr,file,"}")
1082       doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);")
1083     }
1084     
1085     cssCode = Benchfile.create("bencher-css") {
1086       | file |
1087       doublePuts($stderr,file,".pass {\n    font-weight: bold;\n    color: green;\n}\n.fail {\n    font-weight: bold;\n    color: red;\n}\n\#console {\n    white-space: pre-wrap;\n    font-family: monospace;\n}")
1088     }
1089     
1090     preCode = Benchfile.create("bencher-pre") {
1091       | file |
1092       doublePuts($stderr,file,"if (window.testRunner) {")
1093       doublePuts($stderr,file,"    testRunner.dumpAsText(window.enablePixelTesting);")
1094       doublePuts($stderr,file,"    testRunner.waitUntilDone();")
1095       doublePuts($stderr,file,"}")
1096       doublePuts($stderr,file,"")
1097       doublePuts($stderr,file,"function debug(msg)")
1098       doublePuts($stderr,file,"{")
1099       doublePuts($stderr,file,"    var span = document.createElement(\"span\");")
1100       doublePuts($stderr,file,"    document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace")
1101       doublePuts($stderr,file,"    span.innerHTML = msg + '<br />';")
1102       doublePuts($stderr,file,"}")
1103       doublePuts($stderr,file,"")
1104       doublePuts($stderr,file,"function quit() {")
1105       doublePuts($stderr,file,"    testRunner.notifyDone();")
1106       doublePuts($stderr,file,"}")
1107       doublePuts($stderr,file,"")
1108       doublePuts($stderr,file,"__bencher_continuation=null;")
1109       doublePuts($stderr,file,"")
1110       doublePuts($stderr,file,"function reportResult(result) {")
1111       doublePuts($stderr,file,"    __bencher_continuation(result);")
1112       doublePuts($stderr,file,"}")
1113       doublePuts($stderr,file,"")
1114       doublePuts($stderr,file,"function currentTimeInMS(msg)")
1115       doublePuts($stderr,file,"{")
1116       doublePuts($stderr,file,"    return #{curTime};")
1117       doublePuts($stderr,file,"}")
1118       if benchParams.kind == :singleFileTimedCallbackBenchmark
1119         doublePuts($stderr,file,"")
1120         doublePuts($stderr,file,benchParams.callbackDecl)
1121       end
1122       doublePuts($stderr,file,"")
1123       doublePuts($stderr,file,"function __bencher_runImpl(continuation) {")
1124       doublePuts($stderr,file,"    function doit() {")
1125       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"\";")
1126       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";")
1127       doublePuts($stderr,file,"        var testFrame = document.getElementById(\"testframe\");")
1128       doublePuts($stderr,file,"        testFrame.contentDocument.open();")
1129       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");")
1130       if benchParams.kind == :throughputBenchmark or benchParams.kind == :multiFileTimedBenchmark
1131         benchParams.dataPaths.each {
1132           | path |
1133           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{path.inspect.inspect[1..-2]}></script>\");")
1134         }
1135       end
1136       if benchParams.kind == :throughputBenchmark
1137         if benchParams.doWarmup
1138           warmup = $warmup
1139         else
1140           warmup = 0
1141         end
1142         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">\");")
1143         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.setUpCommand.js}\");")
1144         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_before = #{curTime};\");")
1145         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_after = __bencher_before;\");")
1146         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_expected = #{$quantum};\");")
1147         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_runs = 0;\");")
1148         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"while (true) {\");")
1149         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    #{benchParams.command.js}\");")
1150         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_runs++;\");")
1151         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_after = #{curTime};\");")
1152         if benchParams.deterministic
1153           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (true) {\");")
1154         else
1155           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (__bencher_after - __bencher_before >= __bencher_expected) {\");")
1156         end
1157         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        if (__bencher_runs >= #{benchParams.minimumIterations} || window.parent.__bencher_count < #{warmup})\");")
1158         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"            break;\");")
1159         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        __bencher_expected += #{$quantum}\");")
1160         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    }\");")
1161         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"}\");")
1162         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.tearDownCommand.js}\");")
1163         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"window.parent.reportResult((__bencher_after - __bencher_before) / __bencher_runs);\");")
1164         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</script>\");")
1165       else
1166         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">var __bencher_before = #{curTime};</script>\");")
1167         if benchParams.kind == :multiFileTimedBenchmark
1168           doublePuts($stderr,file,"        testFrame.contentDocument.write(#{benchParams.command.html.inspect});")
1169         else
1170           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{benchParams.benchPath.inspect.inspect[1..-2]}></script>\");")
1171         end
1172         unless benchParams.kind == :singleFileTimedCallbackBenchmark
1173           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">window.parent.reportResult(#{curTime} - __bencher_before);</script>\");")
1174         end
1175       end
1176       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</body></html>\");")
1177       doublePuts($stderr,file,"        testFrame.contentDocument.close();")
1178       doublePuts($stderr,file,"    }")
1179       doublePuts($stderr,file,"    __bencher_continuation = continuation;")
1180       doublePuts($stderr,file,"    window.setTimeout(doit, 10);")
1181       doublePuts($stderr,file,"}")
1182     }
1183
1184     Benchfile.create(["bencher-htmldoc",".html"]) {
1185       | file |
1186       doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>")
1187     }
1188   else
1189     raise
1190   end
1191 end
1192
1193 def emitBenchRunCode(name, plan, benchParams)
1194   plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchParams), plan)
1195 end
1196
1197 class FileCreator
1198   def initialize(filename)
1199     @filename = filename
1200     @state = :empty
1201   end
1202   
1203   def puts(text)
1204     $script.print "echo #{Shellwords.shellescape(text)}"
1205     if @state == :empty
1206       $script.print " > "
1207       @state = :nonEmpty
1208     else
1209       $script.print " >> "
1210     end
1211     $script.puts "#{Shellwords.shellescape(@filename)}"
1212   end
1213   
1214   def close
1215     if @state == :empty
1216       $script.puts "rm -f #{Shellwords.shellescape(text)}"
1217       $script.puts "touch #{Shellwords.shellescape(text)}"
1218     end
1219   end
1220   
1221   def self.open(filename)
1222     outp = FileCreator.new(filename)
1223     yield outp
1224     outp.close
1225   end
1226 end
1227
1228 def emitSelfContainedBenchRunCode(name, plan, targetFile, configFile, benchmark)
1229   FileCreator.open(configFile) {
1230     | outp |
1231     outp.puts "__bencher_message = \"#{name}: #{plan.vm}: #{plan.iteration}: \";"
1232     outp.puts "__bencher_warmup = #{$warmup};"
1233     outp.puts "__bencher_inner = #{$inner};"
1234     outp.puts "__bencher_benchmark = #{benchmark.to_json};"
1235     case $timeMode
1236     when :preciseTime
1237       outp.puts "__bencher_curTime = (function(){ return testRunner.preciseTime() * 1000; });"
1238     when :date
1239       outp.puts "__bencher_curTime = (function(){ return Date.now(); });"
1240     else
1241       raise
1242     end
1243   }
1244   
1245   plan.vm.emitRunCode(targetFile, plan)
1246 end
1247
1248 def planForDescription(string, plans, benchFullname, vmName, iteration)
1249   raise "Unexpected benchmark full name: #{benchFullname.inspect}, string: #{string.inspect}" unless benchFullname =~ /\//
1250   suiteName = $~.pre_match
1251   return nil if suiteName == "WARMUP"
1252   benchName = $~.post_match
1253   result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration}
1254   raise "Unexpected result dimensions: #{result.inspect}, string: #{string.inspect}" unless result.size == 1
1255   result[0]
1256 end
1257
1258 class ParsedResult
1259   attr_reader :plan, :innerIndex, :time, :result
1260   
1261   def initialize(plan, innerIndex, time)
1262     @plan = plan
1263     @innerIndex = innerIndex
1264     if time == :crashed
1265       @result = :error
1266     else
1267       @time = time
1268       @result = :success
1269     end
1270     
1271     raise unless @plan.is_a? BenchPlan
1272     raise unless @innerIndex.is_a? Integer
1273     raise unless @time.is_a? Numeric or @result == :error
1274   end
1275   
1276   def benchmark
1277     plan.benchmark
1278   end
1279   
1280   def suite
1281     plan.suite
1282   end
1283   
1284   def vm
1285     plan.vm
1286   end
1287   
1288   def outerIndex
1289     plan.iteration
1290   end
1291   
1292   def self.create(plan, innerIndex, time)
1293     if plan
1294       ParsedResult.new(plan, innerIndex, time)
1295     else
1296       nil
1297     end
1298   end
1299   
1300   def self.parse(plans, string)
1301     if string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): Time: /
1302       benchFullname = $1
1303       vmName = $2
1304       outerIndex = $3.to_i
1305       innerIndex = $4.to_i
1306       time = $~.post_match.to_f
1307       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, time)
1308     elsif string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): CRASHED/
1309       benchFullname = $1
1310       vmName = $2
1311       outerIndex = $3.to_i
1312       innerIndex = $4.to_i
1313       time = $~.post_match.to_f
1314       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, :crashed)
1315     else
1316       nil
1317     end
1318   end
1319 end
1320
1321 class VM
1322   @@extraEnvSet = {}
1323   
1324   def initialize(origPath, name, nameKind, svnRevision)
1325     @origPath = origPath.to_s
1326     @path = origPath.to_s
1327     @name = name
1328     @nameKind = nameKind
1329     @extraEnv = {}
1330     
1331     if $forceVMKind
1332       @vmType = $forceVMKind
1333     else
1334       if @origPath =~ /DumpRenderTree$/
1335         @vmType = :dumpRenderTree
1336       elsif @origPath =~ /WebKitTestRunner$/
1337         @vmType = :webkitTestRunner
1338       else
1339         @vmType = :jsc
1340       end
1341     end
1342     
1343     @svnRevision = svnRevision
1344     
1345     # Try to detect information about the VM.
1346     if path =~ /\/WebKitBuild\/(Release|Debug)+\/([a-zA-Z]+)$/
1347       @checkoutPath = $~.pre_match
1348       # FIXME: Use some variant of this: 
1349       # <bdash>   def retrieve_revision
1350       # <bdash>     `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i
1351       # <bdash>   end
1352       unless @svnRevision
1353         begin
1354           Dir.chdir(@checkoutPath) {
1355             $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
1356             IO.popen("svn info", "r") {
1357               | inp |
1358               inp.each_line {
1359                 | line |
1360                 if line =~ /Revision: ([0-9]+)/
1361                   @svnRevision = $1
1362                 end
1363               }
1364             }
1365           }
1366           unless @svnRevision
1367             $stderr.puts "Warning: running svn info for #{name} silently failed."
1368           end
1369         rescue => e
1370           # Failed to detect svn revision.
1371           $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
1372         end
1373       end
1374     else
1375       $stderr.puts "Warning: could not identify checkout location for #{name}"
1376     end
1377     
1378     if @path =~ /\/Release\/([a-zA-Z]+)$/
1379       @libPath, @relativeBinPath = [$~.pre_match+"/Release"], "./#{$1}"
1380     elsif @path =~ /\/Debug\/([a-zA-Z]+)$/
1381       @libPath, @relativeBinPath = [$~.pre_match+"/Debug"], "./#{$1}"
1382     elsif @path =~ /\/Release\/bin(64|32|)\/([\.a-zA-Z]+)$/
1383       @libPath, @relativeBinPath = [$~.pre_match+"/Release/lib#{$1}"], "./#{$2}"
1384     elsif @path =~ /\/Debug\/bin(64|32|)\/([\.a-zA-Z]+)$/
1385       @libPath, @relativeBinPath = [$~.pre_match+"/Debug/lib#{$1}"], "./#{$2}"
1386     elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/
1387       @libPath = [$~.pre_match + "/Contents/Resources", $~.pre_match + "/Contents/Frameworks"]
1388     elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/
1389       @libPath, @relativeBinPath = [$~.pre_match], $&[1..-1]
1390     elsif @path =~ /(DumpRenderTree|webkitTestRunner|jsc)$/
1391       @libPath, @relativeBinPath = [$~.pre_match+"/"], "./#{$1}"
1392     end
1393     @libPath += $dependencies
1394   end
1395   
1396   def canCopyIntoBenchPath
1397     if @libPath and @relativeBinPath
1398       true
1399     else
1400       false
1401     end
1402   end
1403   
1404   def addExtraEnv(key, val)
1405     @extraEnv[key] = val
1406     @@extraEnvSet[key] = true
1407   end
1408   
1409   def copyIntoBenchPath
1410     raise unless canCopyIntoBenchPath
1411     basename, filename = Benchfile.uniqueFilename("vm")
1412     raise unless Dir.mkdir(filename)
1413     @libPath.each {
1414       | libPathPart |
1415       cmd = "cp -a #{Shellwords.shellescape(libPathPart)}/* #{Shellwords.shellescape(filename.to_s)}"
1416       $stderr.puts ">> #{cmd}" if $verbosity>=2
1417       raise unless system(cmd)
1418     }
1419     @path = "#{basename}/#{@relativeBinPath}"
1420     @libPath = [basename]
1421   end
1422   
1423   def to_s
1424     @name
1425   end
1426   
1427   def name
1428     @name
1429   end
1430   
1431   def shouldMeasureGC
1432     $measureGC == true or ($measureGC == name)
1433   end
1434   
1435   def origPath
1436     @origPath
1437   end
1438   
1439   def path
1440     @path
1441   end
1442   
1443   def nameKind
1444     @nameKind
1445   end
1446   
1447   def vmType
1448     @vmType
1449   end
1450   
1451   def checkoutPath
1452     @checkoutPath
1453   end
1454   
1455   def svnRevision
1456     @svnRevision
1457   end
1458   
1459   def extraEnv
1460     @extraEnv
1461   end
1462   
1463   def printFunction
1464     case @vmType
1465     when :jsc
1466       "print"
1467     when :dumpRenderTree, :webkitTestRunner
1468       "debug"
1469     else
1470       raise @vmType
1471     end
1472   end
1473   
1474   def emitRunCode(fileToRun, plan)
1475     myLibPath = @libPath
1476     myLibPath = [] unless myLibPath
1477     @@extraEnvSet.keys.each {
1478       | key |
1479       $script.puts "unset #{Shellwords.shellescape(key)}"
1480     }
1481     $script.puts "export JSC_useDollarVM=true" if @vmType == :jsc
1482     $script.puts "export DYLD_LIBRARY_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1483     $script.puts "export DYLD_FRAMEWORK_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1484     $script.puts "export LD_LIBRARY_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1485
1486     unless myLibPath.empty?
1487         primaryLibPath = myLibPath.first
1488         $script.puts "export TEST_RUNNER_TEST_PLUGIN_PATH=#{Shellwords.shellescape(Pathname.new(primaryLibPath).join('plugins').to_s)}"
1489         $script.puts "export TEST_RUNNER_INJECTED_BUNDLE_FILENAME=#{Shellwords.shellescape(Pathname.new(primaryLibPath).join('libTestRunnerInjectedBundle.so').to_s)}"
1490     end
1491
1492     @extraEnv.each_pair {
1493       | key, val |
1494       $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1495     }
1496     plan.environment.each_pair {
1497         | key, val |
1498         $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1499     }
1500     $script.puts "#{path} #{fileToRun} 2>&1 || {"
1501     $script.puts "    echo " + Shellwords.shellescape("#{name} failed to run!") + " 1>&2"
1502     $inner.times {
1503       | iteration |
1504       $script.puts "    echo " + Shellwords.shellescape("#{plan.prefix}: #{iteration}: CRASHED")
1505     }
1506     $script.puts "}"
1507     plan.environment.keys.each {
1508         | key |
1509         $script.puts "unset #{Shellwords.shellescape(key)}"
1510     }
1511   end
1512 end
1513
1514 class StatsAccumulator
1515   def initialize
1516     @stats = []
1517     ($outer*$inner).times {
1518       @stats << Stats.new
1519     }
1520   end
1521   
1522   def statsForIteration(outerIteration, innerIteration)
1523     @stats[outerIteration*$inner + innerIteration]
1524   end
1525   
1526   def stats
1527     result = Stats.new
1528     @stats.each {
1529       | stat |
1530       if stat.ok?
1531         result.add(yield stat)
1532       else
1533         result.add(nil)
1534       end
1535     }
1536     result
1537   end
1538   
1539   def geometricMeanStats
1540     stats {
1541       | stat |
1542       stat.geometricMean
1543     }
1544   end
1545   
1546   def arithmeticMeanStats
1547     stats {
1548       | stat |
1549       stat.arithmeticMean
1550     }
1551   end
1552 end
1553
1554 module Benchmark
1555   attr_accessor :benchmarkSuite
1556   attr_reader :name
1557   
1558   def fullname
1559     benchmarkSuite.name + "/" + name
1560   end
1561   
1562   def to_s
1563     fullname
1564   end
1565   
1566   def weight
1567     1
1568   end
1569   
1570   def weightString
1571     raise unless weight.is_a? Fixnum
1572     raise unless weight >= 1
1573     if weight == 1
1574       ""
1575     else
1576       "x#{weight} "
1577     end
1578   end
1579   
1580   def environment
1581     {}
1582   end
1583 end
1584
1585 class SunSpiderBenchmark
1586   include Benchmark
1587   
1588   def initialize(name)
1589     @name = name
1590   end
1591   
1592   def emitRunCode(plan)
1593     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js")))
1594   end
1595 end
1596
1597 class LongSpiderBenchmark
1598   include Benchmark
1599   
1600   def initialize(name)
1601     @name = name
1602   end
1603   
1604   def emitRunCode(plan)
1605     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("LongSpider-#{@name}", "#{LONGSPIDER_PATH}/#{@name}.js")))
1606   end
1607 end
1608
1609 class V8Benchmark
1610   include Benchmark
1611   
1612   def initialize(name)
1613     @name = name
1614   end
1615   
1616   def emitRunCode(plan)
1617     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js")))
1618   end
1619 end
1620
1621 class V8RealBenchmark
1622   include Benchmark
1623   
1624   attr_reader :v8SuiteName
1625   
1626   def initialize(v8SuiteName, name, weight, minimumIterations)
1627     @v8SuiteName = v8SuiteName
1628     @name = name
1629     @weight = weight
1630     @minimumIterations = minimumIterations
1631   end
1632   
1633   def weight
1634     @weight
1635   end
1636   
1637   def emitRunCode(plan)
1638     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(["base", @v8SuiteName, "jsc-#{@name}"].collect{|v| ensureFile("V8Real-#{v}", "#{V8_REAL_PATH}/#{v}.js")}, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, false, @minimumIterations))
1639   end
1640 end
1641
1642 class OctaneBenchmark
1643   include Benchmark
1644   
1645   def initialize(files, name, weight, doWarmup, deterministic, minimumIterations)
1646     @files = files
1647     @name = name
1648     @weight = weight
1649     @doWarmup = doWarmup
1650     @deterministic = deterministic
1651     @minimumIterations = minimumIterations
1652   end
1653   
1654   def weight
1655     @weight
1656   end
1657   
1658   def emitRunCode(plan)
1659     files = []
1660     files += (["base"] + @files).collect {
1661       | v |
1662       ensureFile("Octane-#{v}", "#{OCTANE_PATH}/#{v}.js")
1663     }
1664     files += ["jsc-#{@name}"].collect {
1665       | v |
1666       ensureFile("Octane-#{v}", "#{OCTANE_WRAPPER_PATH}/#{v}.js")
1667     }
1668     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(files, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1669   end
1670 end
1671
1672 class CustomTimedBenchmark
1673   include Benchmark
1674   
1675   def initialize(name, fullPath)
1676     @name = name
1677     @fullPath = fullPath
1678   end
1679   
1680   def emitRunCode(plan)
1681     emitBenchRunCode(fullname, plan, SingleFileCustomTimedBenchmarkParameters.new(ensureFile("CustomTimed-#{@name}", @fullPath)))
1682   end
1683 end
1684
1685 class CompileTimeBenchmark
1686   include Benchmark
1687   
1688   def initialize(name, fullPath)
1689     @name = name
1690     @fullPath = fullPath
1691   end
1692   
1693   def emitRunCode(plan)
1694     emitBenchRunCode(fullname, plan, SingleFileCompileTimeBenchmarkParameters.new(ensureFile("CompileTime-#{@name}", @fullPath)))
1695   end
1696   
1697   def environment
1698     {"JSC_useConcurrentJIT" => "false", "JSC_reportTotalCompileTimes" => "true"}
1699   end
1700 end
1701
1702 class KrakenBenchmark
1703   include Benchmark
1704   
1705   def initialize(name)
1706     @name = name
1707   end
1708   
1709   def emitRunCode(plan)
1710     emitBenchRunCode(fullname, plan, MultiFileTimedBenchmarkParameters.new([ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js")], loadCommandForFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js")))
1711   end
1712 end
1713
1714 class JSBenchBenchmark
1715   include Benchmark
1716   
1717   attr_reader :jsBenchMode
1718   
1719   def initialize(name, jsBenchMode)
1720     @name = name
1721     @jsBenchMode = jsBenchMode
1722   end
1723   
1724   def emitRunCode(plan)
1725     callbackDecl  = "function JSBNG_handleResult(result) {\n"
1726     callbackDecl += "    if (result.error) {\n"
1727     callbackDecl += "        console.log(\"Did not run benchmark correctly!\");\n"
1728     callbackDecl += "        quit();\n"
1729     callbackDecl += "    }\n"
1730     callbackDecl += "    reportResult(result.time);\n"
1731     callbackDecl += "}\n"
1732     emitBenchRunCode(fullname, plan, SingleFileTimedCallbackBenchmarkParameters.new(callbackDecl, ensureFile("JSBench-#{@name}", "#{JSBENCH_PATH}/#{@name}/#{@jsBenchMode}.js")))
1733   end
1734 end
1735
1736 class TailBenchBenchmark
1737   include Benchmark
1738
1739   def initialize(name)
1740     @name = name
1741   end
1742
1743   def emitRunCode(plan)
1744     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("TailBench-#{@name}", "#{TAILBENCH_PATH}/#{@name}.js")))
1745   end
1746 end
1747
1748 class MicrobenchmarksBenchmark
1749   include Benchmark
1750   
1751   def initialize(name)
1752     @name = name
1753   end
1754   
1755   def emitRunCode(plan)
1756     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("Microbenchmarks-#{@name}", "#{MICROBENCHMARKS_PATH}/#{@name}.js")))
1757   end
1758 end
1759
1760 class AsmBenchBenchmark
1761   include Benchmark
1762   
1763   def initialize(name)
1764     @name = name
1765   end
1766   
1767   def emitRunCode(plan)
1768     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("AsmBench-#{@name}", "#{ASMBENCH_PATH}/#{@name}.js")))
1769   end
1770 end
1771
1772 class SixSpeedBenchmark
1773   include Benchmark
1774
1775   def initialize(name, path, iterations)
1776     @name = name
1777     @path = path
1778     @iterations = iterations
1779   end
1780
1781   def emitRunCode(plan)
1782     emitBenchRunCode(fullname, plan, MultiFileTimedBenchmarkParameters.new([ensureFile("SixSpeed-#{@name}-wrapper", "#{SIXSPEED_WRAPPER_PATH}/wrapper.js"), ensureFile("SixSpeed-#{@name}", "#{SIXSPEED_PATH}/#{@path}")], simpleCommand("jscRun(#{@iterations});")))
1783   end
1784 end
1785
1786
1787 class CompressionBenchBenchmark
1788   include Benchmark
1789   
1790   def initialize(files, name, model)
1791     @files = files
1792     @name = name;
1793     @name = name + "-" + model if !model.empty?
1794     @name = @name.gsub(" ", "-").downcase
1795     @scriptName = name
1796     @weight = 1
1797     @doWarmup = true
1798     @deterministic = true
1799     @minimumIterations = 1
1800     @model = model
1801   end
1802   
1803   def weight
1804     @weight
1805   end
1806   
1807   def emitRunCode(plan)
1808     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new((["base"] + @files + ["jsc-#{@scriptName}"]).collect{|v| ensureFile("Compression-#{v}", "#{COMPRESSIONBENCH_PATH}/#{v}.js")}, simpleCommand("jscSetUp('#{@model}');"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1809   end
1810 end
1811
1812 class DSPJSFiltrrBenchmark
1813   include Benchmark
1814   
1815   def initialize(name, filterKey)
1816     @name = name
1817     @filterKey = filterKey
1818   end
1819   
1820   def emitRunCode(plan)
1821     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr.js")
1822     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr_back.jpg")
1823     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-jquery.min.js")
1824     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-bencher.html")
1825     emitSelfContainedBenchRunCode(fullname, plan, "filtrr-bencher.html", "bencher-config.js", @filterKey)
1826   end
1827 end
1828
1829 class DSPJSVP8Benchmark
1830   include Benchmark
1831   
1832   def initialize
1833     @name = "route9-vp8"
1834   end
1835   
1836   def weight
1837     5
1838   end
1839   
1840   def emitRunCode(plan)
1841     ensureBenchmarkFiles(DSPJS_ROUTE9_PATH)
1842     emitSelfContainedBenchRunCode(fullname, plan, "route9-bencher.html", "bencher-config.js", "")
1843   end
1844 end
1845
1846 class DSPStarfieldBenchmark
1847   include Benchmark
1848
1849   def initialize
1850     @name = "starfield"
1851   end
1852   
1853   def weight
1854     5
1855   end
1856
1857   def emitRunCode(plan)
1858     ensureBenchmarkFiles(DSPJS_STARFIELD_PATH)
1859     emitSelfContainedBenchRunCode(fullname, plan, "starfield-bencher.html", "bencher-config.js", "")
1860   end
1861 end
1862
1863 class DSPJSJSLinuxBenchmark
1864   include Benchmark
1865   def initialize
1866     @name = "bellard-jslinux"
1867   end
1868
1869   def weight
1870     5
1871   end
1872
1873   def emitRunCode(plan)
1874     ensureBenchmarkFiles(DSPJS_JSLINUX_PATH)
1875     emitSelfContainedBenchRunCode(fullname, plan, "jslinux-bencher.html", "bencher-config.js", "")
1876   end
1877 end
1878
1879 class DSPJSQuake3Benchmark
1880   include Benchmark
1881
1882   def initialize
1883     @name = "zynaps-quake3"
1884   end
1885
1886   def weight
1887     5
1888   end
1889
1890   def emitRunCode(plan)
1891     ensureBenchmarkFiles(DSPJS_QUAKE3_PATH)
1892     emitSelfContainedBenchRunCode(fullname, plan, "quake-bencher.html", "bencher-config.js", "")
1893   end
1894 end
1895
1896 class DSPJSMandelbrotBenchmark
1897   include Benchmark
1898
1899   def initialize
1900     @name = "zynaps-mandelbrot"
1901   end
1902
1903   def weight
1904     5
1905   end
1906
1907   def emitRunCode(plan)
1908     ensureBenchmarkFiles(DSPJS_MANDELBROT_PATH)
1909     emitSelfContainedBenchRunCode(fullname, plan, "mandelbrot-bencher.html", "bencher-config.js", "")
1910   end
1911 end
1912
1913 class DSPJSAmmoJSASMBenchmark
1914   include Benchmark
1915
1916   def initialize
1917     @name = "ammojs-asm-js"
1918   end
1919
1920   def weight
1921     5
1922   end
1923
1924   def emitRunCode(plan)
1925     ensureBenchmarkFiles(DSPJS_AMMOJS_ASMJS_PATH)
1926     emitSelfContainedBenchRunCode(fullname, plan, "ammo-asmjs-bencher.html", "bencher-config.js", "")
1927   end
1928 end
1929
1930 class DSPJSAmmoJSRegularBenchmark
1931   include Benchmark
1932
1933   def initialize
1934     @name = "ammojs-regular-js"
1935   end
1936
1937   def weight
1938     5
1939   end
1940
1941   def emitRunCode(plan)
1942     ensureBenchmarkFiles(DSPJS_AMMOJS_REGULAR_PATH)
1943     emitSelfContainedBenchRunCode(fullname, plan, "ammo-regular-bencher.html", "bencher-config.js", "")
1944   end
1945 end
1946
1947 class BrowsermarkJSBenchmark
1948   include Benchmark
1949     
1950   def initialize(name)
1951     @name = name
1952   end
1953   
1954   def emitRunCode(plan)
1955     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new([ensureFile(name, "#{BROWSERMARK_JS_PATH}/#{name}/test.js"), ensureFile("browsermark-bencher", "#{BROWSERMARK_JS_PATH}/browsermark-bencher.js")], simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, 32))
1956   end
1957 end
1958
1959 class BrowsermarkDOMBenchmark
1960   include Benchmark
1961     
1962   def initialize(name)
1963     @name = name
1964   end
1965   
1966   def emitRunCode(plan)
1967     ensureBenchmarkFiles(BROWSERMARK_PATH)
1968     emitSelfContainedBenchRunCode(fullname, plan, "tests/benchmarks/dom/#{name}/index.html", "bencher-config.js", name)
1969   end
1970 end
1971
1972 class BenchmarkSuite
1973   def initialize(name, preferredMean, decimalShift)
1974     @name = name
1975     @preferredMean = preferredMean
1976     @benchmarks = []
1977     @subSuites = []
1978     @decimalShift = decimalShift
1979   end
1980   
1981   def name
1982     @name
1983   end
1984   
1985   def to_s
1986     @name
1987   end
1988   
1989   def decimalShift
1990     @decimalShift
1991   end
1992   
1993   def addIgnoringPattern(benchmark)
1994     benchmark.benchmarkSuite = self
1995     @benchmarks << benchmark
1996   end
1997   
1998   def add(benchmark)
1999     if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern
2000         addIgnoringPattern(benchmark)
2001     end
2002   end
2003   
2004   def addSubSuite(subSuite)
2005     @subSuites << subSuite
2006   end
2007   
2008   def benchmarks
2009     @benchmarks
2010   end
2011   
2012   def benchmarkForName(name)
2013     result = @benchmarks.select{|v| v.name == name}
2014     raise unless result.length == 1
2015     result[0]
2016   end
2017   
2018   def hasBenchmark(benchmark)
2019     array = @benchmarks.select{|v| v == benchmark}
2020     raise unless array.length == 1 or array.length == 0
2021     array.length == 1
2022   end
2023   
2024   def subSuites
2025     @subSuites
2026   end
2027   
2028   def suites
2029     [self] + @subSuites
2030   end
2031   
2032   def suitesWithBenchmark(benchmark)
2033     result = [self]
2034     @subSuites.each {
2035       | subSuite |
2036       if subSuite.hasBenchmark(benchmark)
2037         result << subSuite
2038       end
2039     }
2040     result
2041   end
2042   
2043   def empty?
2044     @benchmarks.empty?
2045   end
2046   
2047   def retain_if
2048     @benchmarks.delete_if {
2049       | benchmark |
2050       not yield benchmark
2051     }
2052   end
2053   
2054   def preferredMean
2055     @preferredMean
2056   end
2057   
2058   def computeMean(stat)
2059     if stat.ok?
2060       (stat.send @preferredMean) * (10 ** decimalShift)
2061     else
2062       nil
2063     end
2064   end
2065 end
2066
2067 class BenchRunPlan
2068   def initialize(benchmark, vm, iteration)
2069     @benchmark = benchmark
2070     @vm = vm
2071     @iteration = iteration
2072     @environment = {}
2073     if $environment.has_key?(vm.name)
2074       if $environment[vm.name].has_key?(benchmark.benchmarkSuite.name)
2075         if $environment[vm.name][benchmark.benchmarkSuite.name].has_key?(benchmark.name)
2076           @environment = $environment[vm.name][benchmark.benchmarkSuite.name][benchmark.name]
2077         end
2078       end
2079     end
2080   end
2081   
2082   def benchmark
2083     @benchmark
2084   end
2085   
2086   def suite
2087     @benchmark.benchmarkSuite
2088   end
2089   
2090   def vm
2091     @vm
2092   end
2093   
2094   def iteration
2095     @iteration
2096   end
2097  
2098   def environment
2099     result = @environment.clone
2100     benchmark.environment.each_pair {
2101       | key, value |
2102       result[key] = value
2103     }
2104     result
2105   end
2106  
2107   def prefix
2108     "#{@benchmark.fullname}: #{vm.name}: #{iteration}"
2109   end
2110   
2111   def emitRunCode
2112     @benchmark.emitRunCode(self)
2113   end
2114   
2115   def to_s
2116     benchmark.to_s + "/" + vm.to_s
2117   end
2118 end
2119
2120 class BenchmarkOnVM
2121   def initialize(benchmark, suiteOnVM, subSuitesOnVM)
2122     @benchmark = benchmark
2123     @suiteOnVM = suiteOnVM
2124     @subSuitesOnVM = subSuitesOnVM
2125     @stats = Stats.new
2126   end
2127   
2128   def to_s
2129     "#{@benchmark} on #{@suiteOnVM.vm}"
2130   end
2131   
2132   def benchmark
2133     @benchmark
2134   end
2135   
2136   def vm
2137     @suiteOnVM.vm
2138   end
2139   
2140   def vmStats
2141     @suiteOnVM.vmStats
2142   end
2143   
2144   def suite
2145     @benchmark.benchmarkSuite
2146   end
2147   
2148   def suiteOnVM
2149     @suiteOnVM
2150   end
2151   
2152   def subSuitesOnVM
2153     @subSuitesOnVM
2154   end
2155   
2156   def stats
2157     @stats
2158   end
2159   
2160   def parseResult(result)
2161     raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm
2162     raise unless result.benchmark == @benchmark
2163     @stats.add(result.time)
2164   end
2165 end
2166
2167 class NamedStatsAccumulator < StatsAccumulator
2168   def initialize(name)
2169     super()
2170     @name = name
2171   end
2172   
2173   def reportingName
2174     @name
2175   end
2176 end
2177
2178 class SuiteOnVM < StatsAccumulator
2179   def initialize(vm, vmStats, suite)
2180     super()
2181     @vm = vm
2182     @vmStats = vmStats
2183     @suite = suite
2184     
2185     raise unless @vm.is_a? VM
2186     raise unless @vmStats.is_a? StatsAccumulator
2187     raise unless @suite.is_a? BenchmarkSuite
2188   end
2189   
2190   def to_s
2191     "#{@suite} on #{@vm}"
2192   end
2193   
2194   def suite
2195     @suite
2196   end
2197   
2198   def vm
2199     @vm
2200   end
2201
2202   def reportingName
2203     @vm.name
2204   end
2205   
2206   def vmStats
2207     raise unless @vmStats
2208     @vmStats
2209   end
2210 end
2211
2212 class SubSuiteOnVM < StatsAccumulator
2213   def initialize(vm, suite)
2214     super()
2215     @vm = vm
2216     @suite = suite
2217     raise unless @vm.is_a? VM
2218     raise unless @suite.is_a? BenchmarkSuite
2219   end
2220   
2221   def to_s
2222     "#{@suite} on #{@vm}"
2223   end
2224   
2225   def suite
2226     @suite
2227   end
2228   
2229   def vm
2230     @vm
2231   end
2232   
2233   def reportingName
2234     @vm.name
2235   end
2236 end
2237
2238 class BenchPlan
2239   def initialize(benchmarkOnVM, iteration)
2240     @benchmarkOnVM = benchmarkOnVM
2241     @iteration = iteration
2242   end
2243   
2244   def to_s
2245     "#{@benchmarkOnVM} \##{@iteration+1}"
2246   end
2247   
2248   def benchmarkOnVM
2249     @benchmarkOnVM
2250   end
2251   
2252   def benchmark
2253     @benchmarkOnVM.benchmark
2254   end
2255   
2256   def suite
2257     @benchmarkOnVM.suite
2258   end
2259   
2260   def vm
2261     @benchmarkOnVM.vm
2262   end
2263   
2264   def iteration
2265     @iteration
2266   end
2267   
2268   def parseResult(result)
2269     raise unless result.plan == self
2270     @benchmarkOnVM.parseResult(result)
2271     benchmark.weight.times {
2272       @benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time)
2273       @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time)
2274       @benchmarkOnVM.subSuitesOnVM.each {
2275         | subSuite |
2276         subSuite.statsForIteration(@iteration, result.innerIndex).add(result.time)
2277       }
2278     }
2279   end
2280 end
2281
2282 def lpad(str,chars)
2283   if str.length>chars
2284     str
2285   else
2286     "%#{chars}s"%(str)
2287   end
2288 end
2289
2290 def rpad(str,chars)
2291   while str.length < chars
2292     str+=" "
2293   end
2294   str
2295 end
2296
2297 def center(str,chars)
2298   while str.length<chars
2299     str+=" "
2300     if str.length<chars
2301       str=" "+str
2302     end
2303   end
2304   str
2305 end
2306
2307 def statsToStr(stats, decimalShift)
2308   if stats.error?
2309     lpad(center("ERROR", 10+10+2), 12+10+2)
2310   elsif $inner*$outer == 1
2311     string = numToStr(stats.mean, decimalShift)
2312     raise unless string =~ /\./
2313     left = $~.pre_match
2314     right = $~.post_match
2315     lpad(left, 13 - decimalShift) + "." + rpad(right, 10 + decimalShift)
2316   else
2317     lpad(numToStr(stats.mean, decimalShift), 12) + "+-" + rpad(numToStr(stats.confInt, decimalShift), 10)
2318   end
2319 end
2320
2321 def plural(num)
2322   if num == 1
2323     ""
2324   else
2325     "s"
2326   end
2327 end
2328
2329 def wrap(str, columns)
2330   array = str.split
2331   result = ""
2332   curLine = array.shift
2333   array.each {
2334     | curStr |
2335     if (curLine + " " + curStr).size > columns
2336       result += curLine + "\n"
2337       curLine = curStr
2338     else
2339       curLine += " " + curStr
2340     end
2341   }
2342   result + curLine + "\n"
2343 end
2344   
2345 def runAndGetResults
2346   results = nil
2347   Dir.chdir(BENCH_DATA_PATH) {
2348     $stderr.puts ">> sh ./runscript" if $verbosity >= 2
2349     raise "Script did not complete correctly: #{$?}" unless system("sh ./runscript > runlog")
2350     results = IO::read("runlog")
2351   }
2352   raise unless results
2353   results
2354 end
2355
2356 def parseAndDisplayResults(results)
2357   vmStatses = []
2358   $vms.each {
2359     | vm |
2360     vmStatses << NamedStatsAccumulator.new(vm.name)
2361   }
2362   
2363   suitesOnVMs = []
2364   suitesOnVMsForSuite = {}
2365   subSuitesOnVMsForSubSuite = {}
2366   $suites.each {
2367     | suite |
2368     suitesOnVMsForSuite[suite] = []
2369     suite.subSuites.each {
2370       | subSuite |
2371       subSuitesOnVMsForSubSuite[subSuite] = []
2372     }
2373   }
2374   suitesOnVMsForVM = {}
2375   $vms.each {
2376     | vm |
2377     suitesOnVMsForVM[vm] = []
2378   }
2379   
2380   benchmarksOnVMs = []
2381   benchmarksOnVMsForBenchmark = {}
2382   $benchmarks.each {
2383     | benchmark |
2384     benchmarksOnVMsForBenchmark[benchmark] = []
2385   }
2386   
2387   $vms.each_with_index {
2388     | vm, vmIndex |
2389     vmStats = vmStatses[vmIndex]
2390     $suites.each {
2391       | suite |
2392       suiteOnVM = SuiteOnVM.new(vm, vmStats, suite)
2393       subSuitesOnVM = suite.subSuites.map {
2394         | subSuite |
2395         result = SubSuiteOnVM.new(vm, subSuite)
2396         subSuitesOnVMsForSubSuite[subSuite] << result
2397         result
2398       }
2399       suitesOnVMs << suiteOnVM
2400       suitesOnVMsForSuite[suite] << suiteOnVM
2401       suitesOnVMsForVM[vm] << suiteOnVM
2402       suite.benchmarks.each {
2403         | benchmark |
2404         subSuitesOnVMForThisBenchmark = []
2405         subSuitesOnVM.each {
2406           | subSuiteOnVM |
2407           if subSuiteOnVM.suite.hasBenchmark(benchmark)
2408             subSuitesOnVMForThisBenchmark << subSuiteOnVM
2409           end
2410         }
2411         benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM, subSuitesOnVMForThisBenchmark)
2412         benchmarksOnVMs << benchmarkOnVM
2413         benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
2414       }
2415     }
2416   }
2417   
2418   plans = []
2419   benchmarksOnVMs.each {
2420     | benchmarkOnVM |
2421     $outer.times {
2422       | iteration |
2423       plans << BenchPlan.new(benchmarkOnVM, iteration)
2424     }
2425   }
2426
2427   hostname = nil
2428   hwmodel = nil
2429   results.each_line {
2430     | line |
2431     line.chomp!
2432     if line =~ /HOSTNAME:([^.]+)/
2433       hostname = $1
2434     elsif line =~ /HARDWARE:hw\.model: /
2435       hwmodel = $~.post_match.chomp
2436     else
2437       result = ParsedResult.parse(plans, line)
2438       if result
2439         result.plan.parseResult(result)
2440       end
2441     end
2442   }
2443   
2444   # Compute the geomean of the preferred means of results on a SuiteOnVM
2445   overallResults = []
2446   $vms.each {
2447     | vm |
2448     result = Stats.new
2449     $outer.times {
2450       | outerIndex |
2451       $inner.times {
2452         | innerIndex |
2453         curResult = Stats.new
2454         suitesOnVMsForVM[vm].each {
2455           | suiteOnVM |
2456           # For a given iteration, suite, and VM, compute the suite's preferred mean
2457           # over the data collected for all benchmarks in that suite. We'll have one
2458           # sample per benchmark. For example on V8 this will be the geomean of 1
2459           # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
2460           # splay.
2461           curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
2462         }
2463         
2464         # curResult now holds 1 sample for each of the means computed in the above
2465         # loop. Compute the geomean over this, and store it.
2466         if curResult.ok?
2467           result.add(curResult.geometricMean)
2468         else
2469           result.add(nil)
2470         end
2471       }
2472     }
2473
2474     # $overallResults will have a Stats for each VM. That Stats object will hold
2475     # $inner*$outer geomeans, allowing us to compute the arithmetic mean and
2476     # confidence interval of the geomeans of preferred means. Convoluted, but
2477     # useful and probably sound.
2478     overallResults << result
2479   }
2480   
2481   if $verbosity >= 2
2482     benchmarksOnVMs.each {
2483       | benchmarkOnVM |
2484       $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
2485     }
2486     
2487     $vms.each_with_index {
2488       | vm, vmIndex |
2489       vmStats = vmStatses[vmIndex]
2490       $stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}"
2491       $stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}"
2492     }
2493   end
2494
2495   if $outputName
2496     reportName = $outputName
2497   else
2498     reportName =
2499       (if ($vms.collect {
2500              | vm |
2501              vm.nameKind
2502            }.index :auto)
2503          ""
2504        else
2505          text = $vms.collect {
2506            | vm |
2507            vm.to_s
2508          }.join("_") + "_"
2509          if text.size >= 40
2510            ""
2511          else
2512            text
2513          end
2514        end) +
2515       ($suites.collect {
2516          | suite |
2517          suite.to_s
2518        }.join("")) + "_" +
2519       (if hostname
2520          hostname + "_"
2521        else
2522          ""
2523        end)+
2524       (begin
2525          time = Time.now
2526          "%04d%02d%02d_%02d%02d" %
2527            [ time.year, time.month, time.day,
2528              time.hour, time.min ]
2529        end)
2530   end
2531
2532   unless $brief
2533     puts "Generating benchmark report at #{Dir.pwd}/#{reportName}_report.txt"
2534     puts "And raw data at #{Dir.pwd}/#{reportName}.json"
2535   end
2536   
2537   outp = $stdout
2538   json = {}
2539   begin
2540     outp = File.open(reportName + "_report.txt","w")
2541   rescue => e
2542     $stderr.puts "Error: could not save report to #{reportName}_report.txt: #{e}"
2543     $stderr.puts
2544   end
2545   
2546   def createVMsString
2547     result = ""
2548     result += "   " if $allSuites.size > 1
2549     result += rpad("", $benchpad + $weightpad)
2550     result += " "
2551     $vms.size.times {
2552       | index |
2553       if index != 0
2554         result += " "+NoChange.new(0).shortForm
2555       end
2556       result += lpad(center($vms[index].name, 10+10+2), 12+10+2)
2557     }
2558     result += "    "
2559     if $vms.size >= 3
2560       result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
2561     elsif $vms.size >= 2
2562       result += " "*26
2563     end
2564     result
2565   end
2566   
2567   def andJoin(list)
2568     if list.size == 1
2569       list[0].to_s
2570     elsif list.size == 2
2571       "#{list[0]} and #{list[1]}"
2572     else
2573       "#{list[0..-2].join(', ')}, and #{list[-1]}"
2574     end
2575   end
2576   
2577   json["vms"] = $vms.collect{|v| v.name}
2578   json["suites"] = {}
2579   json["runlog"] = results
2580   
2581   columns = [createVMsString.size, 78].max
2582   
2583   outp.print "Benchmark report for "
2584   outp.print andJoin($suites)
2585   if hostname
2586     outp.print " on #{hostname}"
2587   end
2588   if hwmodel
2589     outp.print " (#{hwmodel})"
2590   end
2591   outp.puts "."
2592   outp.puts
2593   
2594   outp.puts "VMs tested:"
2595   $vms.each {
2596     | vm |
2597     outp.print "\"#{vm.name}\" at #{vm.origPath}"
2598     if vm.svnRevision
2599       outp.print " (r#{vm.svnRevision})"
2600     end
2601     outp.puts
2602     vm.extraEnv.each_pair {
2603       | key, val |
2604       outp.puts "    export #{key}=#{val}"
2605     }
2606   }
2607   
2608   outp.puts
2609   
2610   outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
2611                  "with #{$outer} VM invocation#{plural($outer)} per benchmark."+
2612                  (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
2613                                       "total time of those iterations, for each sample.")
2614                   else "" end)+
2615                  (if $measureGC == true then (" No manual garbage collection invocations were "+
2616                                               "emitted.")
2617                   elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
2618                                          "all VMs except #{$measureGC}.")
2619                   else (" Emitted a call to gc() between sample measurements.") end)+
2620                  (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
2621                                         "began with the very first iteration.")
2622                   else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
2623                         "invocation for warm-up.") end)+
2624                  (case $timeMode
2625                   when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
2626                                           "microsecond-level timing.")
2627                   when :date then (" Used the portable Date.now() method to get millisecond-"+
2628                                    "level timing.")
2629                   else raise end)+
2630                  " Reporting benchmark execution times with 95% confidence "+
2631                  "intervals in milliseconds.",
2632                  columns)
2633   
2634   outp.puts
2635   
2636   def printVMs(outp)
2637     outp.puts createVMsString
2638   end
2639   
2640   def summaryStats(outp, json, accumulators, name, decimalShift, &proc)
2641     resultingJson = {}
2642     outp.print "   " if $allSuites.size > 1
2643     outp.print rpad(name, $benchpad + $weightpad)
2644     outp.print " "
2645     accumulators.size.times {
2646       | index |
2647       if index != 0
2648         outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
2649       end
2650       outp.print statsToStr(accumulators[index].stats(&proc), decimalShift)
2651       resultingJson[accumulators[index].reportingName] = accumulators[index].stats(&proc).jsonMap
2652     }
2653     if accumulators.size>=2
2654       outp.print("    "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm)
2655     end
2656     outp.puts
2657     json[name] = resultingJson
2658   end
2659
2660   def allSummaryStats(outp, json, accumulators, preferredMean, decimalShift)
2661     meanLabel = '<' + preferredMean.to_s.sub(/Mean$/, '') + '>'
2662     summaryStats(outp, json, accumulators, meanLabel, decimalShift) {
2663       | stat |
2664       stat.send(preferredMean)
2665     }
2666   end
2667   
2668   $suites.each {
2669     | suite |
2670     suiteJson = {}
2671     subSuiteJsons = {}
2672     suite.subSuites.each {
2673       | subSuite |
2674       subSuiteJsons[subSuite] = {}
2675     }
2676     
2677     printVMs(outp)
2678     if $allSuites.size > 1
2679       outp.puts(andJoin(suite.suites.map{|v| v.name}) + ":")
2680     else
2681       outp.puts
2682     end
2683     suite.benchmarks.each {
2684       | benchmark |
2685       benchmarkJson = {}
2686       outp.print "   " if $allSuites.size > 1
2687       outp.print rpad(benchmark.name, $benchpad) + rpad(benchmark.weightString, $weightpad)
2688       if benchmark.name.size > $benchNameClip
2689         outp.puts
2690         outp.print "   " if $allSuites.size > 1
2691         outp.print((" " * $benchpad) + (" " * $weightpad))
2692       end
2693       outp.print " "
2694       myConfigs = benchmarksOnVMsForBenchmark[benchmark]
2695       myConfigs.size.times {
2696         | index |
2697         if index != 0
2698           outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
2699         end
2700         outp.print statsToStr(myConfigs[index].stats, suite.decimalShift)
2701         benchmarkJson[myConfigs[index].vm.name] = myConfigs[index].stats.jsonMap
2702       }
2703       if $vms.size>=2
2704         outp.print("    "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
2705       end
2706       outp.puts
2707       suiteJson[benchmark.name] = benchmarkJson
2708       suite.subSuites.each {
2709         | subSuite |
2710         if subSuite.hasBenchmark(benchmark)
2711           subSuiteJsons[subSuite][benchmark.name] = benchmarkJson
2712         end
2713       }
2714     }
2715     outp.puts
2716     unless suite.subSuites.empty?
2717       suite.subSuites.each {
2718         | subSuite |
2719         outp.puts "#{subSuite.name}:"
2720         allSummaryStats(outp, subSuiteJsons[subSuite], subSuitesOnVMsForSubSuite[subSuite], subSuite.preferredMean, subSuite.decimalShift)
2721         outp.puts
2722       }
2723       outp.puts "#{suite.name} including #{andJoin(suite.subSuites.map{|v| v.name})}:"
2724     end
2725     allSummaryStats(outp, suiteJson, suitesOnVMsForSuite[suite], suite.preferredMean, suite.decimalShift)
2726     outp.puts if $allSuites.size > 1
2727     
2728     json["suites"][suite.name] = suiteJson
2729     suite.subSuites.each {
2730       | subSuite |
2731       json["suites"][subSuite.name] = subSuiteJsons[subSuite]
2732     }
2733   }
2734   
2735   if $suites.size > 1
2736     scaledResultJson = {}
2737     printVMs(outp)
2738     outp.puts "Geomean of preferred means:"
2739     outp.print "   "
2740     outp.print rpad("<scaled-result>", $benchpad + $weightpad)
2741     outp.print " "
2742     $vms.size.times {
2743       | index |
2744       if index != 0
2745         outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm
2746       end
2747       outp.print statsToStr(overallResults[index], 0)
2748       scaledResultJson[$vms[index].name] = overallResults[index].jsonMap
2749     }
2750     if overallResults.size>=2
2751       outp.print("    "+overallResults[-1].compareTo(overallResults[0]).longForm)
2752     end
2753     outp.puts
2754     
2755     json["<scaled-result>"] = scaledResultJson
2756   end
2757   outp.puts
2758   
2759   if outp != $stdout
2760     outp.close
2761   end
2762   
2763   if outp != $stdout and not $brief
2764     puts
2765     File.open(reportName + "_report.txt") {
2766       | inp |
2767       puts inp.read
2768     }
2769   end
2770   
2771   if $brief
2772     puts(overallResults.collect{|stats| stats.mean}.join("\t"))
2773     puts(overallResults.collect{|stats| stats.confInt}.join("\t"))
2774   end
2775   
2776   File.open(reportName + ".json", "w") {
2777     | outp |
2778     outp.puts json.to_json
2779   }
2780 end
2781
2782 begin
2783   $sawBenchOptions = false
2784   
2785   def resetBenchOptionsIfNecessary
2786     unless $sawBenchOptions
2787       $includeSunSpider = false
2788       $includeSunSpiderCompileTime = false
2789       $includeLongSpider = false
2790       $includeV8 = false
2791       $includeV8CompileTime = false
2792       $includeKraken = false
2793       $includeJSBench = false
2794       $includeMicrobenchmarks = false
2795       $includeAsmBench = false
2796       $includeDSPJS = false
2797       $includeBrowsermarkJS = false
2798       $includeBrowsermarkDOM = false
2799       $includeOctane = false
2800       $includeCompressionBench = false
2801       $includeSixSpeed = false
2802       $includeTailBench = false;
2803       $sawBenchOptions = true
2804     end
2805   end
2806   
2807   GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
2808                  ['--inner', GetoptLong::REQUIRED_ARGUMENT],
2809                  ['--outer', GetoptLong::REQUIRED_ARGUMENT],
2810                  ['--warmup', GetoptLong::REQUIRED_ARGUMENT],
2811                  ['--no-ss-warmup', GetoptLong::NO_ARGUMENT],
2812                  ['--quantum', GetoptLong::REQUIRED_ARGUMENT],
2813                  ['--minimum', GetoptLong::REQUIRED_ARGUMENT],
2814                  ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
2815                  ['--sunspider', GetoptLong::NO_ARGUMENT],
2816                  ['--sunspider-compile-time', GetoptLong::NO_ARGUMENT],
2817                  ['--longspider', GetoptLong::NO_ARGUMENT],
2818                  ['--v8-spider', GetoptLong::NO_ARGUMENT],
2819                  ['--v8-spider-compile-time', GetoptLong::NO_ARGUMENT],
2820                  ['--kraken', GetoptLong::NO_ARGUMENT],
2821                  ['--js-bench', GetoptLong::NO_ARGUMENT],
2822                  ['--microbenchmarks', GetoptLong::NO_ARGUMENT],
2823                  ['--asm-bench', GetoptLong::NO_ARGUMENT],
2824                  ['--dsp', GetoptLong::NO_ARGUMENT],
2825                  ['--browsermark-js', GetoptLong::NO_ARGUMENT],
2826                  ['--browsermark-dom', GetoptLong::NO_ARGUMENT],
2827                  ['--octane', GetoptLong::NO_ARGUMENT],
2828                  ['--compression-bench', GetoptLong::NO_ARGUMENT],
2829                  ['--six-speed', GetoptLong::NO_ARGUMENT],
2830                  ['--tail-bench', GetoptLong::NO_ARGUMENT],
2831                  ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
2832                  ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
2833                  ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
2834                  ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
2835                  ['--dont-copy-vms', GetoptLong::NO_ARGUMENT],
2836                  ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
2837                  ['--brief', GetoptLong::NO_ARGUMENT],
2838                  ['--silent', GetoptLong::NO_ARGUMENT],
2839                  ['--remote', GetoptLong::REQUIRED_ARGUMENT],
2840                  ['--local', GetoptLong::NO_ARGUMENT],
2841                  ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
2842                  ['--slave', GetoptLong::NO_ARGUMENT],
2843                  ['--prepare-only', GetoptLong::NO_ARGUMENT],
2844                  ['--analyze', GetoptLong::REQUIRED_ARGUMENT],
2845                  ['--vms', GetoptLong::REQUIRED_ARGUMENT],
2846                  ['--output-name', GetoptLong::REQUIRED_ARGUMENT],
2847                  ['--environment', GetoptLong::REQUIRED_ARGUMENT],
2848                  ['--dependencies', GetoptLong::REQUIRED_ARGUMENT],
2849                  ['--config', GetoptLong::REQUIRED_ARGUMENT],
2850                  ['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
2851     | opt, arg |
2852     case opt
2853     when '--rerun'
2854       $rerun = intArg(opt,arg,1,nil)
2855     when '--inner'
2856       $inner = intArg(opt,arg,1,nil)
2857     when '--outer'
2858       $outer = intArg(opt,arg,1,nil)
2859     when '--warmup'
2860       $warmup = intArg(opt,arg,0,nil)
2861     when '--no-ss-warmup'
2862       $sunSpiderWarmup = false
2863     when '--quantum'
2864       $quantum = intArg(opt,arg,1,nil)
2865     when '--minimum'
2866       $minimum = intArg(opt,arg,1,nil)
2867     when '--timing-mode'
2868       if arg.upcase == "PRECISETIME"
2869         $timeMode = :preciseTime
2870       elsif arg.upcase == "DATE"
2871         $timeMode = :date
2872       elsif arg.upcase == "AUTO"
2873         $timeMode = :auto
2874       else
2875         quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
2876                   "Invalid argument for command-line option")
2877       end
2878     when '--force-vm-kind'
2879       if arg.upcase == "JSC"
2880         $forceVMKind = :jsc
2881       elsif arg.upcase == "DUMPRENDERTREE"
2882         $forceVMKind = :dumpRenderTree
2883       elsif arg.upcase == "WEBKITTESTRUNNER"
2884         $forceVMKind = :webkitTestRunner
2885       elsif arg.upcase == "AUTO"
2886         $forceVMKind = nil
2887       else
2888         quickFail("Expected 'jsc', 'DumpRenderTree', or 'WebKitTestRunner' for --force-vm-kind, but got '#{arg}'.",
2889                   "Invalid argument for command-line option")
2890       end
2891     when '--force-vm-copy'
2892       $needToCopyVMs = true
2893     when '--dont-copy-vms'
2894       $dontCopyVMs = true
2895     when '--sunspider'
2896       resetBenchOptionsIfNecessary
2897       $includeSunSpider = true
2898     when '--sunspider-compile-time'
2899       resetBenchOptionsIfNecessary
2900       $includeSunSpiderCompileTime = true
2901     when '--longspider'
2902       resetBenchOptionsIfNecessary
2903       $includeLongSpider = true
2904     when '--v8-spider'
2905       resetBenchOptionsIfNecessary
2906       $includeV8 = true
2907     when '--v8-spider-compile-time'
2908       resetBenchOptionsIfNecessary
2909       $includeV8CompileTime = true
2910     when '--kraken'
2911       resetBenchOptionsIfNecessary
2912       $includeKraken = true
2913     when '--js-bench'
2914       resetBenchOptionsIfNecessary
2915       $includeJSBench = true
2916     when '--tail-bench'
2917       resetBenchOptionsIfNecessary
2918       $includeTailBench = true
2919     when '--microbenchmarks'
2920       resetBenchOptionsIfNecessary
2921       $includeMicrobenchmarks = true
2922     when '--asm-bench'
2923       resetBenchOptionsIfNecessary
2924       $includeAsmBench = true
2925     when '--dsp'
2926       resetBenchOptionsIfNecessary
2927       $includeDSPJS = true
2928     when '--browsermark-js'
2929       resetBenchOptionsIfNecessary
2930       $includeBrowsermarkJS = true
2931     when '--browsermark-dom'
2932       resetBenchOptionsIfNecessary
2933       $includeBrowsermarkDOM = true
2934     when '--octane'
2935       resetBenchOptionsIfNecessary
2936       $includeOctane = true
2937     when '--compression-bench'
2938       resetBenchOptionsIfNecessary
2939       $includeCompressionBench = true
2940     when '--six-speed'
2941       resetBenchOptionsIfNecessary
2942       $includeSixSpeed = true
2943     when '--benchmarks'
2944       $benchmarkPattern = Regexp.new(arg)
2945     when '--measure-gc'
2946       if arg == ''
2947         $measureGC = true
2948       else
2949         $measureGC = arg
2950       end
2951     when '--verbose'
2952       $verbosity += 1
2953     when '--brief'
2954       $brief = true
2955     when '--silent'
2956       $silent = true
2957     when '--remote'
2958       $remoteHosts += arg.split(',')
2959       $needToCopyVMs = true
2960     when '--ssh-options'
2961       $sshOptions << arg
2962     when '--local'
2963       $alsoLocal = true
2964     when '--prepare-only'
2965       $run = false
2966     when '--analyze'
2967       $prepare = false
2968       $run = false
2969       $analyze << arg
2970     when '--output-name'
2971       $outputName = arg
2972     when '--vms'
2973       JSON::parse(IO::read(arg)).each {
2974         | vmDescription |
2975         path = Pathname.new(vmDescription["path"]).realpath
2976         if vmDescription["name"]
2977           name = vmDescription["name"]
2978           nameKind = :given
2979         else
2980           name = "Conf\##{$vms.length+1}"
2981           nameKind = :auto
2982         end
2983         vm = VM.new(path, name, nameKind, nil)
2984         if vmDescription["env"]
2985           vmDescription["env"].each_pair {
2986             | key, val |
2987             vm.addExtraEnv(key, val)
2988           }
2989         end
2990         $vms << vm
2991       }
2992     when '--environment'
2993       $environment = JSON::parse(IO::read(arg))
2994     when '--dependencies'
2995       $dependencies.push(Pathname.new(arg).realpath)
2996     when '--config'
2997       $configPath = Pathname.new(arg)
2998     when '--help'
2999       usage
3000     else
3001       raise "bad option: #{opt}"
3002     end
3003   }
3004   
3005   # Figure out the configuration
3006   if $configPath.file?
3007     config = JSON::parse(IO::read($configPath.to_s))
3008   else
3009     config = {}
3010   end
3011
3012   def pathname_if_exist config
3013     if config
3014       Pathname.new config
3015     else
3016       config
3017     end
3018   end
3019
3020   OCTANE_PATH = pathname_if_exist(config["OctanePath"])
3021   BROWSERMARK_PATH = pathname_if_exist(config["BrowserMarkPath"])
3022   BROWSERMARK_JS_PATH = pathname_if_exist(config["BrowserMarkJSPath"])
3023   BROWSERMARK_DOM_PATH = pathname_if_exist(config["BrowserMarkDOMPath"])
3024   ASMBENCH_PATH = pathname_if_exist(config["AsmBenchPath"])
3025   COMPRESSIONBENCH_PATH = pathname_if_exist(config["CompressionBenchPath"])
3026   DSPJS_FILTRR_PATH = pathname_if_exist(config["DSPJSFiltrrPath"])
3027   DSPJS_ROUTE9_PATH = pathname_if_exist(config["DSPJSRoute9Path"])
3028   DSPJS_STARFIELD_PATH = pathname_if_exist(config["DSPJSStarfieldPath"])
3029   DSPJS_QUAKE3_PATH = pathname_if_exist(config["DSPJSQuake3Path"])
3030   DSPJS_MANDELBROT_PATH = pathname_if_exist(config["DSPJSMandelbrotPath"])
3031   DSPJS_JSLINUX_PATH = pathname_if_exist(config["DSPJSLinuxPath"])
3032   DSPJS_AMMOJS_ASMJS_PATH = pathname_if_exist(config["DSPJSAmmoJSAsmJSPath"])
3033   DSPJS_AMMOJS_REGULAR_PATH = pathname_if_exist(config["DSPJSAmmoJSRegularPath"])
3034   KRAKEN_PATH = pathname_if_exist(config["KrakenPath"])
3035   
3036   # If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option.
3037   if $dontCopyVMs
3038     $needToCopyVMs = false
3039   end
3040   
3041   ARGV.each {
3042     | vm |
3043     if vm =~ /([a-zA-Z0-9_ .]+):/
3044       name = $1
3045       nameKind = :given
3046       vm = $~.post_match
3047     else
3048       name = "Conf\##{$vms.length+1}"
3049       nameKind = :auto
3050     end
3051     envs = []
3052     while vm =~ /([a-zA-Z0-9_]+)=([a-zA-Z0-9_:.\/-]+):/
3053       envs << [$1, $2]
3054       vm = $~.post_match
3055     end
3056     $stderr.puts "#{name}: #{vm}" if $verbosity >= 1
3057     vm = VM.new(Pathname.new(vm).realpath, name, nameKind, nil)
3058     envs.each {
3059       | pair |
3060       vm.addExtraEnv(pair[0], pair[1])
3061     }
3062     $vms << vm
3063   }
3064   
3065   if $vms.empty?
3066     quickFail("Please specify at least on configuraiton on the command line.",
3067               "Insufficient arguments")
3068   end
3069   
3070   $vms.each {
3071     | vm |
3072     if vm.vmType == :jsc
3073       $allDRT = false
3074     end
3075   }
3076   
3077   SUNSPIDER = BenchmarkSuite.new("SunSpider", :arithmeticMean, 0)
3078   SUNSPIDER_COMPILE_TIME = BenchmarkSuite.new("SunSpider-CompileTime", :arithmeticMean, 0)
3079   WARMUP = BenchmarkSuite.new("WARMUP", :arithmeticMean, 0)
3080   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
3081    "access-fannkuch", "access-nbody", "access-nsieve",
3082    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
3083    "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
3084    "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
3085    "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
3086    "string-base64", "string-fasta", "string-tagcloud",
3087    "string-unpack-code", "string-validate-input"].each {
3088     | name |
3089     SUNSPIDER.add SunSpiderBenchmark.new(name)
3090     SUNSPIDER_COMPILE_TIME.add CompileTimeBenchmark.new(name, "#{SUNSPIDER_PATH}/#{name}.js")
3091     WARMUP.addIgnoringPattern SunSpiderBenchmark.new(name)
3092   }
3093
3094   LONGSPIDER = BenchmarkSuite.new("LongSpider", :geometricMean, 0)
3095   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
3096    "access-fannkuch", "access-nbody", "access-nsieve",
3097    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-nsieve-bits",
3098    "controlflow-recursive", "crypto-aes", "crypto-md5", "crypto-sha1",
3099    "date-format-tofte", "date-format-xparb", "hash-map", "math-cordic",
3100    "math-partial-sums", "math-spectral-norm", "string-base64",
3101    "string-fasta", "string-tagcloud"].each {
3102     | name |
3103     LONGSPIDER.add LongSpiderBenchmark.new(name)
3104   }
3105
3106   V8 = BenchmarkSuite.new("V8Spider", :geometricMean, 0)
3107   V8_COMPILE_TIME = BenchmarkSuite.new("V8Spider-CompileTime", :geometricMean, 0)
3108   ["crypto", "deltablue", "earley-boyer", "raytrace",
3109    "regexp", "richards", "splay"].each {
3110     | name |
3111     V8.add V8Benchmark.new(name)
3112     V8_COMPILE_TIME.add CompileTimeBenchmark.new(name, "#{V8_PATH}/v8-#{name}.js")
3113   }
3114
3115   OCTANE = BenchmarkSuite.new("Octane", :geometricMean, 1)
3116   [[["crypto"], "encrypt", 1, true, false, 32],
3117    [["crypto"], "decrypt", 1, true, false, 32],
3118    [["deltablue"], "deltablue", 2, true, false, 32],
3119    [["earley-boyer"], "earley", 1, true, false, 32],
3120    [["earley-boyer"], "boyer", 1, true, false, 32],
3121    [["navier-stokes"], "navier-stokes", 2, true, false, 16],
3122    [["raytrace"], "raytrace", 2, true, false, 32],
3123    [["richards"], "richards", 2, true, false, 32],
3124    [["splay"], "splay", 2, true, false, 32],
3125    [["regexp"], "regexp", 2, true, false, 16],
3126    [["pdfjs"], "pdfjs", 2, false, false, 4],
3127    [["mandreel"], "mandreel", 2, false, false, 4],
3128    [["gbemu-part1", "gbemu-part2"], "gbemu", 2, false, false, 4],
3129    [["code-load"], "closure", 1, false, false, 16],
3130    [["code-load"], "jquery", 1, false, false, 16],
3131    [["box2d"], "box2d", 2, false, false, 8],
3132    [["zlib", "zlib-data"], "zlib", 2, false, true, 3],
3133    [["typescript", "typescript-input", "typescript-compiler"], "typescript", 2, false, true, 1]].each {
3134     | args |
3135     OCTANE.add OctaneBenchmark.new(*args)
3136   }
3137   OCTANE.add CustomTimedBenchmark.new("splay-latency", (OPENSOURCE_OCTANE_PATH + "splay.js").to_s)
3138
3139   KRAKEN = BenchmarkSuite.new("Kraken", :arithmeticMean, -1)
3140   ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
3141    "audio-oscillator", "imaging-darkroom", "imaging-desaturate",
3142    "imaging-gaussian-blur", "json-parse-financial",
3143    "json-stringify-tinderbox", "stanford-crypto-aes",
3144    "stanford-crypto-ccm", "stanford-crypto-pbkdf2",
3145    "stanford-crypto-sha256-iterative"].each {
3146     | name |
3147     KRAKEN.add KrakenBenchmark.new(name)
3148   }
3149   
3150   JSBENCH = BenchmarkSuite.new("JSBench", :geometricMean, 0)
3151   [["amazon-chrome", "urem"], ["amazon-chrome-win", "urem"], ["amazon-firefox", "urm"],
3152    ["amazon-firefox-win", "urm"], ["amazon-safari", "urem"], ["facebook-chrome", "urem"],
3153    ["facebook-chrome-win", "urem"], ["facebook-firefox", "urem"],
3154    ["facebook-firefox-win", "urem"], ["facebook-safari", "urem"],
3155    ["google-chrome", "urem"], ["google-chrome-win", "urem"], ["google-firefox", "uem"],
3156    ["google-firefox-win", "urem"], ["google-safari", "urem"], ["twitter-chrome", "urem"],
3157    ["twitter-chrome-win", "rem"], ["twitter-firefox", "urem"],
3158    ["twitter-firefox-win", "urem"], ["twitter-safari", "urem"], ["yahoo-chrome", "urem"],
3159    ["yahoo-chrome-win", "urem"], ["yahoo-firefox", "urem"], ["yahoo-firefox-win", "urem"],
3160    ["yahoo-safari", "urem"]].each {
3161     | nameAndMode |
3162     JSBENCH.add JSBenchBenchmark.new(*nameAndMode)
3163   }
3164
3165   TAILBENCH = BenchmarkSuite.new("TailBench", :geometricMean, 0)
3166   ["n-body", "merge-sort", "merge-sort-cps", "bf-interpreter"].each {
3167     | name |
3168     TAILBENCH.add TailBenchBenchmark.new(name);
3169   }
3170
3171   MICROBENCHMARKS = BenchmarkSuite.new("Microbenchmarks", :geometricMean, 0)
3172   Dir.foreach(MICROBENCHMARKS_PATH) {
3173     | filename |
3174     if filename =~ /\.js$/
3175       name = $~.pre_match
3176       MICROBENCHMARKS.add MicrobenchmarksBenchmark.new(name)
3177     end
3178   }
3179   
3180   ASMBENCH = BenchmarkSuite.new("AsmBench", :geometricMean, 0)
3181   if ASMBENCH_PATH
3182     Dir.foreach(ASMBENCH_PATH) {
3183       | filename |
3184       if filename =~ /\.js$/
3185         name = $~.pre_match
3186         ASMBENCH.add AsmBenchBenchmark.new(name)
3187       end
3188     }
3189   end
3190
3191   COMPRESSIONBENCH = BenchmarkSuite.new("CompressionBench", :geometricMean, 0)
3192   [[["huffman", "compression-data"], "huffman", ""],
3193    [["arithmetic", "compression-data"], "arithmetic", "Simple"],
3194    [["arithmetic", "compression-data"], "arithmetic", "Precise"],
3195    [["arithmetic", "compression-data"], "arithmetic", "Complex Precise"],
3196    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 0"],
3197    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 1"],
3198    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 2"],
3199    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 1"],
3200    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 2"],
3201    [["lz-string", "compression-data"], "lz-string", ""]
3202   ].each {
3203     | args |
3204     COMPRESSIONBENCH.add CompressionBenchBenchmark.new(*args)
3205   }
3206
3207   SIXSPEED = BenchmarkSuite.new("SixSpeed", :geometricMean, 0)
3208   [[ "template_string", 200000000 ],
3209    [ "defaults", 100000000 ],
3210    [ "map-set-lookup", 200000 ],
3211    [ "spread", 1000000 ],
3212    [ "object-assign", 600000 ],
3213    [ "spread-literal", 1000000 ],
3214    [ "map-set", 10000 ],
3215    [ "destructuring-simple", 20000000 ],
3216    [ "super", 3000000 ],
3217    [ "for-of-object", 1000000 ],
3218    [ "rest", 500000 ],
3219    [ "regex-u", 1000000 ],
3220    [ "arrow", 20000000 ],
3221    [ "bindings-compound", 20000000 ],
3222    [ "classes", 10000000 ],
3223    [ "template_string_tag", 2000000 ],
3224    [ "map-string", 30000000 ],
3225    [ "arrow-declare", 30000000 ],
3226    [ "spread-generator", 1000000 ],
3227    [ "object-literal-ext", 1000000 ],
3228    [ "generator", 3000000 ],
3229    [ "arrow-args", 20000000 ],
3230    [ "for-of-array", 5000000 ],
3231    [ "bindings", 20000000 ],
3232    [ "destructuring", 20000000 ],
3233    [ "map-set-object", 5000 ],
3234   ].each {
3235     | name, iterations |
3236     SIXSPEED.add SixSpeedBenchmark.new("#{name}.es5", "#{name}/#{name}.es5", iterations)
3237     SIXSPEED.add SixSpeedBenchmark.new("#{name}.es6", "#{name}/#{name}.es6", iterations)
3238   }
3239
3240   DSPJS = BenchmarkSuite.new("DSP", :geometricMean, 0)
3241   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-posterize-tint", "e2")
3242   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-contrast-sat-bright", "e5")
3243   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-sat-adj-contr-mult", "e7")
3244   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-blur-overlay-sat-contr", "e8")
3245   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sat-blur-mult-sharpen-contr", "e9")
3246   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sepia-bias", "e10")
3247   DSPJS.add DSPJSVP8Benchmark.new
3248   DSPJS.add DSPStarfieldBenchmark.new
3249   DSPJS.add DSPJSJSLinuxBenchmark.new
3250   DSPJS.add DSPJSQuake3Benchmark.new
3251   DSPJS.add DSPJSMandelbrotBenchmark.new
3252   DSPJS.add DSPJSAmmoJSASMBenchmark.new
3253   DSPJS.add DSPJSAmmoJSRegularBenchmark.new
3254
3255   BROWSERMARK_JS = BenchmarkSuite.new("BrowsermarkJS", :geometricMean, 1)
3256   ["array_blur", "array_weighted", "string_chat", "string_filter", "string_weighted"].each {
3257     | name |
3258       BROWSERMARK_JS.add BrowsermarkJSBenchmark.new(name)
3259   }
3260   
3261   BROWSERMARK_DOM = BenchmarkSuite.new("BrowsermarkDOM", :geometricMean, 1)
3262   ["advanced_search", "create_source", "dynamic_create", "search"].each {
3263     | name |
3264       BROWSERMARK_DOM.add BrowsermarkDOMBenchmark.new(name)
3265   }
3266
3267   $suites = []
3268   
3269   if $includeSunSpider and not SUNSPIDER.empty?
3270     $suites << SUNSPIDER
3271   end
3272   
3273   if $includeSunSpiderCompileTime and not SUNSPIDER_COMPILE_TIME.empty?
3274     $suites << SUNSPIDER_COMPILE_TIME
3275   end
3276   
3277   if $includeLongSpider and not LONGSPIDER.empty?
3278     $suites << LONGSPIDER
3279   end
3280   
3281   if $includeV8 and not V8.empty?
3282     $suites << V8
3283   end
3284   
3285   if $includeV8CompileTime and not V8_COMPILE_TIME.empty?
3286     $suites << V8_COMPILE_TIME
3287   end
3288   
3289   if $includeOctane and not OCTANE.empty?
3290     if OCTANE_PATH
3291       $suites << OCTANE
3292     else
3293       $stderr.puts "Warning: refusing to run Octane because \"OctanePath\" isn't set in #{$configPath}."
3294     end
3295   end
3296   
3297   if $includeKraken and not KRAKEN.empty?
3298     if KRAKEN_PATH
3299       $suites << KRAKEN
3300     else
3301       $stderr.puts "Warning: refusing to run Kraken because \"KrakenPath\" isn't set in #{$configPath}."
3302     end
3303   end
3304   
3305   if $includeJSBench and not JSBENCH.empty?
3306     if $allDRT
3307       if JSBENCH_PATH
3308         $suites << JSBENCH
3309       else
3310         $stderr.puts "Warning: refusing to run JSBench because \"JSBenchPath\" isn't set in #{$configPath}"
3311       end
3312     else
3313       $stderr.puts "Warning: refusing to run JSBench because not all VMs are DumpRenderTree or WebKitTestRunner."
3314     end
3315   end
3316   
3317   if $includeTailBench and not TAILBENCH.empty?
3318     $suites << TAILBENCH
3319   end
3320
3321   if $includeMicrobenchmarks and not MICROBENCHMARKS.empty?
3322     $suites << MICROBENCHMARKS
3323   end
3324
3325   if $includeAsmBench and not ASMBENCH.empty?
3326     if ASMBENCH_PATH
3327       $suites << ASMBENCH
3328     else
3329       $stderr.puts "Warning: refusing to run AsmBench because \"AsmBenchPath\" isn't set in #{$configPath}."
3330     end
3331   end
3332   
3333   if $includeDSPJS and not DSPJS.empty?
3334     if $allDRT
3335       if DSPJS_FILTRR_PATH and DSPJS_ROUTE9_PATH and DSPJS_STARFIELD_PATH and DSPJS_QUAKE3_PATH and DSPJS_MANDELBROT_PATH and DSPJS_JSLINUX_PATH and DSPJS_AMMOJS_ASMJS_PATH and DSPJS_AMMOJS_REGULAR_PATH
3336         $suites << DSPJS
3337       else
3338         $stderr.puts "Warning: refusing to run DSPJS because one of the following isn't set in #{$configPath}: \"DSPJSFiltrrPath\", \"DSPJSRoute9Path\", \"DSPJSStarfieldPath\", \"DSPJSQuake3Path\", \"DSPJSMandelbrotPath\", \"DSPJSLinuxPath\", \"DSPJSAmmoJSAsmJSPath\", \"DSPJSAmmoJSRegularPath\"."
3339       end
3340     else
3341       $stderr.puts "Warning: refusing to run DSPJS because not all VMs are DumpRenderTree or WebKitTestRunner."
3342     end
3343   end
3344   
3345   if $includeBrowsermarkJS and not BROWSERMARK_JS.empty?
3346     if BROWSERMARK_PATH and BROWSERMARK_JS_PATH
3347       $suites << BROWSERMARK_JS
3348     else
3349       $stderr.puts "Warning: refusing to run Browsermark-JS because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\" or \"BrowserMarkJSPath\"."
3350     end
3351   end
3352
3353   if $includeBrowsermarkDOM and not BROWSERMARK_DOM.empty?
3354     if $allDRT
3355       if BROWSERMARK_PATH and BROWSERMARK_JS_PATH and BROWSERMARK_DOM_PATH
3356         $suites << BROWSERMARK_DOM
3357       else
3358         $stderr.puts "Warning: refusing to run Browsermark-DOM because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\", \"BrowserMarkJSPath\", or \"BrowserMarkDOMPath\"."
3359       end
3360     else
3361       $stderr.puts "Warning: refusing to run Browsermark-DOM because not all VMs are DumpRenderTree or WebKitTestRunner."
3362     end
3363   end
3364
3365   if $includeCompressionBench and not COMPRESSIONBENCH.empty?
3366     if COMPRESSIONBENCH_PATH
3367       $suites << COMPRESSIONBENCH
3368     else
3369       $stderr.puts "Warning: refusing to run CompressionBench because \"CompressionBenchPath\" isn't set in #{$configPath}"
3370     end
3371   end
3372
3373   if $includeSixSpeed and not SIXSPEED.empty?
3374     if SIXSPEED_PATH
3375       $suites << SIXSPEED
3376     else
3377       $stderr.puts "Warning: refusing to run SixSpeed because \"SixSpeedPath\" isn't set in #{$configPath}."
3378     end
3379   end
3380
3381
3382   $allSuites = $suites.map{|v| v.suites}.flatten(1)
3383   
3384   $benchmarks = []
3385   $suites.each {
3386     | suite |
3387     $benchmarks += suite.benchmarks
3388   }
3389   
3390   if $suites.empty? or $benchmarks.empty?
3391     $stderr.puts "No benchmarks found.  Bailing out."
3392     exit 1
3393   end
3394   
3395   if $outer*$inner == 1
3396     $stderr.puts "Warning: will only collect one sample per benchmark/VM.  Confidence interval calculation will fail."
3397   end
3398   
3399   $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
3400   
3401   $runPlans = []
3402   $vms.each {
3403     | vm |
3404     $benchmarks.each {
3405       | benchmark |
3406       $outer.times {
3407         | iteration |
3408         $runPlans << BenchRunPlan.new(benchmark, vm, iteration)
3409       }
3410     }
3411   }
3412   
3413   $runPlans.shuffle!
3414   
3415   if $sunSpiderWarmup
3416     warmupPlans = []
3417     $vms.each {
3418       | vm |
3419       WARMUP.benchmarks.each {
3420         | benchmark |
3421         warmupPlans << BenchRunPlan.new(benchmark, vm, 0)
3422       }
3423     }
3424     
3425     $runPlans = warmupPlans.shuffle + $runPlans
3426   end
3427   
3428   $suitepad = $suites.collect {
3429     | suite |
3430     suite.to_s.size
3431   }.max + 1
3432   
3433   $planpad = $runPlans.collect {
3434     | plan |
3435     plan.to_s.size
3436   }.max + 1
3437   
3438   maxBenchNameLength =
3439     ($benchmarks + ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
3440     | benchmark |
3441     if benchmark.respond_to? :name
3442       benchmark.name.size
3443     else
3444       benchmark.size
3445     end
3446   }.max
3447   $benchNameClip = 40
3448   $benchpad = [maxBenchNameLength, $benchNameClip].min + 1
3449   
3450   $weightpad = $benchmarks.collect {
3451     | benchmark |
3452     benchmark.weightString.size
3453   }.max
3454
3455   $vmpad = $vms.collect {
3456     | vm |
3457     vm.to_s.size
3458   }.max + 1
3459   
3460   $analyze.each_with_index {
3461     | filename, index |
3462     if index >= 1
3463       puts
3464     end
3465     parseAndDisplayResults(IO::read(filename))
3466   }
3467   
3468   if not $prepare and not $run
3469     exit 0
3470   end
3471   
3472   if FileTest.exist? BENCH_DATA_PATH
3473     cmd = "rm -rf #{BENCH_DATA_PATH}"
3474     $stderr.puts ">> #{cmd}" if $verbosity >= 2
3475     raise unless system cmd
3476   end
3477   
3478   Dir.mkdir BENCH_DATA_PATH
3479   
3480   if $needToCopyVMs
3481     canCopyIntoBenchPath = true
3482     $vms.each {
3483       | vm |
3484       canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath
3485     }
3486     
3487     if canCopyIntoBenchPath
3488       $vms.each {
3489         | vm |
3490         $stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..."
3491         vm.copyIntoBenchPath
3492       }
3493       $stderr.puts "All VMs are in place."
3494     else
3495       $stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it."
3496     end
3497   end
3498   
3499   if $measureGC and $measureGC != true
3500     found = false
3501     $vms.each {
3502       | vm |
3503       if vm.name == $measureGC
3504         found = true
3505       end
3506     }
3507     unless found
3508       $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
3509     end
3510   end
3511   
3512   if $prepare
3513     File.open("#{BENCH_DATA_PATH}/runscript", "w") {
3514       | file |
3515       file.puts "echo -e \"HOSTNAME:\\c\""
3516       file.puts "hostname"
3517       file.puts "echo"
3518       file.puts "echo -e \"HARDWARE:\\c\""
3519       file.puts "/usr/sbin/sysctl hw.model"
3520       file.puts "echo"
3521       file.puts "set -e"
3522       $script = file
3523       $runPlans.each_with_index {
3524         | plan, idx |
3525         if $verbosity == 0 and not $silent
3526           text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s
3527           text2 = plan.to_s
3528           file.puts("echo -e " + Shellwords.shellescape("\r#{text1} #{rpad(text2,$planpad)}") + "\"\\c\" 1>&2")
3529           file.puts("echo -e " + Shellwords.shellescape("\r#{text1} #{text2}") + "\"\\c\" 1>&2")
3530         end
3531         plan.emitRunCode
3532       }
3533       if $verbosity == 0 and not $silent
3534         file.puts("echo -e " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}") + "\"\\c\" 1>&2")
3535         file.puts("echo -e " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size}") + " 1>&2")
3536       end
3537     }
3538   end
3539   
3540   if $run
3541     unless $remoteHosts.empty?
3542       $stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0
3543       Dir.chdir(TEMP_PATH) {
3544         cmd = "tar -czf payload.tar.gz benchdata"
3545         $stderr.puts ">> #{cmd}" if $verbosity>=2
3546         raise unless system(cmd)
3547       }
3548       
3549       def grokHost(host)
3550         if host =~ /:([0-9]+)$/
3551           "-p " + $1 + " " + Shellwords.shellescape($~.pre_match)
3552         else
3553           Shellwords.shellescape(host)
3554         end
3555       end
3556       
3557       def sshRead(host, command)
3558         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3559         $stderr.puts ">> #{cmd}" if $verbosity>=2
3560         result = ""
3561         IO.popen(cmd, "r") {
3562           | inp |
3563           inp.each_line {
3564             | line |
3565             $stderr.puts "#{host}: #{line}" if $verbosity>=2
3566             result += line
3567           }
3568         }
3569         raise "#{$?}" unless $?.success?
3570         result
3571       end
3572       
3573       def sshWrite(host, command, data)
3574         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3575         $stderr.puts ">> #{cmd}" if $verbosity>=2
3576         IO.popen(cmd, "w") {
3577           | outp |
3578           outp.write(data)
3579         }
3580         raise "#{$?}" unless $?.success?
3581       end
3582       
3583       $remoteHosts.each {
3584         | host |
3585         $stderr.puts "Sending benchmark payload to #{host}..." if $verbosity==0
3586         
3587         remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"]
3588         raise unless remoteTempPath
3589         
3590         sshWrite(host, "cd #{Shellwords.shellescape(remoteTempPath)} && rm -rf benchdata && tar -xz", IO::read("#{TEMP_PATH}/payload.tar.gz"))
3591         
3592         $stderr.puts "Running on #{host}..." if $verbosity==0
3593         
3594         parseAndDisplayResults(sshRead(host, "cd #{Shellwords.shellescape(remoteTempPath + '/benchdata')} && sh runscript"))
3595       }
3596     end
3597     
3598     if not $remoteHosts.empty? and $alsoLocal
3599       $stderr.puts "Running locally..."
3600     end
3601     
3602     if $remoteHosts.empty? or $alsoLocal
3603       parseAndDisplayResults(runAndGetResults)
3604     end
3605   end
3606   
3607   if $prepare and not $run and $analyze.empty?
3608     puts wrap("Benchmarking script and data are in #{BENCH_DATA_PATH}. You can run "+
3609               "the benchmarks and get the results by doing:", 78)
3610     puts
3611     puts "cd #{BENCH_DATA_PATH}"
3612     puts "sh runscript > results.txt"
3613     puts
3614     puts wrap("Then you can analyze the results by running bencher with the same arguments "+
3615               "as now, but replacing --prepare-only with --analyze results.txt.", 78)
3616   end
3617 rescue => e
3618   fail(e)
3619 end
3620