Support the TailBench9000 benchmark in run-jsc-benchmarks
[WebKit-https.git] / Tools / Scripts / run-jsc-benchmarks
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2011-2015 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 # 1. Redistributions of source code must retain the above copyright
9 #    notice, this list of conditions and the following disclaimer.
10 # 2. Redistributions in binary form must reproduce the above copyright
11 #    notice, this list of conditions and the following disclaimer in the
12 #    documentation and/or other materials provided with the distribution.
13 #
14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24 # THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'rubygems'
27
28 require 'getoptlong'
29 require 'pathname'
30 require 'shellwords'
31 require 'socket'
32
33 begin
34   require 'json'
35 rescue LoadError => e
36   $stderr.puts "It does not appear that you have the 'json' package installed.  Try running 'sudo gem install json'."
37   exit 1
38 end
39
40 SCRIPT_PATH = Pathname.new(__FILE__).realpath
41
42 raise unless SCRIPT_PATH.dirname.basename.to_s == "Scripts"
43 raise unless SCRIPT_PATH.dirname.dirname.basename.to_s == "Tools"
44
45 OPENSOURCE_PATH = SCRIPT_PATH.dirname.dirname.dirname
46 PERFORMANCETESTS_PATH = OPENSOURCE_PATH + "PerformanceTests"
47
48 SUNSPIDER_PATH = PERFORMANCETESTS_PATH + "SunSpider" + "tests" + "sunspider-1.0"
49 LONGSPIDER_PATH = PERFORMANCETESTS_PATH + "LongSpider"
50 V8_PATH = PERFORMANCETESTS_PATH + "SunSpider" + "tests" + "v8-v6"
51 TAILBENCH_PATH = PERFORMANCETESTS_PATH + "TailBench9000"
52 MICROBENCHMARKS_PATH = OPENSOURCE_PATH + "JSTests" + "microbenchmarks"
53 OCTANE_WRAPPER_PATH = PERFORMANCETESTS_PATH + "Octane" + "wrappers"
54 JSBENCH_PATH = PERFORMANCETESTS_PATH + "JSBench"
55 SIXSPEED_PATH = PERFORMANCETESTS_PATH + "SixSpeed" + "tests"
56 SIXSPEED_WRAPPER_PATH = PERFORMANCETESTS_PATH + "SixSpeed" + "wrappers"
57
58 TEMP_PATH = OPENSOURCE_PATH + "BenchmarkTemp"
59
60 if TEMP_PATH.exist?
61   raise unless TEMP_PATH.directory?
62 else
63   Dir.mkdir(TEMP_PATH)
64 end
65
66 BENCH_DATA_PATH = TEMP_PATH + "benchdata"
67
68 IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933, 
69             0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684, 
70             0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227, 
71             0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674, 
72             0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652, 
73             0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604, 
74             0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338, 
75             0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538, 
76             0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185, 
77             0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694, 
78             0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244, 
79             0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067, 
80             0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323, 
81             0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129, 
82             0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574, 
83             0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722, 
84             0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624, 
85             0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321, 
86             0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843, 
87             0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217, 
88             0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464, 
89             0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599, 
90             0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638, 
91             0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592, 
92             0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471, 
93             0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283, 
94             0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037, 
95             0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738, 
96             0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481, 
97             0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085, 
98             0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651, 
99             0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183, 
100             0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683, 
101             0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154, 
102             0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599, 
103             0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019, 
104             0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417, 
105             0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795, 
106             0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153, 
107             0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494, 
108             0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818, 
109             0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128, 
110             0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423, 
111             0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704, 
112             0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974, 
113             0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232, 
114             0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479, 
115             0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716, 
116             0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943, 
117             0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192, 
118             0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401, 
119             0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963, 
120             0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822, 
121             0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007, 
122             0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186, 
123             0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358, 
124             0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525, 
125             0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686, 
126             0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841, 
127             0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991, 
128             0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137, 
129             0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278, 
130             0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414, 
131             0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547, 
132             0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675, 
133             0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799, 
134             0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192, 
135             0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037, 
136             0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151, 
137             0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262, 
138             0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369, 
139             0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489, 
140             0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259, 
141             0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689, 
142             0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785, 
143             0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879, 
144             0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297, 
145             0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059, 
146             0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145, 
147             0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242, 
148             0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324, 
149             0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404, 
150             0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483, 
151             0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559, 
152             0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634, 
153             0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707, 
154             0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778, 
155             0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848, 
156             0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916, 
157             0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983, 
158             0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057, 
159             0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121, 
160             0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183, 
161             0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244, 
162             0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304, 
163             0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363, 
164             0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442, 
165             0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476, 
166             0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539, 
167             0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946, 
168             0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653, 
169             0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712, 
170             0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762, 
171             0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812, 
172             0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867, 
173             0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915, 
174             0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962, 
175             0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007, 
176             0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059, 
177             0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103, 
178             0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146, 
179             0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189, 
180             0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231, 
181             0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272, 
182             0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312, 
183             0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352, 
184             0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391, 
185             0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429, 
186             0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467, 
187             0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504, 
188             0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546, 
189             0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582, 
190             0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617, 
191             0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652, 
192             0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686, 
193             0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724, 
194             0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757, 
195             0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579, 
196             0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822, 
197             0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853, 
198             0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885, 
199             0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915, 
200             0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595, 
201             0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979, 
202             0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013, 
203             0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041, 
204             0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074, 
205             0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102, 
206             0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129, 
207             0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156, 
208             0.99616, 0.996164]
209
210 # Run-time configuration parameters (can be set with command-line options)
211
212 $rerun=1
213 $inner=1
214 $warmup=1
215 $outer=4
216 $quantum=1000
217 $includeSunSpider=true
218 $includeLongSpider=true
219 $includeV8=true
220 $includeKraken=true
221 $includeJSBench=true
222 $includeMicrobenchmarks=true
223 $includeAsmBench=true
224 $includeDSPJS=true
225 $includeBrowsermarkJS=false
226 $includeBrowsermarkDOM=false
227 $includeOctane=true
228 $includeCompressionBench = true
229 $includeSixSpeed = true
230 $includeTailBench = true
231 $measureGC=false
232 $benchmarkPattern=nil
233 $verbosity=0
234 $timeMode=:preciseTime
235 $forceVMKind=nil
236 $brief=false
237 $silent=false
238 $remoteHosts=[]
239 $alsoLocal=false
240 $sshOptions=[]
241 $vms = []
242 $environment = {}
243 $dependencies = []
244 $needToCopyVMs = false
245 $dontCopyVMs = false
246 $allDRT = true
247 $outputName = nil
248 $sunSpiderWarmup = true
249 $configPath = Pathname.new(ENV["HOME"]) + ".run-jsc-benchmarks"
250
251 $prepare = true
252 $run = true
253 $analyze = []
254
255 # Helpful functions and classes
256
257 def smallUsage
258   puts "Use the --help option to get basic usage information."
259   exit 1
260 end
261
262 def usage
263   puts "run-jsc-benchmarks [options] <vm1> [<vm2> ...]"
264   puts
265   puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
266   puts "benchmarks, and reports detailed statistics.  What makes run-jsc-benchmarks"
267   puts "special is that each benchmark/VM configuration is run in a single VM invocation,"
268   puts "and the invocations are run in random order.  This minimizes systematics due to"
269   puts "one benchmark polluting the running time of another.  The fine-grained"
270   puts "interleaving of VM invocations further minimizes systematics due to changes in"
271   puts "the performance or behavior of your machine."
272   puts 
273   puts "Run-jsc-benchmarks is highly configurable.  You can compare as many VMs as you"
274   puts "like.  You can change the amount of warm-up iterations, number of iterations"
275   puts "executed per VM invocation, and the number of VM invocations per benchmark."
276   puts
277   puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
278   puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
279   puts "the executable and <name> is the name that you would like to give the"
280   puts "configuration for the purposeof reporting.  If no name is given, a generic name"
281   puts "of the form Conf#<n> will be ascribed to the configuration automatically."
282   puts
283   puts "It's also possible to specify per-VM environment variables. For example, you"
284   puts "might specify a VM like Foo:JSC_useJIT=false:/path/to/jsc, in which case the"
285   puts "harness will set the JSC_useJIT environment variable to false just before running"
286   puts "the given VM. Note that the harness will not unset the environment variable, so"
287   puts "you must ensure that your other VMs will use the opposite setting"
288   puts "(JSC_useJIT=true in this case)."
289   puts
290   puts "Options:"
291   puts "--rerun <n>          Set the number of iterations of the benchmark that"
292   puts "                     contribute to the measured run time.  Default is #{$rerun}."
293   puts "--inner <n>          Set the number of inner (per-runtime-invocation)"
294   puts "                     iterations.  Default is #{$inner}."
295   puts "--outer <n>          Set the number of runtime invocations for each benchmark."
296   puts "                     Default is #{$outer}."
297   puts "--warmup <n>         Set the number of warm-up runs per invocation.  Default"
298   puts "                     is #{$warmup}. This has a different effect on different kinds"
299   puts "                     benchmarks. Some benchmarks have no notion of warm-up."
300   puts "--no-ss-warmup       Disable SunSpider-based warm-up runs."
301   puts "--quantum <n>        Set the duration in milliseconds for which an iteration of"
302   puts "                     a throughput benchmark should be run.  Default is #{$quantum}."
303   puts "--timing-mode        Set the way that time is measured.  Possible values"
304   puts "                     are 'preciseTime' and 'date'.  Default is 'preciseTime'."
305   puts "--force-vm-kind      Turn off auto-detection of VM kind, and assume that it is"
306   puts "                     the one specified.  Valid arguments are 'jsc', "
307   puts "                     'DumpRenderTree', or 'WebKitTestRunner'."
308   puts "--force-vm-copy      Force VM builds to be copied to the working directory."
309   puts "                     This may reduce pathologies resulting from path names."
310   puts "--dont-copy-vms      Don't copy VMs even when doing a remote benchmarking run;"
311   puts "                     instead assume that they are already there."
312   puts "--sunspider          Only run SunSpider."
313   puts "--v8-spider          Only run V8."
314   puts "--kraken             Only run Kraken."
315   puts "--js-bench           Only run JSBench."
316   puts "--microbenchmarks    Only run microbenchmarks."
317   puts "--dsp                Only run DSP."
318   puts "--asm-bench          Only run AsmBench."
319   puts "--browsermark-js     Only run browsermark-js."
320   puts "--browsermark-dom    Only run browsermark-dom."
321   puts "--octane             Only run Octane."
322   puts "--tail-bench         Only run TailBench"
323   puts "--compression-bench  Only run compression bench"
324   puts "                     The default is to run all benchmarks. The above options can"
325   puts "                     be combined to run any subset (so --sunspider --dsp will run"
326   puts "                     both SunSpider and DSP)."
327   puts "--six-speed          Only run SixSpeed."
328   puts "--benchmarks         Only run benchmarks matching the given regular expression."
329   puts "--measure-gc         Turn off manual calls to gc(), so that GC time is measured."
330   puts "                     Works best with large values of --inner.  You can also say"
331   puts "                     --measure-gc <conf>, which turns this on for one"
332   puts "                     configuration only."
333   puts "--verbose or -v      Print more stuff."
334   puts "--brief              Print only the final result for each VM."
335   puts "--silent             Don't print progress. This might slightly reduce some"
336   puts "                     performance perturbation."
337   puts "--remote <sshhosts>  Perform performance measurements remotely, on the given"
338   puts "                     SSH host(s). Easiest way to use this is to specify the SSH"
339   puts "                     user@host string. However, you can also supply a comma-"
340   puts "                     separated list of SSH hosts. Alternatively, you can use this"
341   puts "                     option multiple times to specify multiple hosts. This"
342   puts "                     automatically copies the WebKit release builds of the VMs"
343   puts "                     you specified to all of the hosts."
344   puts "--ssh-options        Pass additional options to SSH."
345   puts "--local              Also do a local benchmark run even when doing --remote."
346   puts "--vms                Use a JSON file to specify which VMs to run, as opposed to"
347   puts "                     specifying them on the command line."
348   puts "--prepare-only       Only prepare the runscript (a shell script that"
349   puts "                     invokes the VMs to run benchmarks) but don't run it."
350   puts "--analyze            Only read the output of the runscript but don't do anything"
351   puts "                     else. This requires passing the same arguments that you"
352   puts "                     passed when running --prepare-only."
353   puts "--output-name        Base of the filenames to put results into. Will write a file"
354   puts "                     called <base>_report.txt and <base>.json.  By default this"
355   puts "                     name is automatically synthesized from the machine name,"
356   puts "                     date, set of benchmarks run, and set of configurations."
357   puts "--environment        JSON file that specifies the environment variables that should"
358   puts "                     be used for particular VMs and benchmarks."
359   puts "--config <path>      Specify the path of the configuration file. Defaults to"
360   puts "                     ~/.run-jsc-benchmarks"
361   puts "--dependencies       Additional dependent library paths."
362   puts "--help or -h         Display this message."
363   puts
364   puts "Example:"
365   puts "run-jsc-benchmarks TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
366   exit 1
367 end
368
369 def fail(reason)
370   if reason.respond_to? :backtrace
371     puts "FAILED: #{reason.inspect}"
372     puts "Stack trace:"
373     puts reason.backtrace.join("\n")
374   else
375     puts "FAILED: #{reason.inspect}"
376   end
377   smallUsage
378 end
379
380 def quickFail(r1,r2)
381   $stderr.puts "#{$0}: #{r1}"
382   puts
383   fail(r2)
384 end
385
386 def intArg(argName,arg,min,max)
387   result=arg.to_i
388   unless result.to_s == arg
389     quickFail("Expected an integer value for #{argName}, but got #{arg}.",
390               "Invalid argument for command-line option")
391   end
392   if min and result<min
393     quickFail("Argument for #{argName} cannot be smaller than #{min}.",
394               "Invalid argument for command-line option")
395   end
396   if max and result>max
397     quickFail("Argument for #{argName} cannot be greater than #{max}.",
398               "Invalid argument for command-line option")
399   end
400   result
401 end
402
403 def computeMean(array)
404   sum=0.0
405   array.each {
406     | value |
407     sum += value
408   }
409   sum/array.length
410 end
411
412 def computeGeometricMean(array)
413   sum = 0.0
414   array.each {
415     | value |
416     sum += Math.log(value)
417   }
418   Math.exp(sum * (1.0/array.length))
419 end
420
421 def computeHarmonicMean(array)
422   1.0 / computeMean(array.collect{ | value | 1.0 / value })
423 end
424
425 def computeStdDev(array)
426   case array.length
427   when 0
428     0.0/0.0
429   when 1
430     0.0
431   else
432     begin
433       mean=computeMean(array)
434       sum=0.0
435       array.each {
436         | value |
437         sum += (value-mean)**2
438       }
439       Math.sqrt(sum/(array.length-1))
440     rescue
441       0.0/0.0
442     end
443   end
444 end
445
446 class Array
447   def shuffle!
448     size.downto(1) { |n| push delete_at(rand(n)) }
449     self
450   end
451 end
452
453 def inverseBetaRegularized(n)
454   IBR_LOOKUP[n-1]
455 end
456
457 def numToStr(num, decimalShift)
458   ("%." + (4 + decimalShift).to_s + "f") % (num.to_f)
459 end
460
461 class CantSay
462   def initialize
463   end
464   
465   def shortForm
466     " "
467   end
468   
469   def longForm
470     ""
471   end
472   
473   def to_s
474     ""
475   end
476 end
477   
478 class NoChange
479   attr_reader :amountFaster
480   
481   def initialize(amountFaster)
482     @amountFaster = amountFaster
483   end
484   
485   def shortForm
486     " "
487   end
488   
489   def longForm
490     "  might be #{numToStr(@amountFaster, 0)}x faster"
491   end
492   
493   def to_s
494     if @amountFaster < 1.01
495       ""
496     else
497       longForm
498     end
499   end
500 end
501
502 class Faster
503   attr_reader :amountFaster
504   
505   def initialize(amountFaster)
506     @amountFaster = amountFaster
507   end
508   
509   def shortForm
510     "^"
511   end
512   
513   def longForm
514     "^ definitely #{numToStr(@amountFaster, 0)}x faster"
515   end
516   
517   def to_s
518     longForm
519   end
520 end
521
522 class Slower
523   attr_reader :amountSlower
524   
525   def initialize(amountSlower)
526     @amountSlower = amountSlower
527   end
528   
529   def shortForm
530     "!"
531   end
532   
533   def longForm
534     "! definitely #{numToStr(@amountSlower, 0)}x slower"
535   end
536   
537   def to_s
538     longForm
539   end
540 end
541
542 class MayBeSlower
543   attr_reader :amountSlower
544   
545   def initialize(amountSlower)
546     @amountSlower = amountSlower
547   end
548   
549   def shortForm
550     "?"
551   end
552   
553   def longForm
554     "? might be #{numToStr(@amountSlower, 0)}x slower"
555   end
556   
557   def to_s
558     if @amountSlower < 1.01
559       "?"
560     else
561       longForm
562     end
563   end
564 end
565
566 def jsonSanitize(value)
567   if value.is_a? Fixnum
568     value
569   elsif value.is_a? Float
570     if value.nan? or value.infinite?
571       value.to_s
572     else
573       value
574     end
575   elsif value.is_a? Array
576     value.map{|v| jsonSanitize(v)}
577   elsif value.nil?
578     value
579   else
580     raise "Unrecognized value #{value.inspect}"
581   end
582 end
583
584 class Stats
585   def initialize
586     @array = []
587   end
588   
589   def add(value)
590     if not value or not @array
591       @array = nil
592     elsif value.is_a? Float
593       if value.nan? or value.infinite?
594         @array = nil
595       else
596         @array << value
597       end
598     elsif value.is_a? Stats
599       add(value.array)
600     elsif value.respond_to? :each
601       value.each {
602         | v |
603         add(v)
604       }
605     else
606       @array << value.to_f
607     end
608   end
609   
610   def status
611     if @array
612       :ok
613     else
614       :error
615     end
616   end
617   
618   def error?
619     # TODO: We're probably still not handling this case correctly. 
620     not @array or @array.empty?
621   end
622   
623   def ok?
624     not not @array
625   end
626     
627   def array
628     @array
629   end
630   
631   def sum
632     result=0
633     @array.each {
634       | value |
635       result += value
636     }
637     result
638   end
639   
640   def min
641     @array.min
642   end
643   
644   def max
645     @array.max
646   end
647   
648   def size
649     @array.length
650   end
651   
652   def mean
653     computeMean(array)
654   end
655   
656   def arithmeticMean
657     mean
658   end
659   
660   def stdDev
661     computeStdDev(array)
662   end
663
664   def stdErr
665     stdDev/Math.sqrt(size)
666   end
667   
668   # Computes a 95% Student's t distribution confidence interval
669   def confInt
670     if size < 2
671       0.0/0.0
672     else
673       raise if size > 1000
674       Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
675     end
676   end
677   
678   def lower
679     mean-confInt
680   end
681   
682   def upper
683     mean+confInt
684   end
685   
686   def geometricMean
687     computeGeometricMean(array)
688   end
689   
690   def harmonicMean
691     computeHarmonicMean(array)
692   end
693   
694   def compareTo(other)
695     return CantSay.new unless ok? and other.ok?
696     
697     if upper < other.lower
698       Faster.new(other.mean/mean)
699     elsif lower > other.upper
700       Slower.new(mean/other.mean)
701     elsif mean > other.mean
702       MayBeSlower.new(mean/other.mean)
703     else
704       NoChange.new(other.mean/mean)
705     end
706   end
707   
708   def to_s
709     "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
710   end
711   
712   def jsonMap
713     if ok?
714       {"data"=>jsonSanitize(@array), "mean"=>jsonSanitize(mean), "confInt"=>jsonSanitize(confInt)}
715     else
716       "ERROR"
717     end
718   end
719 end
720
721 def doublePuts(out1,out2,msg)
722   out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
723   out2.puts msg
724 end
725
726 class Benchfile < File
727   @@counter = 0
728   
729   attr_reader :filename, :basename
730   
731   def initialize(name)
732     @basename, @filename = Benchfile.uniqueFilename(name)
733     super(@filename, "w")
734   end
735   
736   def self.uniqueFilename(name)
737     if name.is_a? Array
738       basename = name[0] + @@counter.to_s + name[1]
739     else
740       basename = name + @@counter.to_s
741     end
742     filename = BENCH_DATA_PATH + basename
743     @@counter += 1
744     raise "Benchfile #{filename} already exists" if FileTest.exist?(filename)
745     [basename, filename]
746   end
747   
748   def self.create(name)
749     file = Benchfile.new(name)
750     yield file
751     file.close
752     file.basename
753   end
754 end
755
756 $dataFiles={}
757 def ensureFile(key, filename)
758   unless $dataFiles[key]
759     $dataFiles[key] = Benchfile.create(key) {
760       | outp |
761       doublePuts($stderr,outp,IO::read(filename))
762     }
763   end
764   $dataFiles[key]
765 end
766
767 # Helper for files that cannot be renamed.
768 $absoluteFiles={}
769 def ensureAbsoluteFile(filename, basedir=nil)
770   return if $absoluteFiles[filename]
771   filename = Pathname.new(filename)
772
773   directory = Pathname.new('')
774   if basedir and filename.dirname != basedir
775     remainingPath = filename.dirname
776     while remainingPath != basedir
777       directory = remainingPath.basename + directory
778       remainingPath = remainingPath.dirname
779     end
780     if not $absoluteFiles[directory]
781       cmd = "mkdir -p #{Shellwords.shellescape((BENCH_DATA_PATH + directory).to_s)}"
782       $stderr.puts ">> #{cmd}" if $verbosity >= 2
783       raise unless system(cmd)
784       intermediateDirectory = Pathname.new(directory)
785       while intermediateDirectory.basename.to_s != "."
786         $absoluteFiles[intermediateDirectory] = true
787         intermediateDirectory = intermediateDirectory.dirname
788       end
789     end
790   end
791   
792   cmd = "cp #{Shellwords.shellescape(filename.to_s)} #{Shellwords.shellescape((BENCH_DATA_PATH + directory + filename.basename).to_s)}"
793   $stderr.puts ">> #{cmd}" if $verbosity >= 2
794   raise unless system(cmd)
795   $absoluteFiles[filename] = true
796 end
797
798 # Helper for large benchmarks with lots of files and directories.
799 def ensureBenchmarkFiles(rootdir)
800     toProcess = [rootdir]
801     while not toProcess.empty?
802       currdir = toProcess.pop
803       Dir.foreach(currdir.to_s) {
804         | filename |
805         path = currdir + filename
806         next if filename.match(/^\./)
807         toProcess.push(path) if File.directory?(path.to_s)
808         ensureAbsoluteFile(path, rootdir) if File.file?(path.to_s)
809       }
810     end
811 end
812
813 class JSCommand
814   attr_reader :js, :html
815   def initialize(js, html)
816     @js = js
817     @html = html
818   end
819 end
820
821 def loadCommandForFile(key, filename)
822   file = ensureFile(key, filename)
823   JSCommand.new("load(#{file.inspect});", "<script src=#{file.inspect}></script>")
824 end
825
826 def simpleCommand(command)
827   JSCommand.new(command, "<script type=\"text/javascript\">#{command}</script>")
828 end
829
830 # Benchmark that consists of a single file and must be loaded in its own global object each
831 # time (i.e. run()).
832 class SingleFileTimedBenchmarkParameters
833   attr_reader :benchPath
834   
835   def initialize(benchPath)
836     @benchPath = benchPath
837   end
838   
839   def kind
840     :singleFileTimedBenchmark
841   end
842 end
843
844 # Benchmark that consists of one or more data files that should be loaded globally, followed
845 # by a command to run the benchmark.
846 class MultiFileTimedBenchmarkParameters
847   attr_reader :dataPaths, :command
848
849   def initialize(dataPaths, command)
850     @dataPaths = dataPaths
851     @command = command
852   end
853   
854   def kind
855     :multiFileTimedBenchmark
856   end
857 end
858
859 # Benchmark that consists of one or more data files that should be loaded globally, followed
860 # by a command to run a short tick of the benchmark. The benchmark should be run for as many
861 # ticks as possible, for one quantum (quantum is 1000ms by default).
862 class ThroughputBenchmarkParameters
863   attr_reader :dataPaths, :setUpCommand, :command, :tearDownCommand, :doWarmup, :deterministic, :minimumIterations
864
865   def initialize(dataPaths, setUpCommand, command, tearDownCommand, doWarmup, deterministic, minimumIterations)
866     @dataPaths = dataPaths
867     @setUpCommand = setUpCommand
868     @command = command
869     @tearDownCommand = tearDownCommand
870     @doWarmup = doWarmup
871     @deterministic = deterministic
872     @minimumIterations = minimumIterations
873   end
874   
875   def kind
876     :throughputBenchmark
877   end
878 end
879
880 # Benchmark that can only run in DumpRenderTree or WebKitTestRunner, that has its own callback for reporting
881 # results. Other than that it's just like SingleFileTimedBenchmark.
882 class SingleFileTimedCallbackBenchmarkParameters
883   attr_reader :callbackDecl, :benchPath
884   
885   def initialize(callbackDecl, benchPath)
886     @callbackDecl = callbackDecl
887     @benchPath = benchPath
888   end
889   
890   def kind
891     :singleFileTimedCallbackBenchmark
892   end
893 end
894
895 def emitTimerFunctionCode(file)
896   case $timeMode
897   when :preciseTime
898     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
899     doublePuts($stderr,file,"   return preciseTime()*1000")
900     doublePuts($stderr,file,"}")
901   when :date
902     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
903     doublePuts($stderr,file,"   return Date.now()")
904     doublePuts($stderr,file,"}")
905   else
906     raise
907   end
908 end
909
910 def emitBenchRunCodeFile(name, plan, benchParams)
911   case plan.vm.vmType
912   when :jsc
913     Benchfile.create("bencher") {
914       | file |
915       emitTimerFunctionCode(file)
916       
917       if benchParams.kind == :multiFileTimedBenchmark
918         benchParams.dataPaths.each {
919           | path |
920           doublePuts($stderr,file,"load(#{path.inspect});")
921         }
922         doublePuts($stderr,file,"gc();")
923         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
924         doublePuts($stderr,file,"   var __before = __bencher_curTimeMS();")
925         $rerun.times {
926           doublePuts($stderr,file,"   #{benchParams.command.js}")
927         }
928         doublePuts($stderr,file,"   var __after = __bencher_curTimeMS();")
929         doublePuts($stderr,file,"   if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(__after-__before));");
930         doublePuts($stderr,file,"   gc();") unless plan.vm.shouldMeasureGC
931         doublePuts($stderr,file,"}")
932       elsif benchParams.kind == :throughputBenchmark
933         emitTimerFunctionCode(file)
934         benchParams.dataPaths.each {
935           | path |
936           doublePuts($stderr,file,"load(#{path.inspect});")
937         }
938         doublePuts($stderr,file,"#{benchParams.setUpCommand.js}")
939         if benchParams.doWarmup
940           warmup = $warmup
941         else
942           warmup = 0
943         end
944         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{warmup + $inner}; __bencher_index++) {")
945         doublePuts($stderr,file,"    var __before = __bencher_curTimeMS();")
946         doublePuts($stderr,file,"    var __after = __before;")
947         doublePuts($stderr,file,"    var __runs = 0;")
948         doublePuts($stderr,file,"    var __expected = #{$quantum};")
949         doublePuts($stderr,file,"    while (true) {")
950         $rerun.times {
951           doublePuts($stderr,file,"       #{benchParams.command.js}")
952         }
953         doublePuts($stderr,file,"       __runs++;")
954         doublePuts($stderr,file,"       __after = __bencher_curTimeMS();")
955         if benchParams.deterministic
956           doublePuts($stderr,file,"       if (true) {")
957         else
958           doublePuts($stderr,file,"       if (__after - __before >= __expected) {")
959         end
960         doublePuts($stderr,file,"           if (__runs >= #{benchParams.minimumIterations} || __bencher_index < #{warmup})")
961         doublePuts($stderr,file,"               break;")
962         doublePuts($stderr,file,"           __expected += #{$quantum}")
963         doublePuts($stderr,file,"       }")
964         doublePuts($stderr,file,"    }")
965         doublePuts($stderr,file,"    if (__bencher_index >= #{warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{warmup}) + \": Time: \"+((__after-__before)/__runs));")
966         doublePuts($stderr,file,"}")
967         doublePuts($stderr,file,"#{benchParams.tearDownCommand.js}")
968       else
969         raise unless benchParams.kind == :singleFileTimedBenchmark
970         doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
971         doublePuts($stderr,file,"   var __bencher_before = __bencher_curTimeMS();")
972         $rerun.times {
973           doublePuts($stderr,file,"   run(__bencher_what);")
974         }
975         doublePuts($stderr,file,"   var __bencher_after = __bencher_curTimeMS();")
976         doublePuts($stderr,file,"   return __bencher_after - __bencher_before;")
977         doublePuts($stderr,file,"}")
978         $warmup.times {
979           doublePuts($stderr,file,"__bencher_run(#{benchParams.benchPath.inspect})")
980           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
981         }
982         $inner.times {
983           | innerIndex |
984           doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchParams.benchPath.inspect}));")
985           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
986         }
987       end
988     }
989   when :dumpRenderTree, :webkitTestRunner
990     case $timeMode
991     when :preciseTime
992       curTime = "(testRunner.preciseTime()*1000)"
993     when :date
994       curTime = "(Date.now())"
995     else
996       raise
997     end
998
999     mainCode = Benchfile.create("bencher") {
1000       | file |
1001       doublePuts($stderr,file,"__bencher_count = 0;")
1002       doublePuts($stderr,file,"function __bencher_doNext(result) {")
1003       doublePuts($stderr,file,"    if (__bencher_count >= #{$warmup})")
1004       doublePuts($stderr,file,"        debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);")
1005       doublePuts($stderr,file,"    __bencher_count++;")
1006       doublePuts($stderr,file,"    if (__bencher_count < #{$inner+$warmup})")
1007       doublePuts($stderr,file,"        __bencher_runImpl(__bencher_doNext);")
1008       doublePuts($stderr,file,"    else")
1009       doublePuts($stderr,file,"        quit();")
1010       doublePuts($stderr,file,"}")
1011       doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);")
1012     }
1013     
1014     cssCode = Benchfile.create("bencher-css") {
1015       | file |
1016       doublePuts($stderr,file,".pass {\n    font-weight: bold;\n    color: green;\n}\n.fail {\n    font-weight: bold;\n    color: red;\n}\n\#console {\n    white-space: pre-wrap;\n    font-family: monospace;\n}")
1017     }
1018     
1019     preCode = Benchfile.create("bencher-pre") {
1020       | file |
1021       doublePuts($stderr,file,"if (window.testRunner) {")
1022       doublePuts($stderr,file,"    testRunner.dumpAsText(window.enablePixelTesting);")
1023       doublePuts($stderr,file,"    testRunner.waitUntilDone();")
1024       doublePuts($stderr,file,"}")
1025       doublePuts($stderr,file,"")
1026       doublePuts($stderr,file,"function debug(msg)")
1027       doublePuts($stderr,file,"{")
1028       doublePuts($stderr,file,"    var span = document.createElement(\"span\");")
1029       doublePuts($stderr,file,"    document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace")
1030       doublePuts($stderr,file,"    span.innerHTML = msg + '<br />';")
1031       doublePuts($stderr,file,"}")
1032       doublePuts($stderr,file,"")
1033       doublePuts($stderr,file,"function quit() {")
1034       doublePuts($stderr,file,"    testRunner.notifyDone();")
1035       doublePuts($stderr,file,"}")
1036       doublePuts($stderr,file,"")
1037       doublePuts($stderr,file,"__bencher_continuation=null;")
1038       doublePuts($stderr,file,"")
1039       doublePuts($stderr,file,"function reportResult(result) {")
1040       doublePuts($stderr,file,"    __bencher_continuation(result);")
1041       doublePuts($stderr,file,"}")
1042       doublePuts($stderr,file,"")
1043       doublePuts($stderr,file,"function currentTimeInMS(msg)")
1044       doublePuts($stderr,file,"{")
1045       doublePuts($stderr,file,"    return #{curTime};")
1046       doublePuts($stderr,file,"}")
1047       if benchParams.kind == :singleFileTimedCallbackBenchmark
1048         doublePuts($stderr,file,"")
1049         doublePuts($stderr,file,benchParams.callbackDecl)
1050       end
1051       doublePuts($stderr,file,"")
1052       doublePuts($stderr,file,"function __bencher_runImpl(continuation) {")
1053       doublePuts($stderr,file,"    function doit() {")
1054       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"\";")
1055       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";")
1056       doublePuts($stderr,file,"        var testFrame = document.getElementById(\"testframe\");")
1057       doublePuts($stderr,file,"        testFrame.contentDocument.open();")
1058       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");")
1059       if benchParams.kind == :throughputBenchmark or benchParams.kind == :multiFileTimedBenchmark
1060         benchParams.dataPaths.each {
1061           | path |
1062           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{path.inspect.inspect[1..-2]}></script>\");")
1063         }
1064       end
1065       if benchParams.kind == :throughputBenchmark
1066         if benchParams.doWarmup
1067           warmup = $warmup
1068         else
1069           warmup = 0
1070         end
1071         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">\");")
1072         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.setUpCommand.js}\");")
1073         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_before = #{curTime};\");")
1074         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_after = __bencher_before;\");")
1075         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_expected = #{$quantum};\");")
1076         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_runs = 0;\");")
1077         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"while (true) {\");")
1078         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    #{benchParams.command.js}\");")
1079         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_runs++;\");")
1080         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_after = #{curTime};\");")
1081         if benchParams.deterministic
1082           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (true) {\");")
1083         else
1084           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (__bencher_after - __bencher_before >= __bencher_expected) {\");")
1085         end
1086         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        if (__bencher_runs >= #{benchParams.minimumIterations} || window.parent.__bencher_count < #{warmup})\");")
1087         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"            break;\");")
1088         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        __bencher_expected += #{$quantum}\");")
1089         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    }\");")
1090         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"}\");")
1091         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.tearDownCommand.js}\");")
1092         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"window.parent.reportResult((__bencher_after - __bencher_before) / __bencher_runs);\");")
1093         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</script>\");")
1094       else
1095         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">var __bencher_before = #{curTime};</script>\");")
1096         if benchParams.kind == :multiFileTimedBenchmark
1097           doublePuts($stderr,file,"        testFrame.contentDocument.write(#{benchParams.command.html.inspect});")
1098         else
1099           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{benchParams.benchPath.inspect.inspect[1..-2]}></script>\");")
1100         end
1101         unless benchParams.kind == :singleFileTimedCallbackBenchmark
1102           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">window.parent.reportResult(#{curTime} - __bencher_before);</script>\");")
1103         end
1104       end
1105       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</body></html>\");")
1106       doublePuts($stderr,file,"        testFrame.contentDocument.close();")
1107       doublePuts($stderr,file,"    }")
1108       doublePuts($stderr,file,"    __bencher_continuation = continuation;")
1109       doublePuts($stderr,file,"    window.setTimeout(doit, 10);")
1110       doublePuts($stderr,file,"}")
1111     }
1112
1113     Benchfile.create(["bencher-htmldoc",".html"]) {
1114       | file |
1115       doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>")
1116     }
1117   else
1118     raise
1119   end
1120 end
1121
1122 def emitBenchRunCode(name, plan, benchParams)
1123   plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchParams), plan)
1124 end
1125
1126 class FileCreator
1127   def initialize(filename)
1128     @filename = filename
1129     @state = :empty
1130   end
1131   
1132   def puts(text)
1133     $script.print "echo #{Shellwords.shellescape(text)}"
1134     if @state == :empty
1135       $script.print " > "
1136       @state = :nonEmpty
1137     else
1138       $script.print " >> "
1139     end
1140     $script.puts "#{Shellwords.shellescape(@filename)}"
1141   end
1142   
1143   def close
1144     if @state == :empty
1145       $script.puts "rm -f #{Shellwords.shellescape(text)}"
1146       $script.puts "touch #{Shellwords.shellescape(text)}"
1147     end
1148   end
1149   
1150   def self.open(filename)
1151     outp = FileCreator.new(filename)
1152     yield outp
1153     outp.close
1154   end
1155 end
1156
1157 def emitSelfContainedBenchRunCode(name, plan, targetFile, configFile, benchmark)
1158   FileCreator.open(configFile) {
1159     | outp |
1160     outp.puts "__bencher_message = \"#{name}: #{plan.vm}: #{plan.iteration}: \";"
1161     outp.puts "__bencher_warmup = #{$warmup};"
1162     outp.puts "__bencher_inner = #{$inner};"
1163     outp.puts "__bencher_benchmark = #{benchmark.to_json};"
1164     case $timeMode
1165     when :preciseTime
1166       outp.puts "__bencher_curTime = (function(){ return testRunner.preciseTime() * 1000; });"
1167     when :date
1168       outp.puts "__bencher_curTime = (function(){ return Date.now(); });"
1169     else
1170       raise
1171     end
1172   }
1173   
1174   plan.vm.emitRunCode(targetFile, plan)
1175 end
1176
1177 def planForDescription(string, plans, benchFullname, vmName, iteration)
1178   raise "Unexpected benchmark full name: #{benchFullname.inspect}, string: #{string.inspect}" unless benchFullname =~ /\//
1179   suiteName = $~.pre_match
1180   return nil if suiteName == "WARMUP"
1181   benchName = $~.post_match
1182   result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration}
1183   raise "Unexpected result dimensions: #{result.inspect}, string: #{string.inspect}" unless result.size == 1
1184   result[0]
1185 end
1186
1187 class ParsedResult
1188   attr_reader :plan, :innerIndex, :time, :result
1189   
1190   def initialize(plan, innerIndex, time)
1191     @plan = plan
1192     @innerIndex = innerIndex
1193     if time == :crashed
1194       @result = :error
1195     else
1196       @time = time
1197       @result = :success
1198     end
1199     
1200     raise unless @plan.is_a? BenchPlan
1201     raise unless @innerIndex.is_a? Integer
1202     raise unless @time.is_a? Numeric or @result == :error
1203   end
1204   
1205   def benchmark
1206     plan.benchmark
1207   end
1208   
1209   def suite
1210     plan.suite
1211   end
1212   
1213   def vm
1214     plan.vm
1215   end
1216   
1217   def outerIndex
1218     plan.iteration
1219   end
1220   
1221   def self.create(plan, innerIndex, time)
1222     if plan
1223       ParsedResult.new(plan, innerIndex, time)
1224     else
1225       nil
1226     end
1227   end
1228   
1229   def self.parse(plans, string)
1230     if string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): Time: /
1231       benchFullname = $1
1232       vmName = $2
1233       outerIndex = $3.to_i
1234       innerIndex = $4.to_i
1235       time = $~.post_match.to_f
1236       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, time)
1237     elsif string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): CRASHED/
1238       benchFullname = $1
1239       vmName = $2
1240       outerIndex = $3.to_i
1241       innerIndex = $4.to_i
1242       time = $~.post_match.to_f
1243       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, :crashed)
1244     else
1245       nil
1246     end
1247   end
1248 end
1249
1250 class VM
1251   @@extraEnvSet = {}
1252   
1253   def initialize(origPath, name, nameKind, svnRevision)
1254     @origPath = origPath.to_s
1255     @path = origPath.to_s
1256     @name = name
1257     @nameKind = nameKind
1258     @extraEnv = {}
1259     
1260     if $forceVMKind
1261       @vmType = $forceVMKind
1262     else
1263       if @origPath =~ /DumpRenderTree$/
1264         @vmType = :dumpRenderTree
1265       elsif @origPath =~ /WebKitTestRunner$/
1266         @vmType = :webkitTestRunner
1267       else
1268         @vmType = :jsc
1269       end
1270     end
1271     
1272     @svnRevision = svnRevision
1273     
1274     # Try to detect information about the VM.
1275     if path =~ /\/WebKitBuild\/(Release|Debug)+\/([a-zA-Z]+)$/
1276       @checkoutPath = $~.pre_match
1277       # FIXME: Use some variant of this: 
1278       # <bdash>   def retrieve_revision
1279       # <bdash>     `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i
1280       # <bdash>   end
1281       unless @svnRevision
1282         begin
1283           Dir.chdir(@checkoutPath) {
1284             $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
1285             IO.popen("svn info", "r") {
1286               | inp |
1287               inp.each_line {
1288                 | line |
1289                 if line =~ /Revision: ([0-9]+)/
1290                   @svnRevision = $1
1291                 end
1292               }
1293             }
1294           }
1295           unless @svnRevision
1296             $stderr.puts "Warning: running svn info for #{name} silently failed."
1297           end
1298         rescue => e
1299           # Failed to detect svn revision.
1300           $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
1301         end
1302       end
1303     else
1304       $stderr.puts "Warning: could not identify checkout location for #{name}"
1305     end
1306     
1307     if @path =~ /\/Release\/([a-zA-Z]+)$/
1308       @libPath, @relativeBinPath = [$~.pre_match+"/Release"], "./#{$1}"
1309     elsif @path =~ /\/Debug\/([a-zA-Z]+)$/
1310       @libPath, @relativeBinPath = [$~.pre_match+"/Debug"], "./#{$1}"
1311     elsif @path =~ /\/Release\/bin\/([a-zA-Z]+)$/
1312       @libPath, @relativeBinPath = [$~.pre_match+"/Release/lib"], "./#{$1}"
1313     elsif @path =~ /\/Debug\/bin\/([a-zA-Z]+)$/
1314       @libPath, @relativeBinPath = [$~.pre_match+"/Debug/lib"], "./#{$1}"
1315     elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/
1316       @libPath = [$~.pre_match + "/Contents/Resources", $~.pre_match + "/Contents/Frameworks"]
1317     elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/
1318       @libPath, @relativeBinPath = [$~.pre_match], $&[1..-1]
1319     elsif @path =~ /(DumpRenderTree|webkitTestRunner|jsc)$/
1320       @libPath, @relativeBinPath = [$~.pre_match+"/"], "./#{$1}"
1321     end
1322     @libPath += $dependencies
1323   end
1324   
1325   def canCopyIntoBenchPath
1326     if @libPath and @relativeBinPath
1327       true
1328     else
1329       false
1330     end
1331   end
1332   
1333   def addExtraEnv(key, val)
1334     @extraEnv[key] = val
1335     @@extraEnvSet[key] = true
1336   end
1337   
1338   def copyIntoBenchPath
1339     raise unless canCopyIntoBenchPath
1340     basename, filename = Benchfile.uniqueFilename("vm")
1341     raise unless Dir.mkdir(filename)
1342     @libPath.each {
1343       | libPathPart |
1344       cmd = "cp -a #{Shellwords.shellescape(libPathPart)}/* #{Shellwords.shellescape(filename.to_s)}"
1345       $stderr.puts ">> #{cmd}" if $verbosity>=2
1346       raise unless system(cmd)
1347     }
1348     @path = "#{basename}/#{@relativeBinPath}"
1349     @libPath = [basename]
1350   end
1351   
1352   def to_s
1353     @name
1354   end
1355   
1356   def name
1357     @name
1358   end
1359   
1360   def shouldMeasureGC
1361     $measureGC == true or ($measureGC == name)
1362   end
1363   
1364   def origPath
1365     @origPath
1366   end
1367   
1368   def path
1369     @path
1370   end
1371   
1372   def nameKind
1373     @nameKind
1374   end
1375   
1376   def vmType
1377     @vmType
1378   end
1379   
1380   def checkoutPath
1381     @checkoutPath
1382   end
1383   
1384   def svnRevision
1385     @svnRevision
1386   end
1387   
1388   def extraEnv
1389     @extraEnv
1390   end
1391   
1392   def printFunction
1393     case @vmType
1394     when :jsc
1395       "print"
1396     when :dumpRenderTree, :webkitTestRunner
1397       "debug"
1398     else
1399       raise @vmType
1400     end
1401   end
1402   
1403   def emitRunCode(fileToRun, plan)
1404     myLibPath = @libPath
1405     myLibPath = [] unless myLibPath
1406     @@extraEnvSet.keys.each {
1407       | key |
1408       $script.puts "unset #{Shellwords.shellescape(key)}"
1409     }
1410     $script.puts "export DYLD_LIBRARY_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1411     $script.puts "export DYLD_FRAMEWORK_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1412     $script.puts "export LD_LIBRARY_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1413
1414     unless myLibPath.empty?
1415         primaryLibPath = myLibPath.first
1416         $script.puts "export TEST_RUNNER_TEST_PLUGIN_PATH=#{Shellwords.shellescape(Pathname.new(primaryLibPath).join('plugins').to_s)}"
1417         $script.puts "export TEST_RUNNER_INJECTED_BUNDLE_FILENAME=#{Shellwords.shellescape(Pathname.new(primaryLibPath).join('libTestRunnerInjectedBundle.so').to_s)}"
1418     end
1419
1420     @extraEnv.each_pair {
1421       | key, val |
1422       $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1423     }
1424     plan.environment.each_pair {
1425         | key, val |
1426         $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1427     }
1428     $script.puts "#{path} #{fileToRun} 2>&1 || {"
1429     $script.puts "    echo " + Shellwords.shellescape("#{name} failed to run!") + " 1>&2"
1430     $inner.times {
1431       | iteration |
1432       $script.puts "    echo " + Shellwords.shellescape("#{plan.prefix}: #{iteration}: CRASHED")
1433     }
1434     $script.puts "}"
1435     plan.environment.keys.each {
1436         | key |
1437         $script.puts "export #{Shellwords.shellescape(key)}="
1438     }
1439   end
1440 end
1441
1442 class StatsAccumulator
1443   def initialize
1444     @stats = []
1445     ($outer*$inner).times {
1446       @stats << Stats.new
1447     }
1448   end
1449   
1450   def statsForIteration(outerIteration, innerIteration)
1451     @stats[outerIteration*$inner + innerIteration]
1452   end
1453   
1454   def stats
1455     result = Stats.new
1456     @stats.each {
1457       | stat |
1458       if stat.ok?
1459         result.add(yield stat)
1460       else
1461         result.add(nil)
1462       end
1463     }
1464     result
1465   end
1466   
1467   def geometricMeanStats
1468     stats {
1469       | stat |
1470       stat.geometricMean
1471     }
1472   end
1473   
1474   def arithmeticMeanStats
1475     stats {
1476       | stat |
1477       stat.arithmeticMean
1478     }
1479   end
1480 end
1481
1482 module Benchmark
1483   attr_accessor :benchmarkSuite
1484   attr_reader :name
1485   
1486   def fullname
1487     benchmarkSuite.name + "/" + name
1488   end
1489   
1490   def to_s
1491     fullname
1492   end
1493   
1494   def weight
1495     1
1496   end
1497   
1498   def weightString
1499     raise unless weight.is_a? Fixnum
1500     raise unless weight >= 1
1501     if weight == 1
1502       ""
1503     else
1504       "x#{weight} "
1505     end
1506   end
1507 end
1508
1509 class SunSpiderBenchmark
1510   include Benchmark
1511   
1512   def initialize(name)
1513     @name = name
1514   end
1515   
1516   def emitRunCode(plan)
1517     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js")))
1518   end
1519 end
1520
1521 class LongSpiderBenchmark
1522   include Benchmark
1523   
1524   def initialize(name)
1525     @name = name
1526   end
1527   
1528   def emitRunCode(plan)
1529     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("LongSpider-#{@name}", "#{LONGSPIDER_PATH}/#{@name}.js")))
1530   end
1531 end
1532
1533 class V8Benchmark
1534   include Benchmark
1535   
1536   def initialize(name)
1537     @name = name
1538   end
1539   
1540   def emitRunCode(plan)
1541     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js")))
1542   end
1543 end
1544
1545 class V8RealBenchmark
1546   include Benchmark
1547   
1548   attr_reader :v8SuiteName
1549   
1550   def initialize(v8SuiteName, name, weight, minimumIterations)
1551     @v8SuiteName = v8SuiteName
1552     @name = name
1553     @weight = weight
1554     @minimumIterations = minimumIterations
1555   end
1556   
1557   def weight
1558     @weight
1559   end
1560   
1561   def emitRunCode(plan)
1562     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(["base", @v8SuiteName, "jsc-#{@name}"].collect{|v| ensureFile("V8Real-#{v}", "#{V8_REAL_PATH}/#{v}.js")}, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, false, @minimumIterations))
1563   end
1564 end
1565
1566 class OctaneBenchmark
1567   include Benchmark
1568   
1569   def initialize(files, name, weight, doWarmup, deterministic, minimumIterations)
1570     @files = files
1571     @name = name
1572     @weight = weight
1573     @doWarmup = doWarmup
1574     @deterministic = deterministic
1575     @minimumIterations = minimumIterations
1576   end
1577   
1578   def weight
1579     @weight
1580   end
1581   
1582   def emitRunCode(plan)
1583     files = []
1584     files += (["base"] + @files).collect {
1585       | v |
1586       ensureFile("Octane-#{v}", "#{OCTANE_PATH}/#{v}.js")
1587     }
1588     files += ["jsc-#{@name}"].collect {
1589       | v |
1590       ensureFile("Octane-#{v}", "#{OCTANE_WRAPPER_PATH}/#{v}.js")
1591     }
1592     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(files, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1593   end
1594 end
1595
1596 class KrakenBenchmark
1597   include Benchmark
1598   
1599   def initialize(name)
1600     @name = name
1601   end
1602   
1603   def emitRunCode(plan)
1604     emitBenchRunCode(fullname, plan, MultiFileTimedBenchmarkParameters.new([ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js")], loadCommandForFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js")))
1605   end
1606 end
1607
1608 class JSBenchBenchmark
1609   include Benchmark
1610   
1611   attr_reader :jsBenchMode
1612   
1613   def initialize(name, jsBenchMode)
1614     @name = name
1615     @jsBenchMode = jsBenchMode
1616   end
1617   
1618   def emitRunCode(plan)
1619     callbackDecl  = "function JSBNG_handleResult(result) {\n"
1620     callbackDecl += "    if (result.error) {\n"
1621     callbackDecl += "        console.log(\"Did not run benchmark correctly!\");\n"
1622     callbackDecl += "        quit();\n"
1623     callbackDecl += "    }\n"
1624     callbackDecl += "    reportResult(result.time);\n"
1625     callbackDecl += "}\n"
1626     emitBenchRunCode(fullname, plan, SingleFileTimedCallbackBenchmarkParameters.new(callbackDecl, ensureFile("JSBench-#{@name}", "#{JSBENCH_PATH}/#{@name}/#{@jsBenchMode}.js")))
1627   end
1628 end
1629
1630 class TailBenchBenchmark
1631   include Benchmark
1632
1633   def initialize(name)
1634     @name = name
1635   end
1636
1637   def emitRunCode(plan)
1638     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("TailBench-#{@name}", "#{TAILBENCH_PATH}/#{@name}.js")))
1639   end
1640 end
1641
1642 class MicrobenchmarksBenchmark
1643   include Benchmark
1644   
1645   def initialize(name)
1646     @name = name
1647   end
1648   
1649   def emitRunCode(plan)
1650     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("Microbenchmarks-#{@name}", "#{MICROBENCHMARKS_PATH}/#{@name}.js")))
1651   end
1652 end
1653
1654 class AsmBenchBenchmark
1655   include Benchmark
1656   
1657   def initialize(name)
1658     @name = name
1659   end
1660   
1661   def emitRunCode(plan)
1662     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("AsmBench-#{@name}", "#{ASMBENCH_PATH}/#{@name}.js")))
1663   end
1664 end
1665
1666 class SixSpeedBenchmark
1667   include Benchmark
1668
1669   def initialize(name, path, iterations)
1670     @name = name
1671     @path = path
1672     @iterations = iterations
1673   end
1674
1675   def emitRunCode(plan)
1676     emitBenchRunCode(fullname, plan, MultiFileTimedBenchmarkParameters.new([ensureFile("SixSpeed-#{@name}-wrapper", "#{SIXSPEED_WRAPPER_PATH}/wrapper.js"), ensureFile("SixSpeed-#{@name}", "#{SIXSPEED_PATH}/#{@path}")], simpleCommand("jscRun(#{@iterations});")))
1677   end
1678 end
1679
1680
1681 class CompressionBenchBenchmark
1682   include Benchmark
1683   
1684   def initialize(files, name, model)
1685     @files = files
1686     @name = name;
1687     @name = name + "-" + model if !model.empty?
1688     @name = @name.gsub(" ", "-").downcase
1689     @scriptName = name
1690     @weight = 1
1691     @doWarmup = true
1692     @deterministic = true
1693     @minimumIterations = 1
1694     @model = model
1695   end
1696   
1697   def weight
1698     @weight
1699   end
1700   
1701   def emitRunCode(plan)
1702     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new((["base"] + @files + ["jsc-#{@scriptName}"]).collect{|v| ensureFile("Compression-#{v}", "#{COMPRESSIONBENCH_PATH}/#{v}.js")}, simpleCommand("jscSetUp('#{@model}');"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1703   end
1704 end
1705
1706 class DSPJSFiltrrBenchmark
1707   include Benchmark
1708   
1709   def initialize(name, filterKey)
1710     @name = name
1711     @filterKey = filterKey
1712   end
1713   
1714   def emitRunCode(plan)
1715     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr.js")
1716     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr_back.jpg")
1717     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-jquery.min.js")
1718     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-bencher.html")
1719     emitSelfContainedBenchRunCode(fullname, plan, "filtrr-bencher.html", "bencher-config.js", @filterKey)
1720   end
1721 end
1722
1723 class DSPJSVP8Benchmark
1724   include Benchmark
1725   
1726   def initialize
1727     @name = "route9-vp8"
1728   end
1729   
1730   def weight
1731     5
1732   end
1733   
1734   def emitRunCode(plan)
1735     ensureBenchmarkFiles(DSPJS_ROUTE9_PATH)
1736     emitSelfContainedBenchRunCode(fullname, plan, "route9-bencher.html", "bencher-config.js", "")
1737   end
1738 end
1739
1740 class DSPStarfieldBenchmark
1741   include Benchmark
1742
1743   def initialize
1744     @name = "starfield"
1745   end
1746   
1747   def weight
1748     5
1749   end
1750
1751   def emitRunCode(plan)
1752     ensureBenchmarkFiles(DSPJS_STARFIELD_PATH)
1753     emitSelfContainedBenchRunCode(fullname, plan, "starfield-bencher.html", "bencher-config.js", "")
1754   end
1755 end
1756
1757 class DSPJSJSLinuxBenchmark
1758   include Benchmark
1759   def initialize
1760     @name = "bellard-jslinux"
1761   end
1762
1763   def weight
1764     5
1765   end
1766
1767   def emitRunCode(plan)
1768     ensureBenchmarkFiles(DSPJS_JSLINUX_PATH)
1769     emitSelfContainedBenchRunCode(fullname, plan, "jslinux-bencher.html", "bencher-config.js", "")
1770   end
1771 end
1772
1773 class DSPJSQuake3Benchmark
1774   include Benchmark
1775
1776   def initialize
1777     @name = "zynaps-quake3"
1778   end
1779
1780   def weight
1781     5
1782   end
1783
1784   def emitRunCode(plan)
1785     ensureBenchmarkFiles(DSPJS_QUAKE3_PATH)
1786     emitSelfContainedBenchRunCode(fullname, plan, "quake-bencher.html", "bencher-config.js", "")
1787   end
1788 end
1789
1790 class DSPJSMandelbrotBenchmark
1791   include Benchmark
1792
1793   def initialize
1794     @name = "zynaps-mandelbrot"
1795   end
1796
1797   def weight
1798     5
1799   end
1800
1801   def emitRunCode(plan)
1802     ensureBenchmarkFiles(DSPJS_MANDELBROT_PATH)
1803     emitSelfContainedBenchRunCode(fullname, plan, "mandelbrot-bencher.html", "bencher-config.js", "")
1804   end
1805 end
1806
1807 class DSPJSAmmoJSASMBenchmark
1808   include Benchmark
1809
1810   def initialize
1811     @name = "ammojs-asm-js"
1812   end
1813
1814   def weight
1815     5
1816   end
1817
1818   def emitRunCode(plan)
1819     ensureBenchmarkFiles(DSPJS_AMMOJS_ASMJS_PATH)
1820     emitSelfContainedBenchRunCode(fullname, plan, "ammo-asmjs-bencher.html", "bencher-config.js", "")
1821   end
1822 end
1823
1824 class DSPJSAmmoJSRegularBenchmark
1825   include Benchmark
1826
1827   def initialize
1828     @name = "ammojs-regular-js"
1829   end
1830
1831   def weight
1832     5
1833   end
1834
1835   def emitRunCode(plan)
1836     ensureBenchmarkFiles(DSPJS_AMMOJS_REGULAR_PATH)
1837     emitSelfContainedBenchRunCode(fullname, plan, "ammo-regular-bencher.html", "bencher-config.js", "")
1838   end
1839 end
1840
1841 class BrowsermarkJSBenchmark
1842   include Benchmark
1843     
1844   def initialize(name)
1845     @name = name
1846   end
1847   
1848   def emitRunCode(plan)
1849     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new([ensureFile(name, "#{BROWSERMARK_JS_PATH}/#{name}/test.js"), ensureFile("browsermark-bencher", "#{BROWSERMARK_JS_PATH}/browsermark-bencher.js")], simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, 32))
1850   end
1851 end
1852
1853 class BrowsermarkDOMBenchmark
1854   include Benchmark
1855     
1856   def initialize(name)
1857     @name = name
1858   end
1859   
1860   def emitRunCode(plan)
1861     ensureBenchmarkFiles(BROWSERMARK_PATH)
1862     emitSelfContainedBenchRunCode(fullname, plan, "tests/benchmarks/dom/#{name}/index.html", "bencher-config.js", name)
1863   end
1864 end
1865
1866 class BenchmarkSuite
1867   def initialize(name, preferredMean, decimalShift)
1868     @name = name
1869     @preferredMean = preferredMean
1870     @benchmarks = []
1871     @subSuites = []
1872     @decimalShift = decimalShift
1873   end
1874   
1875   def name
1876     @name
1877   end
1878   
1879   def to_s
1880     @name
1881   end
1882   
1883   def decimalShift
1884     @decimalShift
1885   end
1886   
1887   def addIgnoringPattern(benchmark)
1888     benchmark.benchmarkSuite = self
1889     @benchmarks << benchmark
1890   end
1891   
1892   def add(benchmark)
1893     if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern
1894         addIgnoringPattern(benchmark)
1895     end
1896   end
1897   
1898   def addSubSuite(subSuite)
1899     @subSuites << subSuite
1900   end
1901   
1902   def benchmarks
1903     @benchmarks
1904   end
1905   
1906   def benchmarkForName(name)
1907     result = @benchmarks.select{|v| v.name == name}
1908     raise unless result.length == 1
1909     result[0]
1910   end
1911   
1912   def hasBenchmark(benchmark)
1913     array = @benchmarks.select{|v| v == benchmark}
1914     raise unless array.length == 1 or array.length == 0
1915     array.length == 1
1916   end
1917   
1918   def subSuites
1919     @subSuites
1920   end
1921   
1922   def suites
1923     [self] + @subSuites
1924   end
1925   
1926   def suitesWithBenchmark(benchmark)
1927     result = [self]
1928     @subSuites.each {
1929       | subSuite |
1930       if subSuite.hasBenchmark(benchmark)
1931         result << subSuite
1932       end
1933     }
1934     result
1935   end
1936   
1937   def empty?
1938     @benchmarks.empty?
1939   end
1940   
1941   def retain_if
1942     @benchmarks.delete_if {
1943       | benchmark |
1944       not yield benchmark
1945     }
1946   end
1947   
1948   def preferredMean
1949     @preferredMean
1950   end
1951   
1952   def computeMean(stat)
1953     if stat.ok?
1954       (stat.send @preferredMean) * (10 ** decimalShift)
1955     else
1956       nil
1957     end
1958   end
1959 end
1960
1961 class BenchRunPlan
1962   def initialize(benchmark, vm, iteration)
1963     @benchmark = benchmark
1964     @vm = vm
1965     @iteration = iteration
1966     @environment = {}
1967     if $environment.has_key?(vm.name)
1968       if $environment[vm.name].has_key?(benchmark.benchmarkSuite.name)
1969         if $environment[vm.name][benchmark.benchmarkSuite.name].has_key?(benchmark.name)
1970           @environment = $environment[vm.name][benchmark.benchmarkSuite.name][benchmark.name]
1971         end
1972       end
1973     end
1974   end
1975   
1976   def benchmark
1977     @benchmark
1978   end
1979   
1980   def suite
1981     @benchmark.benchmarkSuite
1982   end
1983   
1984   def vm
1985     @vm
1986   end
1987   
1988   def iteration
1989     @iteration
1990   end
1991  
1992   def environment
1993     @environment
1994   end
1995  
1996   def prefix
1997     "#{@benchmark.fullname}: #{vm.name}: #{iteration}"
1998   end
1999   
2000   def emitRunCode
2001     @benchmark.emitRunCode(self)
2002   end
2003   
2004   def to_s
2005     benchmark.to_s + "/" + vm.to_s
2006   end
2007 end
2008
2009 class BenchmarkOnVM
2010   def initialize(benchmark, suiteOnVM, subSuitesOnVM)
2011     @benchmark = benchmark
2012     @suiteOnVM = suiteOnVM
2013     @subSuitesOnVM = subSuitesOnVM
2014     @stats = Stats.new
2015   end
2016   
2017   def to_s
2018     "#{@benchmark} on #{@suiteOnVM.vm}"
2019   end
2020   
2021   def benchmark
2022     @benchmark
2023   end
2024   
2025   def vm
2026     @suiteOnVM.vm
2027   end
2028   
2029   def vmStats
2030     @suiteOnVM.vmStats
2031   end
2032   
2033   def suite
2034     @benchmark.benchmarkSuite
2035   end
2036   
2037   def suiteOnVM
2038     @suiteOnVM
2039   end
2040   
2041   def subSuitesOnVM
2042     @subSuitesOnVM
2043   end
2044   
2045   def stats
2046     @stats
2047   end
2048   
2049   def parseResult(result)
2050     raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm
2051     raise unless result.benchmark == @benchmark
2052     @stats.add(result.time)
2053   end
2054 end
2055
2056 class NamedStatsAccumulator < StatsAccumulator
2057   def initialize(name)
2058     super()
2059     @name = name
2060   end
2061   
2062   def reportingName
2063     @name
2064   end
2065 end
2066
2067 class SuiteOnVM < StatsAccumulator
2068   def initialize(vm, vmStats, suite)
2069     super()
2070     @vm = vm
2071     @vmStats = vmStats
2072     @suite = suite
2073     
2074     raise unless @vm.is_a? VM
2075     raise unless @vmStats.is_a? StatsAccumulator
2076     raise unless @suite.is_a? BenchmarkSuite
2077   end
2078   
2079   def to_s
2080     "#{@suite} on #{@vm}"
2081   end
2082   
2083   def suite
2084     @suite
2085   end
2086   
2087   def vm
2088     @vm
2089   end
2090
2091   def reportingName
2092     @vm.name
2093   end
2094   
2095   def vmStats
2096     raise unless @vmStats
2097     @vmStats
2098   end
2099 end
2100
2101 class SubSuiteOnVM < StatsAccumulator
2102   def initialize(vm, suite)
2103     super()
2104     @vm = vm
2105     @suite = suite
2106     raise unless @vm.is_a? VM
2107     raise unless @suite.is_a? BenchmarkSuite
2108   end
2109   
2110   def to_s
2111     "#{@suite} on #{@vm}"
2112   end
2113   
2114   def suite
2115     @suite
2116   end
2117   
2118   def vm
2119     @vm
2120   end
2121   
2122   def reportingName
2123     @vm.name
2124   end
2125 end
2126
2127 class BenchPlan
2128   def initialize(benchmarkOnVM, iteration)
2129     @benchmarkOnVM = benchmarkOnVM
2130     @iteration = iteration
2131   end
2132   
2133   def to_s
2134     "#{@benchmarkOnVM} \##{@iteration+1}"
2135   end
2136   
2137   def benchmarkOnVM
2138     @benchmarkOnVM
2139   end
2140   
2141   def benchmark
2142     @benchmarkOnVM.benchmark
2143   end
2144   
2145   def suite
2146     @benchmarkOnVM.suite
2147   end
2148   
2149   def vm
2150     @benchmarkOnVM.vm
2151   end
2152   
2153   def iteration
2154     @iteration
2155   end
2156   
2157   def parseResult(result)
2158     raise unless result.plan == self
2159     @benchmarkOnVM.parseResult(result)
2160     benchmark.weight.times {
2161       @benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time)
2162       @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time)
2163       @benchmarkOnVM.subSuitesOnVM.each {
2164         | subSuite |
2165         subSuite.statsForIteration(@iteration, result.innerIndex).add(result.time)
2166       }
2167     }
2168   end
2169 end
2170
2171 def lpad(str,chars)
2172   if str.length>chars
2173     str
2174   else
2175     "%#{chars}s"%(str)
2176   end
2177 end
2178
2179 def rpad(str,chars)
2180   while str.length < chars
2181     str+=" "
2182   end
2183   str
2184 end
2185
2186 def center(str,chars)
2187   while str.length<chars
2188     str+=" "
2189     if str.length<chars
2190       str=" "+str
2191     end
2192   end
2193   str
2194 end
2195
2196 def statsToStr(stats, decimalShift)
2197   if stats.error?
2198     lpad(center("ERROR", 10+10+2), 12+10+2)
2199   elsif $inner*$outer == 1
2200     string = numToStr(stats.mean, decimalShift)
2201     raise unless string =~ /\./
2202     left = $~.pre_match
2203     right = $~.post_match
2204     lpad(left, 13 - decimalShift) + "." + rpad(right, 10 + decimalShift)
2205   else
2206     lpad(numToStr(stats.mean, decimalShift), 12) + "+-" + rpad(numToStr(stats.confInt, decimalShift), 10)
2207   end
2208 end
2209
2210 def plural(num)
2211   if num == 1
2212     ""
2213   else
2214     "s"
2215   end
2216 end
2217
2218 def wrap(str, columns)
2219   array = str.split
2220   result = ""
2221   curLine = array.shift
2222   array.each {
2223     | curStr |
2224     if (curLine + " " + curStr).size > columns
2225       result += curLine + "\n"
2226       curLine = curStr
2227     else
2228       curLine += " " + curStr
2229     end
2230   }
2231   result + curLine + "\n"
2232 end
2233   
2234 def runAndGetResults
2235   results = nil
2236   Dir.chdir(BENCH_DATA_PATH) {
2237     $stderr.puts ">> sh ./runscript" if $verbosity >= 2
2238     raise "Script did not complete correctly: #{$?}" unless system("sh ./runscript > runlog")
2239     results = IO::read("runlog")
2240   }
2241   raise unless results
2242   results
2243 end
2244
2245 def parseAndDisplayResults(results)
2246   vmStatses = []
2247   $vms.each {
2248     | vm |
2249     vmStatses << NamedStatsAccumulator.new(vm.name)
2250   }
2251   
2252   suitesOnVMs = []
2253   suitesOnVMsForSuite = {}
2254   subSuitesOnVMsForSubSuite = {}
2255   $suites.each {
2256     | suite |
2257     suitesOnVMsForSuite[suite] = []
2258     suite.subSuites.each {
2259       | subSuite |
2260       subSuitesOnVMsForSubSuite[subSuite] = []
2261     }
2262   }
2263   suitesOnVMsForVM = {}
2264   $vms.each {
2265     | vm |
2266     suitesOnVMsForVM[vm] = []
2267   }
2268   
2269   benchmarksOnVMs = []
2270   benchmarksOnVMsForBenchmark = {}
2271   $benchmarks.each {
2272     | benchmark |
2273     benchmarksOnVMsForBenchmark[benchmark] = []
2274   }
2275   
2276   $vms.each_with_index {
2277     | vm, vmIndex |
2278     vmStats = vmStatses[vmIndex]
2279     $suites.each {
2280       | suite |
2281       suiteOnVM = SuiteOnVM.new(vm, vmStats, suite)
2282       subSuitesOnVM = suite.subSuites.map {
2283         | subSuite |
2284         result = SubSuiteOnVM.new(vm, subSuite)
2285         subSuitesOnVMsForSubSuite[subSuite] << result
2286         result
2287       }
2288       suitesOnVMs << suiteOnVM
2289       suitesOnVMsForSuite[suite] << suiteOnVM
2290       suitesOnVMsForVM[vm] << suiteOnVM
2291       suite.benchmarks.each {
2292         | benchmark |
2293         subSuitesOnVMForThisBenchmark = []
2294         subSuitesOnVM.each {
2295           | subSuiteOnVM |
2296           if subSuiteOnVM.suite.hasBenchmark(benchmark)
2297             subSuitesOnVMForThisBenchmark << subSuiteOnVM
2298           end
2299         }
2300         benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM, subSuitesOnVMForThisBenchmark)
2301         benchmarksOnVMs << benchmarkOnVM
2302         benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
2303       }
2304     }
2305   }
2306   
2307   plans = []
2308   benchmarksOnVMs.each {
2309     | benchmarkOnVM |
2310     $outer.times {
2311       | iteration |
2312       plans << BenchPlan.new(benchmarkOnVM, iteration)
2313     }
2314   }
2315
2316   hostname = nil
2317   hwmodel = nil
2318   results.each_line {
2319     | line |
2320     line.chomp!
2321     if line =~ /HOSTNAME:([^.]+)/
2322       hostname = $1
2323     elsif line =~ /HARDWARE:hw\.model: /
2324       hwmodel = $~.post_match.chomp
2325     else
2326       result = ParsedResult.parse(plans, line)
2327       if result
2328         result.plan.parseResult(result)
2329       end
2330     end
2331   }
2332   
2333   # Compute the geomean of the preferred means of results on a SuiteOnVM
2334   overallResults = []
2335   $vms.each {
2336     | vm |
2337     result = Stats.new
2338     $outer.times {
2339       | outerIndex |
2340       $inner.times {
2341         | innerIndex |
2342         curResult = Stats.new
2343         suitesOnVMsForVM[vm].each {
2344           | suiteOnVM |
2345           # For a given iteration, suite, and VM, compute the suite's preferred mean
2346           # over the data collected for all benchmarks in that suite. We'll have one
2347           # sample per benchmark. For example on V8 this will be the geomean of 1
2348           # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
2349           # splay.
2350           curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
2351         }
2352         
2353         # curResult now holds 1 sample for each of the means computed in the above
2354         # loop. Compute the geomean over this, and store it.
2355         if curResult.ok?
2356           result.add(curResult.geometricMean)
2357         else
2358           result.add(nil)
2359         end
2360       }
2361     }
2362
2363     # $overallResults will have a Stats for each VM. That Stats object will hold
2364     # $inner*$outer geomeans, allowing us to compute the arithmetic mean and
2365     # confidence interval of the geomeans of preferred means. Convoluted, but
2366     # useful and probably sound.
2367     overallResults << result
2368   }
2369   
2370   if $verbosity >= 2
2371     benchmarksOnVMs.each {
2372       | benchmarkOnVM |
2373       $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
2374     }
2375     
2376     $vms.each_with_index {
2377       | vm, vmIndex |
2378       vmStats = vmStatses[vmIndex]
2379       $stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}"
2380       $stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}"
2381     }
2382   end
2383
2384   if $outputName
2385     reportName = $outputName
2386   else
2387     reportName =
2388       (if ($vms.collect {
2389              | vm |
2390              vm.nameKind
2391            }.index :auto)
2392          ""
2393        else
2394          text = $vms.collect {
2395            | vm |
2396            vm.to_s
2397          }.join("_") + "_"
2398          if text.size >= 40
2399            ""
2400          else
2401            text
2402          end
2403        end) +
2404       ($suites.collect {
2405          | suite |
2406          suite.to_s
2407        }.join("")) + "_" +
2408       (if hostname
2409          hostname + "_"
2410        else
2411          ""
2412        end)+
2413       (begin
2414          time = Time.now
2415          "%04d%02d%02d_%02d%02d" %
2416            [ time.year, time.month, time.day,
2417              time.hour, time.min ]
2418        end)
2419   end
2420
2421   unless $brief
2422     puts "Generating benchmark report at #{Dir.pwd}/#{reportName}_report.txt"
2423     puts "And raw data at #{Dir.pwd}/#{reportName}.json"
2424   end
2425   
2426   outp = $stdout
2427   json = {}
2428   begin
2429     outp = File.open(reportName + "_report.txt","w")
2430   rescue => e
2431     $stderr.puts "Error: could not save report to #{reportName}_report.txt: #{e}"
2432     $stderr.puts
2433   end
2434   
2435   def createVMsString
2436     result = ""
2437     result += "   " if $allSuites.size > 1
2438     result += rpad("", $benchpad + $weightpad)
2439     result += " "
2440     $vms.size.times {
2441       | index |
2442       if index != 0
2443         result += " "+NoChange.new(0).shortForm
2444       end
2445       result += lpad(center($vms[index].name, 10+10+2), 12+10+2)
2446     }
2447     result += "    "
2448     if $vms.size >= 3
2449       result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
2450     elsif $vms.size >= 2
2451       result += " "*26
2452     end
2453     result
2454   end
2455   
2456   def andJoin(list)
2457     if list.size == 1
2458       list[0].to_s
2459     elsif list.size == 2
2460       "#{list[0]} and #{list[1]}"
2461     else
2462       "#{list[0..-2].join(', ')}, and #{list[-1]}"
2463     end
2464   end
2465   
2466   json["vms"] = $vms.collect{|v| v.name}
2467   json["suites"] = {}
2468   json["runlog"] = results
2469   
2470   columns = [createVMsString.size, 78].max
2471   
2472   outp.print "Benchmark report for "
2473   outp.print andJoin($suites)
2474   if hostname
2475     outp.print " on #{hostname}"
2476   end
2477   if hwmodel
2478     outp.print " (#{hwmodel})"
2479   end
2480   outp.puts "."
2481   outp.puts
2482   
2483   outp.puts "VMs tested:"
2484   $vms.each {
2485     | vm |
2486     outp.print "\"#{vm.name}\" at #{vm.origPath}"
2487     if vm.svnRevision
2488       outp.print " (r#{vm.svnRevision})"
2489     end
2490     outp.puts
2491     vm.extraEnv.each_pair {
2492       | key, val |
2493       outp.puts "    export #{key}=#{val}"
2494     }
2495   }
2496   
2497   outp.puts
2498   
2499   outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
2500                  "with #{$outer} VM invocation#{plural($outer)} per benchmark."+
2501                  (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
2502                                       "total time of those iterations, for each sample.")
2503                   else "" end)+
2504                  (if $measureGC == true then (" No manual garbage collection invocations were "+
2505                                               "emitted.")
2506                   elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
2507                                          "all VMs except #{$measureGC}.")
2508                   else (" Emitted a call to gc() between sample measurements.") end)+
2509                  (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
2510                                         "began with the very first iteration.")
2511                   else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
2512                         "invocation for warm-up.") end)+
2513                  (case $timeMode
2514                   when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
2515                                           "microsecond-level timing.")
2516                   when :date then (" Used the portable Date.now() method to get millisecond-"+
2517                                    "level timing.")
2518                   else raise end)+
2519                  " Reporting benchmark execution times with 95% confidence "+
2520                  "intervals in milliseconds.",
2521                  columns)
2522   
2523   outp.puts
2524   
2525   def printVMs(outp)
2526     outp.puts createVMsString
2527   end
2528   
2529   def summaryStats(outp, json, accumulators, name, decimalShift, &proc)
2530     resultingJson = {}
2531     outp.print "   " if $allSuites.size > 1
2532     outp.print rpad(name, $benchpad + $weightpad)
2533     outp.print " "
2534     accumulators.size.times {
2535       | index |
2536       if index != 0
2537         outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
2538       end
2539       outp.print statsToStr(accumulators[index].stats(&proc), decimalShift)
2540       resultingJson[accumulators[index].reportingName] = accumulators[index].stats(&proc).jsonMap
2541     }
2542     if accumulators.size>=2
2543       outp.print("    "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm)
2544     end
2545     outp.puts
2546     json[name] = resultingJson
2547   end
2548
2549   def allSummaryStats(outp, json, accumulators, preferredMean, decimalShift)
2550     meanLabel = '<' + preferredMean.to_s.sub(/Mean$/, '') + '>'
2551     summaryStats(outp, json, accumulators, meanLabel, decimalShift) {
2552       | stat |
2553       stat.send(preferredMean)
2554     }
2555   end
2556   
2557   $suites.each {
2558     | suite |
2559     suiteJson = {}
2560     subSuiteJsons = {}
2561     suite.subSuites.each {
2562       | subSuite |
2563       subSuiteJsons[subSuite] = {}
2564     }
2565     
2566     printVMs(outp)
2567     if $allSuites.size > 1
2568       outp.puts(andJoin(suite.suites.map{|v| v.name}) + ":")
2569     else
2570       outp.puts
2571     end
2572     suite.benchmarks.each {
2573       | benchmark |
2574       benchmarkJson = {}
2575       outp.print "   " if $allSuites.size > 1
2576       outp.print rpad(benchmark.name, $benchpad) + rpad(benchmark.weightString, $weightpad)
2577       if benchmark.name.size > $benchNameClip
2578         outp.puts
2579         outp.print "   " if $allSuites.size > 1
2580         outp.print((" " * $benchpad) + (" " * $weightpad))
2581       end
2582       outp.print " "
2583       myConfigs = benchmarksOnVMsForBenchmark[benchmark]
2584       myConfigs.size.times {
2585         | index |
2586         if index != 0
2587           outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
2588         end
2589         outp.print statsToStr(myConfigs[index].stats, suite.decimalShift)
2590         benchmarkJson[myConfigs[index].vm.name] = myConfigs[index].stats.jsonMap
2591       }
2592       if $vms.size>=2
2593         outp.print("    "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
2594       end
2595       outp.puts
2596       suiteJson[benchmark.name] = benchmarkJson
2597       suite.subSuites.each {
2598         | subSuite |
2599         if subSuite.hasBenchmark(benchmark)
2600           subSuiteJsons[subSuite][benchmark.name] = benchmarkJson
2601         end
2602       }
2603     }
2604     outp.puts
2605     unless suite.subSuites.empty?
2606       suite.subSuites.each {
2607         | subSuite |
2608         outp.puts "#{subSuite.name}:"
2609         allSummaryStats(outp, subSuiteJsons[subSuite], subSuitesOnVMsForSubSuite[subSuite], subSuite.preferredMean, subSuite.decimalShift)
2610         outp.puts
2611       }
2612       outp.puts "#{suite.name} including #{andJoin(suite.subSuites.map{|v| v.name})}:"
2613     end
2614     allSummaryStats(outp, suiteJson, suitesOnVMsForSuite[suite], suite.preferredMean, suite.decimalShift)
2615     outp.puts if $allSuites.size > 1
2616     
2617     json["suites"][suite.name] = suiteJson
2618     suite.subSuites.each {
2619       | subSuite |
2620       json["suites"][subSuite.name] = subSuiteJsons[subSuite]
2621     }
2622   }
2623   
2624   if $suites.size > 1
2625     scaledResultJson = {}
2626     printVMs(outp)
2627     outp.puts "Geomean of preferred means:"
2628     outp.print "   "
2629     outp.print rpad("<scaled-result>", $benchpad + $weightpad)
2630     outp.print " "
2631     $vms.size.times {
2632       | index |
2633       if index != 0
2634         outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm
2635       end
2636       outp.print statsToStr(overallResults[index], 0)
2637       scaledResultJson[$vms[index].name] = overallResults[index].jsonMap
2638     }
2639     if overallResults.size>=2
2640       outp.print("    "+overallResults[-1].compareTo(overallResults[0]).longForm)
2641     end
2642     outp.puts
2643     
2644     json["<scaled-result>"] = scaledResultJson
2645   end
2646   outp.puts
2647   
2648   if outp != $stdout
2649     outp.close
2650   end
2651   
2652   if outp != $stdout and not $brief
2653     puts
2654     File.open(reportName + "_report.txt") {
2655       | inp |
2656       puts inp.read
2657     }
2658   end
2659   
2660   if $brief
2661     puts(overallResults.collect{|stats| stats.mean}.join("\t"))
2662     puts(overallResults.collect{|stats| stats.confInt}.join("\t"))
2663   end
2664   
2665   File.open(reportName + ".json", "w") {
2666     | outp |
2667     outp.puts json.to_json
2668   }
2669 end
2670
2671 begin
2672   $sawBenchOptions = false
2673   
2674   def resetBenchOptionsIfNecessary
2675     unless $sawBenchOptions
2676       $includeSunSpider = false
2677       $includeLongSpider = false
2678       $includeV8 = false
2679       $includeKraken = false
2680       $includeJSBench = false
2681       $includeMicrobenchmarks = false
2682       $includeAsmBench = false
2683       $includeDSPJS = false
2684       $includeBrowsermarkJS = false
2685       $includeBrowsermarkDOM = false
2686       $includeOctane = false
2687       $includeCompressionBench = false
2688       $includeSixSpeed = false
2689       $includeTailBench = false;
2690       $sawBenchOptions = true
2691     end
2692   end
2693   
2694   GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
2695                  ['--inner', GetoptLong::REQUIRED_ARGUMENT],
2696                  ['--outer', GetoptLong::REQUIRED_ARGUMENT],
2697                  ['--warmup', GetoptLong::REQUIRED_ARGUMENT],
2698                  ['--no-ss-warmup', GetoptLong::NO_ARGUMENT],
2699                  ['--quantum', GetoptLong::REQUIRED_ARGUMENT],
2700                  ['--minimum', GetoptLong::REQUIRED_ARGUMENT],
2701                  ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
2702                  ['--sunspider', GetoptLong::NO_ARGUMENT],
2703                  ['--longspider', GetoptLong::NO_ARGUMENT],
2704                  ['--v8-spider', GetoptLong::NO_ARGUMENT],
2705                  ['--kraken', GetoptLong::NO_ARGUMENT],
2706                  ['--js-bench', GetoptLong::NO_ARGUMENT],
2707                  ['--microbenchmarks', GetoptLong::NO_ARGUMENT],
2708                  ['--asm-bench', GetoptLong::NO_ARGUMENT],
2709                  ['--dsp', GetoptLong::NO_ARGUMENT],
2710                  ['--browsermark-js', GetoptLong::NO_ARGUMENT],
2711                  ['--browsermark-dom', GetoptLong::NO_ARGUMENT],
2712                  ['--octane', GetoptLong::NO_ARGUMENT],
2713                  ['--compression-bench', GetoptLong::NO_ARGUMENT],
2714                  ['--six-speed', GetoptLong::NO_ARGUMENT],
2715                  ['--tail-bench', GetoptLong::NO_ARGUMENT],
2716                  ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
2717                  ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
2718                  ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
2719                  ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
2720                  ['--dont-copy-vms', GetoptLong::NO_ARGUMENT],
2721                  ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
2722                  ['--brief', GetoptLong::NO_ARGUMENT],
2723                  ['--silent', GetoptLong::NO_ARGUMENT],
2724                  ['--remote', GetoptLong::REQUIRED_ARGUMENT],
2725                  ['--local', GetoptLong::NO_ARGUMENT],
2726                  ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
2727                  ['--slave', GetoptLong::NO_ARGUMENT],
2728                  ['--prepare-only', GetoptLong::NO_ARGUMENT],
2729                  ['--analyze', GetoptLong::REQUIRED_ARGUMENT],
2730                  ['--vms', GetoptLong::REQUIRED_ARGUMENT],
2731                  ['--output-name', GetoptLong::REQUIRED_ARGUMENT],
2732                  ['--environment', GetoptLong::REQUIRED_ARGUMENT],
2733                  ['--dependencies', GetoptLong::REQUIRED_ARGUMENT],
2734                  ['--config', GetoptLong::REQUIRED_ARGUMENT],
2735                  ['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
2736     | opt, arg |
2737     case opt
2738     when '--rerun'
2739       $rerun = intArg(opt,arg,1,nil)
2740     when '--inner'
2741       $inner = intArg(opt,arg,1,nil)
2742     when '--outer'
2743       $outer = intArg(opt,arg,1,nil)
2744     when '--warmup'
2745       $warmup = intArg(opt,arg,0,nil)
2746     when '--no-ss-warmup'
2747       $sunSpiderWarmup = false
2748     when '--quantum'
2749       $quantum = intArg(opt,arg,1,nil)
2750     when '--minimum'
2751       $minimum = intArg(opt,arg,1,nil)
2752     when '--timing-mode'
2753       if arg.upcase == "PRECISETIME"
2754         $timeMode = :preciseTime
2755       elsif arg.upcase == "DATE"
2756         $timeMode = :date
2757       elsif arg.upcase == "AUTO"
2758         $timeMode = :auto
2759       else
2760         quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
2761                   "Invalid argument for command-line option")
2762       end
2763     when '--force-vm-kind'
2764       if arg.upcase == "JSC"
2765         $forceVMKind = :jsc
2766       elsif arg.upcase == "DUMPRENDERTREE"
2767         $forceVMKind = :dumpRenderTree
2768       elsif arg.upcase == "WEBKITTESTRUNNER"
2769         $forceVMKind = :webkitTestRunner
2770       elsif arg.upcase == "AUTO"
2771         $forceVMKind = nil
2772       else
2773         quickFail("Expected 'jsc', 'DumpRenderTree', or 'WebKitTestRunner' for --force-vm-kind, but got '#{arg}'.",
2774                   "Invalid argument for command-line option")
2775       end
2776     when '--force-vm-copy'
2777       $needToCopyVMs = true
2778     when '--dont-copy-vms'
2779       $dontCopyVMs = true
2780     when '--sunspider'
2781       resetBenchOptionsIfNecessary
2782       $includeSunSpider = true
2783     when '--longspider'
2784       resetBenchOptionsIfNecessary
2785       $includeLongSpider = true
2786     when '--v8-spider'
2787       resetBenchOptionsIfNecessary
2788       $includeV8 = true
2789     when '--kraken'
2790       resetBenchOptionsIfNecessary
2791       $includeKraken = true
2792     when '--js-bench'
2793       resetBenchOptionsIfNecessary
2794       $includeJSBench = true
2795     when '--tail-bench'
2796       resetBenchOptionsIfNecessary
2797       $includeTailBench = true
2798     when '--microbenchmarks'
2799       resetBenchOptionsIfNecessary
2800       $includeMicrobenchmarks = true
2801     when '--asm-bench'
2802       resetBenchOptionsIfNecessary
2803       $includeAsmBench = true
2804     when '--dsp'
2805       resetBenchOptionsIfNecessary
2806       $includeDSPJS = true
2807     when '--browsermark-js'
2808       resetBenchOptionsIfNecessary
2809       $includeBrowsermarkJS = true
2810     when '--browsermark-dom'
2811       resetBenchOptionsIfNecessary
2812       $includeBrowsermarkDOM = true
2813     when '--octane'
2814       resetBenchOptionsIfNecessary
2815       $includeOctane = true
2816     when '--compression-bench'
2817       resetBenchOptionsIfNecessary
2818       $includeCompressionBench = true
2819     when '--six-speed'
2820       resetBenchOptionsIfNecessary
2821       $includeSixSpeed = true
2822     when '--benchmarks'
2823       $benchmarkPattern = Regexp.new(arg)
2824     when '--measure-gc'
2825       if arg == ''
2826         $measureGC = true
2827       else
2828         $measureGC = arg
2829       end
2830     when '--verbose'
2831       $verbosity += 1
2832     when '--brief'
2833       $brief = true
2834     when '--silent'
2835       $silent = true
2836     when '--remote'
2837       $remoteHosts += arg.split(',')
2838       $needToCopyVMs = true
2839     when '--ssh-options'
2840       $sshOptions << arg
2841     when '--local'
2842       $alsoLocal = true
2843     when '--prepare-only'
2844       $run = false
2845     when '--analyze'
2846       $prepare = false
2847       $run = false
2848       $analyze << arg
2849     when '--output-name'
2850       $outputName = arg
2851     when '--vms'
2852       JSON::parse(IO::read(arg)).each {
2853         | vmDescription |
2854         path = Pathname.new(vmDescription["path"]).realpath
2855         if vmDescription["name"]
2856           name = vmDescription["name"]
2857           nameKind = :given
2858         else
2859           name = "Conf\##{$vms.length+1}"
2860           nameKind = :auto
2861         end
2862         vm = VM.new(path, name, nameKind, nil)
2863         if vmDescription["env"]
2864           vmDescription["env"].each_pair {
2865             | key, val |
2866             vm.addExtraEnv(key, val)
2867           }
2868         end
2869         $vms << vm
2870       }
2871     when '--environment'
2872       $environment = JSON::parse(IO::read(arg))
2873     when '--dependencies'
2874       $dependencies.push(Pathname.new(arg).realpath)
2875     when '--config'
2876       $configPath = Pathname.new(arg)
2877     when '--help'
2878       usage
2879     else
2880       raise "bad option: #{opt}"
2881     end
2882   }
2883   
2884   # Figure out the configuration
2885   if $configPath.file?
2886     config = JSON::parse(IO::read($configPath.to_s))
2887   else
2888     config = {}
2889   end
2890
2891   def pathname_if_exist config
2892     if config
2893       Pathname.new config
2894     else
2895       config
2896     end
2897   end
2898
2899   OCTANE_PATH = pathname_if_exist(config["OctanePath"])
2900   BROWSERMARK_PATH = pathname_if_exist(config["BrowserMarkPath"])
2901   BROWSERMARK_JS_PATH = pathname_if_exist(config["BrowserMarkJSPath"])
2902   BROWSERMARK_DOM_PATH = pathname_if_exist(config["BrowserMarkDOMPath"])
2903   ASMBENCH_PATH = pathname_if_exist(config["AsmBenchPath"])
2904   COMPRESSIONBENCH_PATH = pathname_if_exist(config["CompressionBenchPath"])
2905   DSPJS_FILTRR_PATH = pathname_if_exist(config["DSPJSFiltrrPath"])
2906   DSPJS_ROUTE9_PATH = pathname_if_exist(config["DSPJSRoute9Path"])
2907   DSPJS_STARFIELD_PATH = pathname_if_exist(config["DSPJSStarfieldPath"])
2908   DSPJS_QUAKE3_PATH = pathname_if_exist(config["DSPJSQuake3Path"])
2909   DSPJS_MANDELBROT_PATH = pathname_if_exist(config["DSPJSMandelbrotPath"])
2910   DSPJS_JSLINUX_PATH = pathname_if_exist(config["DSPJSLinuxPath"])
2911   DSPJS_AMMOJS_ASMJS_PATH = pathname_if_exist(config["DSPJSAmmoJSAsmJSPath"])
2912   DSPJS_AMMOJS_REGULAR_PATH = pathname_if_exist(config["DSPJSAmmoJSRegularPath"])
2913   KRAKEN_PATH = pathname_if_exist(config["KrakenPath"])
2914   
2915   # If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option.
2916   if $dontCopyVMs
2917     $needToCopyVMs = false
2918   end
2919   
2920   ARGV.each {
2921     | vm |
2922     if vm =~ /([a-zA-Z0-9_ .]+):/
2923       name = $1
2924       nameKind = :given
2925       vm = $~.post_match
2926     else
2927       name = "Conf\##{$vms.length+1}"
2928       nameKind = :auto
2929     end
2930     envs = []
2931     while vm =~ /([a-zA-Z0-9_]+)=([a-zA-Z0-9_:.\/-]+):/
2932       envs << [$1, $2]
2933       vm = $~.post_match
2934     end
2935     $stderr.puts "#{name}: #{vm}" if $verbosity >= 1
2936     vm = VM.new(Pathname.new(vm).realpath, name, nameKind, nil)
2937     envs.each {
2938       | pair |
2939       vm.addExtraEnv(pair[0], pair[1])
2940     }
2941     $vms << vm
2942   }
2943   
2944   if $vms.empty?
2945     quickFail("Please specify at least on configuraiton on the command line.",
2946               "Insufficient arguments")
2947   end
2948   
2949   $vms.each {
2950     | vm |
2951     if vm.vmType == :jsc
2952       $allDRT = false
2953     end
2954   }
2955   
2956   SUNSPIDER = BenchmarkSuite.new("SunSpider", :arithmeticMean, 0)
2957   WARMUP = BenchmarkSuite.new("WARMUP", :arithmeticMean, 0)
2958   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
2959    "access-fannkuch", "access-nbody", "access-nsieve",
2960    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
2961    "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
2962    "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
2963    "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
2964    "string-base64", "string-fasta", "string-tagcloud",
2965    "string-unpack-code", "string-validate-input"].each {
2966     | name |
2967     SUNSPIDER.add SunSpiderBenchmark.new(name)
2968     WARMUP.addIgnoringPattern SunSpiderBenchmark.new(name)
2969   }
2970
2971   LONGSPIDER = BenchmarkSuite.new("LongSpider", :geometricMean, 0)
2972   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
2973    "access-fannkuch", "access-nbody", "access-nsieve",
2974    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-nsieve-bits",
2975    "controlflow-recursive", "crypto-aes", "crypto-md5", "crypto-sha1",
2976    "date-format-tofte", "date-format-xparb", "hash-map", "math-cordic",
2977    "math-partial-sums", "math-spectral-norm", "string-base64",
2978    "string-fasta", "string-tagcloud"].each {
2979     | name |
2980     LONGSPIDER.add LongSpiderBenchmark.new(name)
2981   }
2982
2983   V8 = BenchmarkSuite.new("V8Spider", :geometricMean, 0)
2984   ["crypto", "deltablue", "earley-boyer", "raytrace",
2985    "regexp", "richards", "splay"].each {
2986     | name |
2987     V8.add V8Benchmark.new(name)
2988   }
2989
2990   OCTANE = BenchmarkSuite.new("Octane", :geometricMean, 1)
2991   [[["crypto"], "encrypt", 1, true, false, 32],
2992    [["crypto"], "decrypt", 1, true, false, 32],
2993    [["deltablue"], "deltablue", 2, true, false, 32],
2994    [["earley-boyer"], "earley", 1, true, false, 32],
2995    [["earley-boyer"], "boyer", 1, true, false, 32],
2996    [["navier-stokes"], "navier-stokes", 2, true, false, 16],
2997    [["raytrace"], "raytrace", 2, true, false, 32],
2998    [["richards"], "richards", 2, true, false, 32],
2999    [["splay"], "splay", 2, true, false, 32],
3000    [["regexp"], "regexp", 2, true, false, 16],
3001    [["pdfjs"], "pdfjs", 2, false, false, 4],
3002    [["mandreel"], "mandreel", 2, false, false, 4],
3003    [["gbemu-part1", "gbemu-part2"], "gbemu", 2, false, false, 4],
3004    [["code-load"], "closure", 1, false, false, 16],
3005    [["code-load"], "jquery", 1, false, false, 16],
3006    [["box2d"], "box2d", 2, false, false, 8],
3007    [["zlib", "zlib-data"], "zlib", 2, false, true, 3],
3008    [["typescript", "typescript-input", "typescript-compiler"], "typescript", 2, false, true, 1]].each {
3009     | args |
3010     OCTANE.add OctaneBenchmark.new(*args)
3011   }
3012
3013   KRAKEN = BenchmarkSuite.new("Kraken", :arithmeticMean, -1)
3014   ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
3015    "audio-oscillator", "imaging-darkroom", "imaging-desaturate",
3016    "imaging-gaussian-blur", "json-parse-financial",
3017    "json-stringify-tinderbox", "stanford-crypto-aes",
3018    "stanford-crypto-ccm", "stanford-crypto-pbkdf2",
3019    "stanford-crypto-sha256-iterative"].each {
3020     | name |
3021     KRAKEN.add KrakenBenchmark.new(name)
3022   }
3023   
3024   JSBENCH = BenchmarkSuite.new("JSBench", :geometricMean, 0)
3025   [["amazon-chrome", "urem"], ["amazon-chrome-win", "urem"], ["amazon-firefox", "urm"],
3026    ["amazon-firefox-win", "urm"], ["amazon-safari", "urem"], ["facebook-chrome", "urem"],
3027    ["facebook-chrome-win", "urem"], ["facebook-firefox", "urem"],
3028    ["facebook-firefox-win", "urem"], ["facebook-safari", "urem"],
3029    ["google-chrome", "urem"], ["google-chrome-win", "urem"], ["google-firefox", "uem"],
3030    ["google-firefox-win", "urem"], ["google-safari", "urem"], ["twitter-chrome", "urem"],
3031    ["twitter-chrome-win", "rem"], ["twitter-firefox", "urem"],
3032    ["twitter-firefox-win", "urem"], ["twitter-safari", "urem"], ["yahoo-chrome", "urem"],
3033    ["yahoo-chrome-win", "urem"], ["yahoo-firefox", "urem"], ["yahoo-firefox-win", "urem"],
3034    ["yahoo-safari", "urem"]].each {
3035     | nameAndMode |
3036     JSBENCH.add JSBenchBenchmark.new(*nameAndMode)
3037   }
3038
3039   TAILBENCH = BenchmarkSuite.new("TailBench", :geometricMean, 0)
3040   ["n-body", "merge-sort"].each {
3041     | name |
3042     TAILBENCH.add TailBenchBenchmark.new(name);
3043   }
3044
3045   MICROBENCHMARKS = BenchmarkSuite.new("Microbenchmarks", :geometricMean, 0)
3046   Dir.foreach(MICROBENCHMARKS_PATH) {
3047     | filename |
3048     if filename =~ /\.js$/
3049       name = $~.pre_match
3050       MICROBENCHMARKS.add MicrobenchmarksBenchmark.new(name)
3051     end
3052   }
3053   
3054   ASMBENCH = BenchmarkSuite.new("AsmBench", :geometricMean, 0)
3055   if ASMBENCH_PATH
3056     Dir.foreach(ASMBENCH_PATH) {
3057       | filename |
3058       if filename =~ /\.js$/
3059         name = $~.pre_match
3060         ASMBENCH.add AsmBenchBenchmark.new(name)
3061       end
3062     }
3063   end
3064
3065   COMPRESSIONBENCH = BenchmarkSuite.new("CompressionBench", :geometricMean, 0)
3066   [[["huffman", "compression-data"], "huffman", ""],
3067    [["arithmetic", "compression-data"], "arithmetic", "Simple"],
3068    [["arithmetic", "compression-data"], "arithmetic", "Precise"],
3069    [["arithmetic", "compression-data"], "arithmetic", "Complex Precise"],
3070    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 0"],
3071    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 1"],
3072    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 2"],
3073    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 1"],
3074    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 2"],
3075    [["lz-string", "compression-data"], "lz-string", ""]
3076   ].each {
3077     | args |
3078     COMPRESSIONBENCH.add CompressionBenchBenchmark.new(*args)
3079   }
3080
3081   SIXSPEED = BenchmarkSuite.new("SixSpeed", :geometricMean, 0)
3082   [[ "template_string", 200000000 ],
3083    [ "defaults", 100000000 ],
3084    [ "map-set-lookup", 200000 ],
3085    [ "spread", 1000000 ],
3086    [ "object-assign", 600000 ],
3087    [ "spread-literal", 1000000 ],
3088    [ "map-set", 10000 ],
3089    [ "destructuring-simple", 20000000 ],
3090    [ "super", 3000000 ],
3091    [ "for-of-object", 1000000 ],
3092    [ "rest", 500000 ],
3093    [ "regex-u", 1000000 ],
3094    [ "arrow", 20000000 ],
3095    [ "bindings-compound", 20000000 ],
3096    [ "classes", 10000000 ],
3097    [ "template_string_tag", 2000000 ],
3098    [ "map-string", 30000000 ],
3099    [ "arrow-declare", 30000000 ],
3100    [ "spread-generator", 1000000 ],
3101    [ "object-literal-ext", 1000000 ],
3102    [ "generator", 3000000 ],
3103    [ "arrow-args", 20000000 ],
3104    [ "for-of-array", 5000000 ],
3105    [ "bindings", 20000000 ],
3106    [ "destructuring", 20000000 ],
3107    [ "map-set-object", 5000 ],
3108   ].each {
3109     | name, iterations |
3110     SIXSPEED.add SixSpeedBenchmark.new("#{name}.es5", "#{name}/#{name}.es5", iterations)
3111     SIXSPEED.add SixSpeedBenchmark.new("#{name}.es6", "#{name}/#{name}.es6", iterations)
3112   }
3113
3114   DSPJS = BenchmarkSuite.new("DSP", :geometricMean, 0)
3115   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-posterize-tint", "e2")
3116   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-contrast-sat-bright", "e5")
3117   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-sat-adj-contr-mult", "e7")
3118   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-blur-overlay-sat-contr", "e8")
3119   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sat-blur-mult-sharpen-contr", "e9")
3120   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sepia-bias", "e10")
3121   DSPJS.add DSPJSVP8Benchmark.new
3122   DSPJS.add DSPStarfieldBenchmark.new
3123   DSPJS.add DSPJSJSLinuxBenchmark.new
3124   DSPJS.add DSPJSQuake3Benchmark.new
3125   DSPJS.add DSPJSMandelbrotBenchmark.new
3126   DSPJS.add DSPJSAmmoJSASMBenchmark.new
3127   DSPJS.add DSPJSAmmoJSRegularBenchmark.new
3128
3129   BROWSERMARK_JS = BenchmarkSuite.new("BrowsermarkJS", :geometricMean, 1)
3130   ["array_blur", "array_weighted", "string_chat", "string_filter", "string_weighted"].each {
3131     | name |
3132       BROWSERMARK_JS.add BrowsermarkJSBenchmark.new(name)
3133   }
3134   
3135   BROWSERMARK_DOM = BenchmarkSuite.new("BrowsermarkDOM", :geometricMean, 1)
3136   ["advanced_search", "create_source", "dynamic_create", "search"].each {
3137     | name |
3138       BROWSERMARK_DOM.add BrowsermarkDOMBenchmark.new(name)
3139   }
3140
3141   $suites = []
3142   
3143   if $includeSunSpider and not SUNSPIDER.empty?
3144     $suites << SUNSPIDER
3145   end
3146   
3147   if $includeLongSpider and not LONGSPIDER.empty?
3148     $suites << LONGSPIDER
3149   end
3150   
3151   if $includeV8 and not V8.empty?
3152     $suites << V8
3153   end
3154   
3155   if $includeOctane and not OCTANE.empty?
3156     if OCTANE_PATH
3157       $suites << OCTANE
3158     else
3159       $stderr.puts "Warning: refusing to run Octane because \"OctanePath\" isn't set in #{$configPath}."
3160     end
3161   end
3162   
3163   if $includeKraken and not KRAKEN.empty?
3164     if KRAKEN_PATH
3165       $suites << KRAKEN
3166     else
3167       $stderr.puts "Warning: refusing to run Kraken because \"KrakenPath\" isn't set in #{$configPath}."
3168     end
3169   end
3170   
3171   if $includeJSBench and not JSBENCH.empty?
3172     if $allDRT
3173       if JSBENCH_PATH
3174         $suites << JSBENCH
3175       else
3176         $stderr.puts "Warning: refusing to run JSBench because \"JSBenchPath\" isn't set in #{$configPath}"
3177       end
3178     else
3179       $stderr.puts "Warning: refusing to run JSBench because not all VMs are DumpRenderTree or WebKitTestRunner."
3180     end
3181   end
3182   
3183   if $includeTailBench and not TAILBENCH.empty?
3184     $suites << TAILBENCH
3185   end
3186
3187   if $includeMicrobenchmarks and not MICROBENCHMARKS.empty?
3188     $suites << MICROBENCHMARKS
3189   end
3190
3191   if $includeAsmBench and not ASMBENCH.empty?
3192     if ASMBENCH_PATH
3193       $suites << ASMBENCH
3194     else
3195       $stderr.puts "Warning: refusing to run AsmBench because \"AsmBenchPath\" isn't set in #{$configPath}."
3196     end
3197   end
3198   
3199   if $includeDSPJS and not DSPJS.empty?
3200     if $allDRT
3201       if DSPJS_FILTRR_PATH and DSPJS_ROUTE9_PATH and DSPJS_STARFIELD_PATH and DSPJS_QUAKE3_PATH and DSPJS_MANDELBROT_PATH and DSPJS_JSLINUX_PATH and DSPJS_AMMOJS_ASMJS_PATH and DSPJS_AMMOJS_REGULAR_PATH
3202         $suites << DSPJS
3203       else
3204         $stderr.puts "Warning: refusing to run DSPJS because one of the following isn't set in #{$configPath}: \"DSPJSFiltrrPath\", \"DSPJSRoute9Path\", \"DSPJSStarfieldPath\", \"DSPJSQuake3Path\", \"DSPJSMandelbrotPath\", \"DSPJSLinuxPath\", \"DSPJSAmmoJSAsmJSPath\", \"DSPJSAmmoJSRegularPath\"."
3205       end
3206     else
3207       $stderr.puts "Warning: refusing to run DSPJS because not all VMs are DumpRenderTree or WebKitTestRunner."
3208     end
3209   end
3210   
3211   if $includeBrowsermarkJS and not BROWSERMARK_JS.empty?
3212     if BROWSERMARK_PATH and BROWSERMARK_JS_PATH
3213       $suites << BROWSERMARK_JS
3214     else
3215       $stderr.puts "Warning: refusing to run Browsermark-JS because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\" or \"BrowserMarkJSPath\"."
3216     end
3217   end
3218
3219   if $includeBrowsermarkDOM and not BROWSERMARK_DOM.empty?
3220     if $allDRT
3221       if BROWSERMARK_PATH and BROWSERMARK_JS_PATH and BROWSERMARK_DOM_PATH
3222         $suites << BROWSERMARK_DOM
3223       else
3224         $stderr.puts "Warning: refusing to run Browsermark-DOM because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\", \"BrowserMarkJSPath\", or \"BrowserMarkDOMPath\"."
3225       end
3226     else
3227       $stderr.puts "Warning: refusing to run Browsermark-DOM because not all VMs are DumpRenderTree or WebKitTestRunner."
3228     end
3229   end
3230
3231   if $includeCompressionBench and not COMPRESSIONBENCH.empty?
3232     if COMPRESSIONBENCH_PATH
3233       $suites << COMPRESSIONBENCH
3234     else
3235       $stderr.puts "Warning: refusing to run CompressionBench because \"CompressionBenchPath\" isn't set in #{$configPath}"
3236     end
3237   end
3238
3239   if $includeSixSpeed and not SIXSPEED.empty?
3240     if SIXSPEED_PATH
3241       $suites << SIXSPEED
3242     else
3243       $stderr.puts "Warning: refusing to run SixSpeed because \"SixSpeedPath\" isn't set in #{$configPath}."
3244     end
3245   end
3246
3247
3248   $allSuites = $suites.map{|v| v.suites}.flatten(1)
3249   
3250   $benchmarks = []
3251   $suites.each {
3252     | suite |
3253     $benchmarks += suite.benchmarks
3254   }
3255   
3256   if $suites.empty? or $benchmarks.empty?
3257     $stderr.puts "No benchmarks found.  Bailing out."
3258     exit 1
3259   end
3260   
3261   if $outer*$inner == 1
3262     $stderr.puts "Warning: will only collect one sample per benchmark/VM.  Confidence interval calculation will fail."
3263   end
3264   
3265   $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
3266   
3267   $runPlans = []
3268   $vms.each {
3269     | vm |
3270     $benchmarks.each {
3271       | benchmark |
3272       $outer.times {
3273         | iteration |
3274         $runPlans << BenchRunPlan.new(benchmark, vm, iteration)
3275       }
3276     }
3277   }
3278   
3279   $runPlans.shuffle!
3280   
3281   if $sunSpiderWarmup
3282     warmupPlans = []
3283     $vms.each {
3284       | vm |
3285       WARMUP.benchmarks.each {
3286         | benchmark |
3287         warmupPlans << BenchRunPlan.new(benchmark, vm, 0)
3288       }
3289     }
3290     
3291     $runPlans = warmupPlans.shuffle + $runPlans
3292   end
3293   
3294   $suitepad = $suites.collect {
3295     | suite |
3296     suite.to_s.size
3297   }.max + 1
3298   
3299   $planpad = $runPlans.collect {
3300     | plan |
3301     plan.to_s.size
3302   }.max + 1
3303   
3304   maxBenchNameLength =
3305     ($benchmarks + ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
3306     | benchmark |
3307     if benchmark.respond_to? :name
3308       benchmark.name.size
3309     else
3310       benchmark.size
3311     end
3312   }.max
3313   $benchNameClip = 40
3314   $benchpad = [maxBenchNameLength, $benchNameClip].min + 1
3315   
3316   $weightpad = $benchmarks.collect {
3317     | benchmark |
3318     benchmark.weightString.size
3319   }.max
3320
3321   $vmpad = $vms.collect {
3322     | vm |
3323     vm.to_s.size
3324   }.max + 1
3325   
3326   $analyze.each_with_index {
3327     | filename, index |
3328     if index >= 1
3329       puts
3330     end
3331     parseAndDisplayResults(IO::read(filename))
3332   }
3333   
3334   if not $prepare and not $run
3335     exit 0
3336   end
3337   
3338   if FileTest.exist? BENCH_DATA_PATH
3339     cmd = "rm -rf #{BENCH_DATA_PATH}"
3340     $stderr.puts ">> #{cmd}" if $verbosity >= 2
3341     raise unless system cmd
3342   end
3343   
3344   Dir.mkdir BENCH_DATA_PATH
3345   
3346   if $needToCopyVMs
3347     canCopyIntoBenchPath = true
3348     $vms.each {
3349       | vm |
3350       canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath
3351     }
3352     
3353     if canCopyIntoBenchPath
3354       $vms.each {
3355         | vm |
3356         $stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..."
3357         vm.copyIntoBenchPath
3358       }
3359       $stderr.puts "All VMs are in place."
3360     else
3361       $stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it."
3362     end
3363   end
3364   
3365   if $measureGC and $measureGC != true
3366     found = false
3367     $vms.each {
3368       | vm |
3369       if vm.name == $measureGC
3370         found = true
3371       end
3372     }
3373     unless found
3374       $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
3375     end
3376   end
3377   
3378   if $prepare
3379     File.open("#{BENCH_DATA_PATH}/runscript", "w") {
3380       | file |
3381       file.puts "echo -e \"HOSTNAME:\\c\""
3382       file.puts "hostname"
3383       file.puts "echo"
3384       file.puts "echo -e \"HARDWARE:\\c\""
3385       file.puts "/usr/sbin/sysctl hw.model"
3386       file.puts "echo"
3387       file.puts "set -e"
3388       $script = file
3389       $runPlans.each_with_index {
3390         | plan, idx |
3391         if $verbosity == 0 and not $silent
3392           text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s
3393           text2 = plan.to_s
3394           file.puts("echo -e " + Shellwords.shellescape("\r#{text1} #{rpad(text2,$planpad)}") + "\"\\c\" 1>&2")
3395           file.puts("echo -e " + Shellwords.shellescape("\r#{text1} #{text2}") + "\"\\c\" 1>&2")
3396         end
3397         plan.emitRunCode
3398       }
3399       if $verbosity == 0 and not $silent
3400         file.puts("echo -e " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}") + "\"\\c\" 1>&2")
3401         file.puts("echo -e " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size}") + " 1>&2")
3402       end
3403     }
3404   end
3405   
3406   if $run
3407     unless $remoteHosts.empty?
3408       $stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0
3409       Dir.chdir(TEMP_PATH) {
3410         cmd = "tar -czf payload.tar.gz benchdata"
3411         $stderr.puts ">> #{cmd}" if $verbosity>=2
3412         raise unless system(cmd)
3413       }
3414       
3415       def grokHost(host)
3416         if host =~ /:([0-9]+)$/
3417           "-p " + $1 + " " + Shellwords.shellescape($~.pre_match)
3418         else
3419           Shellwords.shellescape(host)
3420         end
3421       end
3422       
3423       def sshRead(host, command)
3424         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3425         $stderr.puts ">> #{cmd}" if $verbosity>=2
3426         result = ""
3427         IO.popen(cmd, "r") {
3428           | inp |
3429           inp.each_line {
3430             | line |
3431             $stderr.puts "#{host}: #{line}" if $verbosity>=2
3432             result += line
3433           }
3434         }
3435         raise "#{$?}" unless $?.success?
3436         result
3437       end
3438       
3439       def sshWrite(host, command, data)
3440         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3441         $stderr.puts ">> #{cmd}" if $verbosity>=2
3442         IO.popen(cmd, "w") {
3443           | outp |
3444           outp.write(data)
3445         }
3446         raise "#{$?}" unless $?.success?
3447       end
3448       
3449       $remoteHosts.each {
3450         | host |
3451         $stderr.puts "Sending benchmark payload to #{host}..." if $verbosity==0
3452         
3453         remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"]
3454         raise unless remoteTempPath
3455         
3456         sshWrite(host, "cd #{Shellwords.shellescape(remoteTempPath)} && rm -rf benchdata && tar -xz", IO::read("#{TEMP_PATH}/payload.tar.gz"))
3457         
3458         $stderr.puts "Running on #{host}..." if $verbosity==0
3459         
3460         parseAndDisplayResults(sshRead(host, "cd #{Shellwords.shellescape(remoteTempPath + '/benchdata')} && sh runscript"))
3461       }
3462     end
3463     
3464     if not $remoteHosts.empty? and $alsoLocal
3465       $stderr.puts "Running locally..."
3466     end
3467     
3468     if $remoteHosts.empty? or $alsoLocal
3469       parseAndDisplayResults(runAndGetResults)
3470     end
3471   end
3472   
3473   if $prepare and not $run and $analyze.empty?
3474     puts wrap("Benchmarking script and data are in #{BENCH_DATA_PATH}. You can run "+
3475               "the benchmarks and get the results by doing:", 78)
3476     puts
3477     puts "cd #{BENCH_DATA_PATH}"
3478     puts "sh runscript > results.txt"
3479     puts
3480     puts wrap("Then you can analyze the results by running bencher with the same arguments "+
3481               "as now, but replacing --prepare-only with --analyze results.txt.", 78)
3482   end
3483 rescue => e
3484   fail(e)
3485 end
3486