190e699dd1d2c20a291d340949e8cd382de18636
[WebKit-https.git] / Tools / Scripts / run-jsc-benchmarks
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2011-2015 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 # 1. Redistributions of source code must retain the above copyright
9 #    notice, this list of conditions and the following disclaimer.
10 # 2. Redistributions in binary form must reproduce the above copyright
11 #    notice, this list of conditions and the following disclaimer in the
12 #    documentation and/or other materials provided with the distribution.
13 #
14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24 # THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'rubygems'
27
28 require 'getoptlong'
29 require 'pathname'
30 require 'shellwords'
31 require 'socket'
32
33 begin
34   require 'json'
35 rescue LoadError => e
36   $stderr.puts "It does not appear that you have the 'json' package installed.  Try running 'sudo gem install json'."
37   exit 1
38 end
39
40 SCRIPT_PATH = Pathname.new(__FILE__).realpath
41
42 raise unless SCRIPT_PATH.dirname.basename.to_s == "Scripts"
43 raise unless SCRIPT_PATH.dirname.dirname.basename.to_s == "Tools"
44
45 OPENSOURCE_PATH = SCRIPT_PATH.dirname.dirname.dirname
46
47 SUNSPIDER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "SunSpider" + "tests" + "sunspider-1.0"
48 LONGSPIDER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "LongSpider"
49 V8_PATH = OPENSOURCE_PATH + "PerformanceTests" + "SunSpider" + "tests" + "v8-v6"
50 JSREGRESS_PATH = OPENSOURCE_PATH + "LayoutTests" + "js" + "regress" + "script-tests"
51 OCTANE_WRAPPER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "Octane" + "wrappers"
52
53 TEMP_PATH = OPENSOURCE_PATH + "BenchmarkTemp"
54
55 if TEMP_PATH.exist?
56   raise unless TEMP_PATH.directory?
57 else
58   Dir.mkdir(TEMP_PATH)
59 end
60
61 BENCH_DATA_PATH = TEMP_PATH + "benchdata"
62
63 IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933, 
64             0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684, 
65             0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227, 
66             0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674, 
67             0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652, 
68             0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604, 
69             0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338, 
70             0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538, 
71             0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185, 
72             0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694, 
73             0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244, 
74             0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067, 
75             0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323, 
76             0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129, 
77             0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574, 
78             0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722, 
79             0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624, 
80             0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321, 
81             0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843, 
82             0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217, 
83             0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464, 
84             0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599, 
85             0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638, 
86             0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592, 
87             0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471, 
88             0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283, 
89             0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037, 
90             0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738, 
91             0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481, 
92             0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085, 
93             0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651, 
94             0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183, 
95             0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683, 
96             0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154, 
97             0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599, 
98             0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019, 
99             0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417, 
100             0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795, 
101             0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153, 
102             0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494, 
103             0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818, 
104             0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128, 
105             0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423, 
106             0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704, 
107             0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974, 
108             0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232, 
109             0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479, 
110             0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716, 
111             0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943, 
112             0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192, 
113             0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401, 
114             0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963, 
115             0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822, 
116             0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007, 
117             0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186, 
118             0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358, 
119             0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525, 
120             0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686, 
121             0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841, 
122             0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991, 
123             0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137, 
124             0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278, 
125             0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414, 
126             0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547, 
127             0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675, 
128             0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799, 
129             0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192, 
130             0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037, 
131             0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151, 
132             0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262, 
133             0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369, 
134             0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489, 
135             0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259, 
136             0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689, 
137             0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785, 
138             0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879, 
139             0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297, 
140             0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059, 
141             0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145, 
142             0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242, 
143             0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324, 
144             0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404, 
145             0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483, 
146             0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559, 
147             0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634, 
148             0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707, 
149             0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778, 
150             0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848, 
151             0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916, 
152             0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983, 
153             0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057, 
154             0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121, 
155             0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183, 
156             0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244, 
157             0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304, 
158             0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363, 
159             0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442, 
160             0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476, 
161             0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539, 
162             0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946, 
163             0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653, 
164             0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712, 
165             0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762, 
166             0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812, 
167             0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867, 
168             0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915, 
169             0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962, 
170             0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007, 
171             0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059, 
172             0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103, 
173             0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146, 
174             0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189, 
175             0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231, 
176             0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272, 
177             0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312, 
178             0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352, 
179             0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391, 
180             0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429, 
181             0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467, 
182             0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504, 
183             0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546, 
184             0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582, 
185             0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617, 
186             0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652, 
187             0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686, 
188             0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724, 
189             0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757, 
190             0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579, 
191             0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822, 
192             0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853, 
193             0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885, 
194             0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915, 
195             0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595, 
196             0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979, 
197             0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013, 
198             0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041, 
199             0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074, 
200             0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102, 
201             0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129, 
202             0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156, 
203             0.99616, 0.996164]
204
205 # Run-time configuration parameters (can be set with command-line options)
206
207 $rerun=1
208 $inner=1
209 $warmup=1
210 $outer=4
211 $quantum=1000
212 $includeSunSpider=true
213 $includeLongSpider=true
214 $includeV8=true
215 $includeKraken=true
216 $includeJSBench=true
217 $includeJSRegress=true
218 $includeAsmBench=true
219 $includeDSPJS=true
220 $includeBrowsermarkJS=false
221 $includeBrowsermarkDOM=false
222 $includeOctane=true
223 $includeCompressionBench = true
224 $measureGC=false
225 $benchmarkPattern=nil
226 $verbosity=0
227 $timeMode=:preciseTime
228 $forceVMKind=nil
229 $brief=false
230 $silent=false
231 $remoteHosts=[]
232 $alsoLocal=false
233 $sshOptions=[]
234 $vms = []
235 $environment = {}
236 $needToCopyVMs = false
237 $dontCopyVMs = false
238 $allDRT = true
239 $outputName = nil
240 $sunSpiderWarmup = true
241 $configPath = Pathname.new(ENV["HOME"]) + ".run-jsc-benchmarks"
242
243 $prepare = true
244 $run = true
245 $analyze = []
246
247 # Helpful functions and classes
248
249 def smallUsage
250   puts "Use the --help option to get basic usage information."
251   exit 1
252 end
253
254 def usage
255   puts "run-jsc-benchmarks [options] <vm1> [<vm2> ...]"
256   puts
257   puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
258   puts "benchmarks, and reports detailed statistics.  What makes run-jsc-benchmarks"
259   puts "special is that each benchmark/VM configuration is run in a single VM invocation,"
260   puts "and the invocations are run in random order.  This minimizes systematics due to"
261   puts "one benchmark polluting the running time of another.  The fine-grained"
262   puts "interleaving of VM invocations further minimizes systematics due to changes in"
263   puts "the performance or behavior of your machine."
264   puts 
265   puts "Run-jsc-benchmarks is highly configurable.  You can compare as many VMs as you"
266   puts "like.  You can change the amount of warm-up iterations, number of iterations"
267   puts "executed per VM invocation, and the number of VM invocations per benchmark."
268   puts
269   puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
270   puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
271   puts "the executable and <name> is the name that you would like to give the"
272   puts "configuration for the purposeof reporting.  If no name is given, a generic name"
273   puts "of the form Conf#<n> will be ascribed to the configuration automatically."
274   puts
275   puts "It's also possible to specify per-VM environment variables. For example, you"
276   puts "might specify a VM like Foo:JSC_useJIT=false:/path/to/jsc, in which case the"
277   puts "harness will set the JSC_useJIT environment variable to false just before running"
278   puts "the given VM. Note that the harness will not unset the environment variable, so"
279   puts "you must ensure that your other VMs will use the opposite setting"
280   puts "(JSC_useJIT=true in this case)."
281   puts
282   puts "Options:"
283   puts "--rerun <n>          Set the number of iterations of the benchmark that"
284   puts "                     contribute to the measured run time.  Default is #{$rerun}."
285   puts "--inner <n>          Set the number of inner (per-runtime-invocation)"
286   puts "                     iterations.  Default is #{$inner}."
287   puts "--outer <n>          Set the number of runtime invocations for each benchmark."
288   puts "                     Default is #{$outer}."
289   puts "--warmup <n>         Set the number of warm-up runs per invocation.  Default"
290   puts "                     is #{$warmup}. This has a different effect on different kinds"
291   puts "                     benchmarks. Some benchmarks have no notion of warm-up."
292   puts "--no-ss-warmup       Disable SunSpider-based warm-up runs."
293   puts "--quantum <n>        Set the duration in milliseconds for which an iteration of"
294   puts "                     a throughput benchmark should be run.  Default is #{$quantum}."
295   puts "--timing-mode        Set the way that time is measured.  Possible values"
296   puts "                     are 'preciseTime' and 'date'.  Default is 'preciseTime'."
297   puts "--force-vm-kind      Turn off auto-detection of VM kind, and assume that it is"
298   puts "                     the one specified.  Valid arguments are 'jsc', "
299   puts "                     'DumpRenderTree', or 'WebKitTestRunner'."
300   puts "--force-vm-copy      Force VM builds to be copied to the working directory."
301   puts "                     This may reduce pathologies resulting from path names."
302   puts "--dont-copy-vms      Don't copy VMs even when doing a remote benchmarking run;"
303   puts "                     instead assume that they are already there."
304   puts "--sunspider          Only run SunSpider."
305   puts "--v8-spider          Only run V8."
306   puts "--kraken             Only run Kraken."
307   puts "--js-bench           Only run JSBench."
308   puts "--js-regress         Only run JSRegress."
309   puts "--dsp                Only run DSP."
310   puts "--asm-bench          Only run AsmBench."
311   puts "--browsermark-js     Only run browsermark-js."
312   puts "--browsermark-dom    Only run browsermark-dom."
313   puts "--octane             Only run Octane."
314   puts "--compression-bench  Only run compression bench"
315   puts "                     The default is to run all benchmarks. The above options can"
316   puts "                     be combined to run any subset (so --sunspider --dsp will run"
317   puts "                     both SunSpider and DSP)."
318   puts "--benchmarks         Only run benchmarks matching the given regular expression."
319   puts "--measure-gc         Turn off manual calls to gc(), so that GC time is measured."
320   puts "                     Works best with large values of --inner.  You can also say"
321   puts "                     --measure-gc <conf>, which turns this on for one"
322   puts "                     configuration only."
323   puts "--verbose or -v      Print more stuff."
324   puts "--brief              Print only the final result for each VM."
325   puts "--silent             Don't print progress. This might slightly reduce some"
326   puts "                     performance perturbation."
327   puts "--remote <sshhosts>  Perform performance measurements remotely, on the given"
328   puts "                     SSH host(s). Easiest way to use this is to specify the SSH"
329   puts "                     user@host string. However, you can also supply a comma-"
330   puts "                     separated list of SSH hosts. Alternatively, you can use this"
331   puts "                     option multiple times to specify multiple hosts. This"
332   puts "                     automatically copies the WebKit release builds of the VMs"
333   puts "                     you specified to all of the hosts."
334   puts "--ssh-options        Pass additional options to SSH."
335   puts "--local              Also do a local benchmark run even when doing --remote."
336   puts "--vms                Use a JSON file to specify which VMs to run, as opposed to"
337   puts "                     specifying them on the command line."
338   puts "--prepare-only       Only prepare the runscript (a shell script that"
339   puts "                     invokes the VMs to run benchmarks) but don't run it."
340   puts "--analyze            Only read the output of the runscript but don't do anything"
341   puts "                     else. This requires passing the same arguments that you"
342   puts "                     passed when running --prepare-only."
343   puts "--output-name        Base of the filenames to put results into. Will write a file"
344   puts "                     called <base>_report.txt and <base>.json.  By default this"
345   puts "                     name is automatically synthesized from the machine name,"
346   puts "                     date, set of benchmarks run, and set of configurations."
347   puts "--environment        JSON file that specifies the environment variables that should"
348   puts "                     be used for particular VMs and benchmarks."
349   puts "--config <path>      Specify the path of the configuration file. Defaults to"
350   puts "                     ~/.run-jsc-benchmarks"
351   puts "--help or -h         Display this message."
352   puts
353   puts "Example:"
354   puts "run-jsc-benchmarks TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
355   exit 1
356 end
357
358 def fail(reason)
359   if reason.respond_to? :backtrace
360     puts "FAILED: #{reason.inspect}"
361     puts "Stack trace:"
362     puts reason.backtrace.join("\n")
363   else
364     puts "FAILED: #{reason.inspect}"
365   end
366   smallUsage
367 end
368
369 def quickFail(r1,r2)
370   $stderr.puts "#{$0}: #{r1}"
371   puts
372   fail(r2)
373 end
374
375 def intArg(argName,arg,min,max)
376   result=arg.to_i
377   unless result.to_s == arg
378     quickFail("Expected an integer value for #{argName}, but got #{arg}.",
379               "Invalid argument for command-line option")
380   end
381   if min and result<min
382     quickFail("Argument for #{argName} cannot be smaller than #{min}.",
383               "Invalid argument for command-line option")
384   end
385   if max and result>max
386     quickFail("Argument for #{argName} cannot be greater than #{max}.",
387               "Invalid argument for command-line option")
388   end
389   result
390 end
391
392 def computeMean(array)
393   sum=0.0
394   array.each {
395     | value |
396     sum += value
397   }
398   sum/array.length
399 end
400
401 def computeGeometricMean(array)
402   sum = 0.0
403   array.each {
404     | value |
405     sum += Math.log(value)
406   }
407   Math.exp(sum * (1.0/array.length))
408 end
409
410 def computeHarmonicMean(array)
411   1.0 / computeMean(array.collect{ | value | 1.0 / value })
412 end
413
414 def computeStdDev(array)
415   case array.length
416   when 0
417     0.0/0.0
418   when 1
419     0.0
420   else
421     begin
422       mean=computeMean(array)
423       sum=0.0
424       array.each {
425         | value |
426         sum += (value-mean)**2
427       }
428       Math.sqrt(sum/(array.length-1))
429     rescue
430       0.0/0.0
431     end
432   end
433 end
434
435 class Array
436   def shuffle!
437     size.downto(1) { |n| push delete_at(rand(n)) }
438     self
439   end
440 end
441
442 def inverseBetaRegularized(n)
443   IBR_LOOKUP[n-1]
444 end
445
446 def numToStr(num, decimalShift)
447   ("%." + (4 + decimalShift).to_s + "f") % (num.to_f)
448 end
449
450 class CantSay
451   def initialize
452   end
453   
454   def shortForm
455     " "
456   end
457   
458   def longForm
459     ""
460   end
461   
462   def to_s
463     ""
464   end
465 end
466   
467 class NoChange
468   attr_reader :amountFaster
469   
470   def initialize(amountFaster)
471     @amountFaster = amountFaster
472   end
473   
474   def shortForm
475     " "
476   end
477   
478   def longForm
479     "  might be #{numToStr(@amountFaster, 0)}x faster"
480   end
481   
482   def to_s
483     if @amountFaster < 1.01
484       ""
485     else
486       longForm
487     end
488   end
489 end
490
491 class Faster
492   attr_reader :amountFaster
493   
494   def initialize(amountFaster)
495     @amountFaster = amountFaster
496   end
497   
498   def shortForm
499     "^"
500   end
501   
502   def longForm
503     "^ definitely #{numToStr(@amountFaster, 0)}x faster"
504   end
505   
506   def to_s
507     longForm
508   end
509 end
510
511 class Slower
512   attr_reader :amountSlower
513   
514   def initialize(amountSlower)
515     @amountSlower = amountSlower
516   end
517   
518   def shortForm
519     "!"
520   end
521   
522   def longForm
523     "! definitely #{numToStr(@amountSlower, 0)}x slower"
524   end
525   
526   def to_s
527     longForm
528   end
529 end
530
531 class MayBeSlower
532   attr_reader :amountSlower
533   
534   def initialize(amountSlower)
535     @amountSlower = amountSlower
536   end
537   
538   def shortForm
539     "?"
540   end
541   
542   def longForm
543     "? might be #{numToStr(@amountSlower, 0)}x slower"
544   end
545   
546   def to_s
547     if @amountSlower < 1.01
548       "?"
549     else
550       longForm
551     end
552   end
553 end
554
555 def jsonSanitize(value)
556   if value.is_a? Fixnum
557     value
558   elsif value.is_a? Float
559     if value.nan? or value.infinite?
560       value.to_s
561     else
562       value
563     end
564   elsif value.is_a? Array
565     value.map{|v| jsonSanitize(v)}
566   elsif value.nil?
567     value
568   else
569     raise "Unrecognized value #{value.inspect}"
570   end
571 end
572
573 class Stats
574   def initialize
575     @array = []
576   end
577   
578   def add(value)
579     if not value or not @array
580       @array = nil
581     elsif value.is_a? Float
582       if value.nan? or value.infinite?
583         @array = nil
584       else
585         @array << value
586       end
587     elsif value.is_a? Stats
588       add(value.array)
589     elsif value.respond_to? :each
590       value.each {
591         | v |
592         add(v)
593       }
594     else
595       @array << value.to_f
596     end
597   end
598   
599   def status
600     if @array
601       :ok
602     else
603       :error
604     end
605   end
606   
607   def error?
608     # TODO: We're probably still not handling this case correctly. 
609     not @array or @array.empty?
610   end
611   
612   def ok?
613     not not @array
614   end
615     
616   def array
617     @array
618   end
619   
620   def sum
621     result=0
622     @array.each {
623       | value |
624       result += value
625     }
626     result
627   end
628   
629   def min
630     @array.min
631   end
632   
633   def max
634     @array.max
635   end
636   
637   def size
638     @array.length
639   end
640   
641   def mean
642     computeMean(array)
643   end
644   
645   def arithmeticMean
646     mean
647   end
648   
649   def stdDev
650     computeStdDev(array)
651   end
652
653   def stdErr
654     stdDev/Math.sqrt(size)
655   end
656   
657   # Computes a 95% Student's t distribution confidence interval
658   def confInt
659     if size < 2
660       0.0/0.0
661     else
662       raise if size > 1000
663       Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
664     end
665   end
666   
667   def lower
668     mean-confInt
669   end
670   
671   def upper
672     mean+confInt
673   end
674   
675   def geometricMean
676     computeGeometricMean(array)
677   end
678   
679   def harmonicMean
680     computeHarmonicMean(array)
681   end
682   
683   def compareTo(other)
684     return CantSay.new unless ok? and other.ok?
685     
686     if upper < other.lower
687       Faster.new(other.mean/mean)
688     elsif lower > other.upper
689       Slower.new(mean/other.mean)
690     elsif mean > other.mean
691       MayBeSlower.new(mean/other.mean)
692     else
693       NoChange.new(other.mean/mean)
694     end
695   end
696   
697   def to_s
698     "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
699   end
700   
701   def jsonMap
702     if ok?
703       {"data"=>jsonSanitize(@array), "mean"=>jsonSanitize(mean), "confInt"=>jsonSanitize(confInt)}
704     else
705       "ERROR"
706     end
707   end
708 end
709
710 def doublePuts(out1,out2,msg)
711   out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
712   out2.puts msg
713 end
714
715 class Benchfile < File
716   @@counter = 0
717   
718   attr_reader :filename, :basename
719   
720   def initialize(name)
721     @basename, @filename = Benchfile.uniqueFilename(name)
722     super(@filename, "w")
723   end
724   
725   def self.uniqueFilename(name)
726     if name.is_a? Array
727       basename = name[0] + @@counter.to_s + name[1]
728     else
729       basename = name + @@counter.to_s
730     end
731     filename = BENCH_DATA_PATH + basename
732     @@counter += 1
733     raise "Benchfile #{filename} already exists" if FileTest.exist?(filename)
734     [basename, filename]
735   end
736   
737   def self.create(name)
738     file = Benchfile.new(name)
739     yield file
740     file.close
741     file.basename
742   end
743 end
744
745 $dataFiles={}
746 def ensureFile(key, filename)
747   unless $dataFiles[key]
748     $dataFiles[key] = Benchfile.create(key) {
749       | outp |
750       doublePuts($stderr,outp,IO::read(filename))
751     }
752   end
753   $dataFiles[key]
754 end
755
756 # Helper for files that cannot be renamed.
757 $absoluteFiles={}
758 def ensureAbsoluteFile(filename, basedir=nil)
759   return if $absoluteFiles[filename]
760   filename = Pathname.new(filename)
761
762   directory = Pathname.new('')
763   if basedir and filename.dirname != basedir
764     remainingPath = filename.dirname
765     while remainingPath != basedir
766       directory = remainingPath.basename + directory
767       remainingPath = remainingPath.dirname
768     end
769     if not $absoluteFiles[directory]
770       cmd = "mkdir -p #{Shellwords.shellescape((BENCH_DATA_PATH + directory).to_s)}"
771       $stderr.puts ">> #{cmd}" if $verbosity >= 2
772       raise unless system(cmd)
773       intermediateDirectory = Pathname.new(directory)
774       while intermediateDirectory.basename.to_s != "."
775         $absoluteFiles[intermediateDirectory] = true
776         intermediateDirectory = intermediateDirectory.dirname
777       end
778     end
779   end
780   
781   cmd = "cp #{Shellwords.shellescape(filename.to_s)} #{Shellwords.shellescape((BENCH_DATA_PATH + directory + filename.basename).to_s)}"
782   $stderr.puts ">> #{cmd}" if $verbosity >= 2
783   raise unless system(cmd)
784   $absoluteFiles[filename] = true
785 end
786
787 # Helper for large benchmarks with lots of files and directories.
788 def ensureBenchmarkFiles(rootdir)
789     toProcess = [rootdir]
790     while not toProcess.empty?
791       currdir = toProcess.pop
792       Dir.foreach(currdir.to_s) {
793         | filename |
794         path = currdir + filename
795         next if filename.match(/^\./)
796         toProcess.push(path) if File.directory?(path.to_s)
797         ensureAbsoluteFile(path, rootdir) if File.file?(path.to_s)
798       }
799     end
800 end
801
802 class JSCommand
803   attr_reader :js, :html
804   def initialize(js, html)
805     @js = js
806     @html = html
807   end
808 end
809
810 def loadCommandForFile(key, filename)
811   file = ensureFile(key, filename)
812   JSCommand.new("load(#{file.inspect});", "<script src=#{file.inspect}></script>")
813 end
814
815 def simpleCommand(command)
816   JSCommand.new(command, "<script type=\"text/javascript\">#{command}</script>")
817 end
818
819 # Benchmark that consists of a single file and must be loaded in its own global object each
820 # time (i.e. run()).
821 class SingleFileTimedBenchmarkParameters
822   attr_reader :benchPath
823   
824   def initialize(benchPath)
825     @benchPath = benchPath
826   end
827   
828   def kind
829     :singleFileTimedBenchmark
830   end
831 end
832
833 # Benchmark that consists of one or more data files that should be loaded globally, followed
834 # by a command to run the benchmark.
835 class MultiFileTimedBenchmarkParameters
836   attr_reader :dataPaths, :command
837
838   def initialize(dataPaths, command)
839     @dataPaths = dataPaths
840     @command = command
841   end
842   
843   def kind
844     :multiFileTimedBenchmark
845   end
846 end
847
848 # Benchmark that consists of one or more data files that should be loaded globally, followed
849 # by a command to run a short tick of the benchmark. The benchmark should be run for as many
850 # ticks as possible, for one quantum (quantum is 1000ms by default).
851 class ThroughputBenchmarkParameters
852   attr_reader :dataPaths, :setUpCommand, :command, :tearDownCommand, :doWarmup, :deterministic, :minimumIterations
853
854   def initialize(dataPaths, setUpCommand, command, tearDownCommand, doWarmup, deterministic, minimumIterations)
855     @dataPaths = dataPaths
856     @setUpCommand = setUpCommand
857     @command = command
858     @tearDownCommand = tearDownCommand
859     @doWarmup = doWarmup
860     @deterministic = deterministic
861     @minimumIterations = minimumIterations
862   end
863   
864   def kind
865     :throughputBenchmark
866   end
867 end
868
869 # Benchmark that can only run in DumpRenderTree or WebKitTestRunner, that has its own callback for reporting
870 # results. Other than that it's just like SingleFileTimedBenchmark.
871 class SingleFileTimedCallbackBenchmarkParameters
872   attr_reader :callbackDecl, :benchPath
873   
874   def initialize(callbackDecl, benchPath)
875     @callbackDecl = callbackDecl
876     @benchPath = benchPath
877   end
878   
879   def kind
880     :singleFileTimedCallbackBenchmark
881   end
882 end
883
884 def emitTimerFunctionCode(file)
885   case $timeMode
886   when :preciseTime
887     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
888     doublePuts($stderr,file,"   return preciseTime()*1000")
889     doublePuts($stderr,file,"}")
890   when :date
891     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
892     doublePuts($stderr,file,"   return Date.now()")
893     doublePuts($stderr,file,"}")
894   else
895     raise
896   end
897 end
898
899 def emitBenchRunCodeFile(name, plan, benchParams)
900   case plan.vm.vmType
901   when :jsc
902     Benchfile.create("bencher") {
903       | file |
904       emitTimerFunctionCode(file)
905       
906       if benchParams.kind == :multiFileTimedBenchmark
907         benchParams.dataPaths.each {
908           | path |
909           doublePuts($stderr,file,"load(#{path.inspect});")
910         }
911         doublePuts($stderr,file,"gc();")
912         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
913         doublePuts($stderr,file,"   var __before = __bencher_curTimeMS();")
914         $rerun.times {
915           doublePuts($stderr,file,"   #{benchParams.command.js}")
916         }
917         doublePuts($stderr,file,"   var __after = __bencher_curTimeMS();")
918         doublePuts($stderr,file,"   if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(__after-__before));");
919         doublePuts($stderr,file,"   gc();") unless plan.vm.shouldMeasureGC
920         doublePuts($stderr,file,"}")
921       elsif benchParams.kind == :throughputBenchmark
922         emitTimerFunctionCode(file)
923         benchParams.dataPaths.each {
924           | path |
925           doublePuts($stderr,file,"load(#{path.inspect});")
926         }
927         doublePuts($stderr,file,"#{benchParams.setUpCommand.js}")
928         if benchParams.doWarmup
929           warmup = $warmup
930         else
931           warmup = 0
932         end
933         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{warmup + $inner}; __bencher_index++) {")
934         doublePuts($stderr,file,"    var __before = __bencher_curTimeMS();")
935         doublePuts($stderr,file,"    var __after = __before;")
936         doublePuts($stderr,file,"    var __runs = 0;")
937         doublePuts($stderr,file,"    var __expected = #{$quantum};")
938         doublePuts($stderr,file,"    while (true) {")
939         $rerun.times {
940           doublePuts($stderr,file,"       #{benchParams.command.js}")
941         }
942         doublePuts($stderr,file,"       __runs++;")
943         doublePuts($stderr,file,"       __after = __bencher_curTimeMS();")
944         if benchParams.deterministic
945           doublePuts($stderr,file,"       if (true) {")
946         else
947           doublePuts($stderr,file,"       if (__after - __before >= __expected) {")
948         end
949         doublePuts($stderr,file,"           if (__runs >= #{benchParams.minimumIterations} || __bencher_index < #{warmup})")
950         doublePuts($stderr,file,"               break;")
951         doublePuts($stderr,file,"           __expected += #{$quantum}")
952         doublePuts($stderr,file,"       }")
953         doublePuts($stderr,file,"    }")
954         doublePuts($stderr,file,"    if (__bencher_index >= #{warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{warmup}) + \": Time: \"+((__after-__before)/__runs));")
955         doublePuts($stderr,file,"}")
956         doublePuts($stderr,file,"#{benchParams.tearDownCommand.js}")
957       else
958         raise unless benchParams.kind == :singleFileTimedBenchmark
959         doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
960         doublePuts($stderr,file,"   var __bencher_before = __bencher_curTimeMS();")
961         $rerun.times {
962           doublePuts($stderr,file,"   run(__bencher_what);")
963         }
964         doublePuts($stderr,file,"   var __bencher_after = __bencher_curTimeMS();")
965         doublePuts($stderr,file,"   return __bencher_after - __bencher_before;")
966         doublePuts($stderr,file,"}")
967         $warmup.times {
968           doublePuts($stderr,file,"__bencher_run(#{benchParams.benchPath.inspect})")
969           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
970         }
971         $inner.times {
972           | innerIndex |
973           doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchParams.benchPath.inspect}));")
974           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
975         }
976       end
977     }
978   when :dumpRenderTree, :webkitTestRunner
979     case $timeMode
980     when :preciseTime
981       curTime = "(testRunner.preciseTime()*1000)"
982     when :date
983       curTime = "(Date.now())"
984     else
985       raise
986     end
987
988     mainCode = Benchfile.create("bencher") {
989       | file |
990       doublePuts($stderr,file,"__bencher_count = 0;")
991       doublePuts($stderr,file,"function __bencher_doNext(result) {")
992       doublePuts($stderr,file,"    if (__bencher_count >= #{$warmup})")
993       doublePuts($stderr,file,"        debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);")
994       doublePuts($stderr,file,"    __bencher_count++;")
995       doublePuts($stderr,file,"    if (__bencher_count < #{$inner+$warmup})")
996       doublePuts($stderr,file,"        __bencher_runImpl(__bencher_doNext);")
997       doublePuts($stderr,file,"    else")
998       doublePuts($stderr,file,"        quit();")
999       doublePuts($stderr,file,"}")
1000       doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);")
1001     }
1002     
1003     cssCode = Benchfile.create("bencher-css") {
1004       | file |
1005       doublePuts($stderr,file,".pass {\n    font-weight: bold;\n    color: green;\n}\n.fail {\n    font-weight: bold;\n    color: red;\n}\n\#console {\n    white-space: pre-wrap;\n    font-family: monospace;\n}")
1006     }
1007     
1008     preCode = Benchfile.create("bencher-pre") {
1009       | file |
1010       doublePuts($stderr,file,"if (window.testRunner) {")
1011       doublePuts($stderr,file,"    testRunner.dumpAsText(window.enablePixelTesting);")
1012       doublePuts($stderr,file,"    testRunner.waitUntilDone();")
1013       doublePuts($stderr,file,"}")
1014       doublePuts($stderr,file,"")
1015       doublePuts($stderr,file,"function debug(msg)")
1016       doublePuts($stderr,file,"{")
1017       doublePuts($stderr,file,"    var span = document.createElement(\"span\");")
1018       doublePuts($stderr,file,"    document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace")
1019       doublePuts($stderr,file,"    span.innerHTML = msg + '<br />';")
1020       doublePuts($stderr,file,"}")
1021       doublePuts($stderr,file,"")
1022       doublePuts($stderr,file,"function quit() {")
1023       doublePuts($stderr,file,"    testRunner.notifyDone();")
1024       doublePuts($stderr,file,"}")
1025       doublePuts($stderr,file,"")
1026       doublePuts($stderr,file,"__bencher_continuation=null;")
1027       doublePuts($stderr,file,"")
1028       doublePuts($stderr,file,"function reportResult(result) {")
1029       doublePuts($stderr,file,"    __bencher_continuation(result);")
1030       doublePuts($stderr,file,"}")
1031       if benchParams.kind == :singleFileTimedCallbackBenchmark
1032         doublePuts($stderr,file,"")
1033         doublePuts($stderr,file,benchParams.callbackDecl)
1034       end
1035       doublePuts($stderr,file,"")
1036       doublePuts($stderr,file,"function __bencher_runImpl(continuation) {")
1037       doublePuts($stderr,file,"    function doit() {")
1038       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"\";")
1039       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";")
1040       doublePuts($stderr,file,"        var testFrame = document.getElementById(\"testframe\");")
1041       doublePuts($stderr,file,"        testFrame.contentDocument.open();")
1042       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");")
1043       if benchParams.kind == :throughputBenchmark or benchParams.kind == :multiFileTimedBenchmark
1044         benchParams.dataPaths.each {
1045           | path |
1046           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{path.inspect.inspect[1..-2]}></script>\");")
1047         }
1048       end
1049       if benchParams.kind == :throughputBenchmark
1050         if benchParams.doWarmup
1051           warmup = $warmup
1052         else
1053           warmup = 0
1054         end
1055         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">\");")
1056         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.setUpCommand.js}\");")
1057         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_before = #{curTime};\");")
1058         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_after = __bencher_before;\");")
1059         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_expected = #{$quantum};\");")
1060         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_runs = 0;\");")
1061         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"while (true) {\");")
1062         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    #{benchParams.command.js}\");")
1063         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_runs++;\");")
1064         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_after = #{curTime};\");")
1065         if benchParams.deterministic
1066           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (true) {\");")
1067         else
1068           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (__bencher_after - __bencher_before >= __bencher_expected) {\");")
1069         end
1070         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        if (__bencher_runs >= #{benchParams.minimumIterations} || window.parent.__bencher_count < #{warmup})\");")
1071         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"            break;\");")
1072         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        __bencher_expected += #{$quantum}\");")
1073         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    }\");")
1074         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"}\");")
1075         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.tearDownCommand.js}\");")
1076         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"window.parent.reportResult((__bencher_after - __bencher_before) / __bencher_runs);\");")
1077         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</script>\");")
1078       else
1079         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">var __bencher_before = #{curTime};</script>\");")
1080         if benchParams.kind == :multiFileTimedBenchmark
1081           doublePuts($stderr,file,"        testFrame.contentDocument.write(#{benchParams.command.html.inspect});")
1082         else
1083           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{benchParams.benchPath.inspect.inspect[1..-2]}></script>\");")
1084         end
1085         unless benchParams.kind == :singleFileTimedCallbackBenchmark
1086           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">window.parent.reportResult(#{curTime} - __bencher_before);</script>\");")
1087         end
1088       end
1089       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</body></html>\");")
1090       doublePuts($stderr,file,"        testFrame.contentDocument.close();")
1091       doublePuts($stderr,file,"    }")
1092       doublePuts($stderr,file,"    __bencher_continuation = continuation;")
1093       doublePuts($stderr,file,"    window.setTimeout(doit, 10);")
1094       doublePuts($stderr,file,"}")
1095     }
1096
1097     Benchfile.create(["bencher-htmldoc",".html"]) {
1098       | file |
1099       doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>")
1100     }
1101   else
1102     raise
1103   end
1104 end
1105
1106 def emitBenchRunCode(name, plan, benchParams)
1107   plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchParams), plan)
1108 end
1109
1110 class FileCreator
1111   def initialize(filename)
1112     @filename = filename
1113     @state = :empty
1114   end
1115   
1116   def puts(text)
1117     $script.print "echo #{Shellwords.shellescape(text)}"
1118     if @state == :empty
1119       $script.print " > "
1120       @state = :nonEmpty
1121     else
1122       $script.print " >> "
1123     end
1124     $script.puts "#{Shellwords.shellescape(@filename)}"
1125   end
1126   
1127   def close
1128     if @state == :empty
1129       $script.puts "rm -f #{Shellwords.shellescape(text)}"
1130       $script.puts "touch #{Shellwords.shellescape(text)}"
1131     end
1132   end
1133   
1134   def self.open(filename)
1135     outp = FileCreator.new(filename)
1136     yield outp
1137     outp.close
1138   end
1139 end
1140
1141 def emitSelfContainedBenchRunCode(name, plan, targetFile, configFile, benchmark)
1142   FileCreator.open(configFile) {
1143     | outp |
1144     outp.puts "__bencher_message = \"#{name}: #{plan.vm}: #{plan.iteration}: \";"
1145     outp.puts "__bencher_warmup = #{$warmup};"
1146     outp.puts "__bencher_inner = #{$inner};"
1147     outp.puts "__bencher_benchmark = #{benchmark.to_json};"
1148     case $timeMode
1149     when :preciseTime
1150       outp.puts "__bencher_curTime = (function(){ return testRunner.preciseTime() * 1000; });"
1151     when :date
1152       outp.puts "__bencher_curTime = (function(){ return Date.now(); });"
1153     else
1154       raise
1155     end
1156   }
1157   
1158   plan.vm.emitRunCode(targetFile, plan)
1159 end
1160
1161 def planForDescription(string, plans, benchFullname, vmName, iteration)
1162   raise "Unexpected benchmark full name: #{benchFullname.inspect}, string: #{string.inspect}" unless benchFullname =~ /\//
1163   suiteName = $~.pre_match
1164   return nil if suiteName == "WARMUP"
1165   benchName = $~.post_match
1166   result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration}
1167   raise "Unexpected result dimensions: #{result.inspect}, string: #{string.inspect}" unless result.size == 1
1168   result[0]
1169 end
1170
1171 class ParsedResult
1172   attr_reader :plan, :innerIndex, :time, :result
1173   
1174   def initialize(plan, innerIndex, time)
1175     @plan = plan
1176     @innerIndex = innerIndex
1177     if time == :crashed
1178       @result = :error
1179     else
1180       @time = time
1181       @result = :success
1182     end
1183     
1184     raise unless @plan.is_a? BenchPlan
1185     raise unless @innerIndex.is_a? Integer
1186     raise unless @time.is_a? Numeric or @result == :error
1187   end
1188   
1189   def benchmark
1190     plan.benchmark
1191   end
1192   
1193   def suite
1194     plan.suite
1195   end
1196   
1197   def vm
1198     plan.vm
1199   end
1200   
1201   def outerIndex
1202     plan.iteration
1203   end
1204   
1205   def self.create(plan, innerIndex, time)
1206     if plan
1207       ParsedResult.new(plan, innerIndex, time)
1208     else
1209       nil
1210     end
1211   end
1212   
1213   def self.parse(plans, string)
1214     if string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): Time: /
1215       benchFullname = $1
1216       vmName = $2
1217       outerIndex = $3.to_i
1218       innerIndex = $4.to_i
1219       time = $~.post_match.to_f
1220       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, time)
1221     elsif string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): CRASHED/
1222       benchFullname = $1
1223       vmName = $2
1224       outerIndex = $3.to_i
1225       innerIndex = $4.to_i
1226       time = $~.post_match.to_f
1227       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, :crashed)
1228     else
1229       nil
1230     end
1231   end
1232 end
1233
1234 class VM
1235   @@extraEnvSet = {}
1236   
1237   def initialize(origPath, name, nameKind, svnRevision)
1238     @origPath = origPath.to_s
1239     @path = origPath.to_s
1240     @name = name
1241     @nameKind = nameKind
1242     @extraEnv = {}
1243     
1244     if $forceVMKind
1245       @vmType = $forceVMKind
1246     else
1247       if @origPath =~ /DumpRenderTree$/
1248         @vmType = :dumpRenderTree
1249       elsif @origPath =~ /WebKitTestRunner$/
1250         @vmType = :webkitTestRunner
1251       else
1252         @vmType = :jsc
1253       end
1254     end
1255     
1256     @svnRevision = svnRevision
1257     
1258     # Try to detect information about the VM.
1259     if path =~ /\/WebKitBuild\/(Release|Debug)+\/([a-zA-Z]+)$/
1260       @checkoutPath = $~.pre_match
1261       # FIXME: Use some variant of this: 
1262       # <bdash>   def retrieve_revision
1263       # <bdash>     `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i
1264       # <bdash>   end
1265       unless @svnRevision
1266         begin
1267           Dir.chdir(@checkoutPath) {
1268             $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
1269             IO.popen("svn info", "r") {
1270               | inp |
1271               inp.each_line {
1272                 | line |
1273                 if line =~ /Revision: ([0-9]+)/
1274                   @svnRevision = $1
1275                 end
1276               }
1277             }
1278           }
1279           unless @svnRevision
1280             $stderr.puts "Warning: running svn info for #{name} silently failed."
1281           end
1282         rescue => e
1283           # Failed to detect svn revision.
1284           $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
1285         end
1286       end
1287     else
1288       $stderr.puts "Warning: could not identify checkout location for #{name}"
1289     end
1290     
1291     if @path =~ /\/Release\/([a-zA-Z]+)$/
1292       @libPath, @relativeBinPath = [$~.pre_match+"/Release"], "./#{$1}"
1293     elsif @path =~ /\/Debug\/([a-zA-Z]+)$/
1294       @libPath, @relativeBinPath = [$~.pre_match+"/Debug"], "./#{$1}"
1295     elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/
1296       @libPath = [$~.pre_match + "/Contents/Resources", $~.pre_match + "/Contents/Frameworks"]
1297     elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/
1298       @libPath, @relativeBinPath = [$~.pre_match], $&[1..-1]
1299     elsif @path =~ /(DumpRenderTree|webkitTestRunner|jsc)$/
1300       @libPath, @relativeBinPath = [$~.pre_match+"/"], "./#{$1}"
1301     end
1302   end
1303   
1304   def canCopyIntoBenchPath
1305     if @libPath and @relativeBinPath
1306       true
1307     else
1308       false
1309     end
1310   end
1311   
1312   def addExtraEnv(key, val)
1313     @extraEnv[key] = val
1314     @@extraEnvSet[key] = true
1315   end
1316   
1317   def copyIntoBenchPath
1318     raise unless canCopyIntoBenchPath
1319     basename, filename = Benchfile.uniqueFilename("vm")
1320     raise unless Dir.mkdir(filename)
1321     @libPath.each {
1322       | libPathPart |
1323       cmd = "cp -a #{Shellwords.shellescape(libPathPart)}/* #{Shellwords.shellescape(filename.to_s)}"
1324       $stderr.puts ">> #{cmd}" if $verbosity>=2
1325       raise unless system(cmd)
1326     }
1327     @path = "#{basename}/#{@relativeBinPath}"
1328     @libPath = [basename]
1329   end
1330   
1331   def to_s
1332     @name
1333   end
1334   
1335   def name
1336     @name
1337   end
1338   
1339   def shouldMeasureGC
1340     $measureGC == true or ($measureGC == name)
1341   end
1342   
1343   def origPath
1344     @origPath
1345   end
1346   
1347   def path
1348     @path
1349   end
1350   
1351   def nameKind
1352     @nameKind
1353   end
1354   
1355   def vmType
1356     @vmType
1357   end
1358   
1359   def checkoutPath
1360     @checkoutPath
1361   end
1362   
1363   def svnRevision
1364     @svnRevision
1365   end
1366   
1367   def extraEnv
1368     @extraEnv
1369   end
1370   
1371   def printFunction
1372     case @vmType
1373     when :jsc
1374       "print"
1375     when :dumpRenderTree, :webkitTestRunner
1376       "debug"
1377     else
1378       raise @vmType
1379     end
1380   end
1381   
1382   def emitRunCode(fileToRun, plan)
1383     myLibPath = @libPath
1384     myLibPath = [] unless myLibPath
1385     @@extraEnvSet.keys.each {
1386       | key |
1387       $script.puts "unset #{Shellwords.shellescape(key)}"
1388     }
1389     $script.puts "export DYLD_LIBRARY_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1390     $script.puts "export DYLD_FRAMEWORK_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1391     @extraEnv.each_pair {
1392       | key, val |
1393       $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1394     }
1395     plan.environment.each_pair {
1396         | key, val |
1397         $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1398     }
1399     $script.puts "#{path} #{fileToRun} 2>&1 || {"
1400     $script.puts "    echo " + Shellwords.shellescape("#{name} failed to run!") + " 1>&2"
1401     $inner.times {
1402       | iteration |
1403       $script.puts "    echo " + Shellwords.shellescape("#{plan.prefix}: #{iteration}: CRASHED")
1404     }
1405     $script.puts "}"
1406     plan.environment.keys.each {
1407         | key |
1408         $script.puts "export #{Shellwords.shellescape(key)}="
1409     }
1410   end
1411 end
1412
1413 class StatsAccumulator
1414   def initialize
1415     @stats = []
1416     ($outer*$inner).times {
1417       @stats << Stats.new
1418     }
1419   end
1420   
1421   def statsForIteration(outerIteration, innerIteration)
1422     @stats[outerIteration*$inner + innerIteration]
1423   end
1424   
1425   def stats
1426     result = Stats.new
1427     @stats.each {
1428       | stat |
1429       if stat.ok?
1430         result.add(yield stat)
1431       else
1432         result.add(nil)
1433       end
1434     }
1435     result
1436   end
1437   
1438   def geometricMeanStats
1439     stats {
1440       | stat |
1441       stat.geometricMean
1442     }
1443   end
1444   
1445   def arithmeticMeanStats
1446     stats {
1447       | stat |
1448       stat.arithmeticMean
1449     }
1450   end
1451 end
1452
1453 module Benchmark
1454   attr_accessor :benchmarkSuite
1455   attr_reader :name
1456   
1457   def fullname
1458     benchmarkSuite.name + "/" + name
1459   end
1460   
1461   def to_s
1462     fullname
1463   end
1464   
1465   def weight
1466     1
1467   end
1468   
1469   def weightString
1470     raise unless weight.is_a? Fixnum
1471     raise unless weight >= 1
1472     if weight == 1
1473       ""
1474     else
1475       "x#{weight} "
1476     end
1477   end
1478 end
1479
1480 class SunSpiderBenchmark
1481   include Benchmark
1482   
1483   def initialize(name)
1484     @name = name
1485   end
1486   
1487   def emitRunCode(plan)
1488     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js")))
1489   end
1490 end
1491
1492 class LongSpiderBenchmark
1493   include Benchmark
1494   
1495   def initialize(name)
1496     @name = name
1497   end
1498   
1499   def emitRunCode(plan)
1500     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("LongSpider-#{@name}", "#{LONGSPIDER_PATH}/#{@name}.js")))
1501   end
1502 end
1503
1504 class V8Benchmark
1505   include Benchmark
1506   
1507   def initialize(name)
1508     @name = name
1509   end
1510   
1511   def emitRunCode(plan)
1512     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js")))
1513   end
1514 end
1515
1516 class V8RealBenchmark
1517   include Benchmark
1518   
1519   attr_reader :v8SuiteName
1520   
1521   def initialize(v8SuiteName, name, weight, minimumIterations)
1522     @v8SuiteName = v8SuiteName
1523     @name = name
1524     @weight = weight
1525     @minimumIterations = minimumIterations
1526   end
1527   
1528   def weight
1529     @weight
1530   end
1531   
1532   def emitRunCode(plan)
1533     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(["base", @v8SuiteName, "jsc-#{@name}"].collect{|v| ensureFile("V8Real-#{v}", "#{V8_REAL_PATH}/#{v}.js")}, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, false, @minimumIterations))
1534   end
1535 end
1536
1537 class OctaneBenchmark
1538   include Benchmark
1539   
1540   def initialize(files, name, weight, doWarmup, deterministic, minimumIterations)
1541     @files = files
1542     @name = name
1543     @weight = weight
1544     @doWarmup = doWarmup
1545     @deterministic = deterministic
1546     @minimumIterations = minimumIterations
1547   end
1548   
1549   def weight
1550     @weight
1551   end
1552   
1553   def emitRunCode(plan)
1554     files = []
1555     files += (["base"] + @files).collect {
1556       | v |
1557       ensureFile("Octane-#{v}", "#{OCTANE_PATH}/#{v}.js")
1558     }
1559     files += ["jsc-#{@name}"].collect {
1560       | v |
1561       ensureFile("Octane-#{v}", "#{OCTANE_WRAPPER_PATH}/#{v}.js")
1562     }
1563     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(files, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1564   end
1565 end
1566
1567 class KrakenBenchmark
1568   include Benchmark
1569   
1570   def initialize(name)
1571     @name = name
1572   end
1573   
1574   def emitRunCode(plan)
1575     emitBenchRunCode(fullname, plan, MultiFileTimedBenchmarkParameters.new([ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js")], loadCommandForFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js")))
1576   end
1577 end
1578
1579 class JSBenchBenchmark
1580   include Benchmark
1581   
1582   attr_reader :jsBenchMode
1583   
1584   def initialize(name, jsBenchMode)
1585     @name = name
1586     @jsBenchMode = jsBenchMode
1587   end
1588   
1589   def emitRunCode(plan)
1590     callbackDecl  = "function JSBNG_handleResult(result) {\n"
1591     callbackDecl += "    if (result.error) {\n"
1592     callbackDecl += "        console.log(\"Did not run benchmark correctly!\");\n"
1593     callbackDecl += "        quit();\n"
1594     callbackDecl += "    }\n"
1595     callbackDecl += "    reportResult(result.time);\n"
1596     callbackDecl += "}\n"
1597     emitBenchRunCode(fullname, plan, SingleFileTimedCallbackBenchmarkParameters.new(callbackDecl, ensureFile("JSBench-#{@name}", "#{JSBENCH_PATH}/#{@name}/#{@jsBenchMode}.js")))
1598   end
1599 end
1600
1601 class JSRegressBenchmark
1602   include Benchmark
1603   
1604   def initialize(name)
1605     @name = name
1606   end
1607   
1608   def emitRunCode(plan)
1609     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("JSRegress-#{@name}", "#{JSREGRESS_PATH}/#{@name}.js")))
1610   end
1611 end
1612
1613 class AsmBenchBenchmark
1614   include Benchmark
1615   
1616   def initialize(name)
1617     @name = name
1618   end
1619   
1620   def emitRunCode(plan)
1621     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("AsmBench-#{@name}", "#{ASMBENCH_PATH}/#{@name}.js")))
1622   end
1623 end
1624
1625
1626 class CompressionBenchBenchmark
1627   include Benchmark
1628   
1629   def initialize(files, name, model)
1630     @files = files
1631     @name = name;
1632     @name = name + "-" + model if !model.empty?
1633     @name = @name.gsub(" ", "-").downcase
1634     @scriptName = name
1635     @weight = 1
1636     @doWarmup = true
1637     @deterministic = true
1638     @minimumIterations = 1
1639     @model = model
1640   end
1641   
1642   def weight
1643     @weight
1644   end
1645   
1646   def emitRunCode(plan)
1647     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new((["base"] + @files + ["jsc-#{@scriptName}"]).collect{|v| ensureFile("Compression-#{v}", "#{COMPRESSIONBENCH_PATH}/#{v}.js")}, simpleCommand("jscSetUp('#{@model}');"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1648   end
1649 end
1650
1651 class DSPJSFiltrrBenchmark
1652   include Benchmark
1653   
1654   def initialize(name, filterKey)
1655     @name = name
1656     @filterKey = filterKey
1657   end
1658   
1659   def emitRunCode(plan)
1660     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr.js")
1661     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr_back.jpg")
1662     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-jquery.min.js")
1663     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-bencher.html")
1664     emitSelfContainedBenchRunCode(fullname, plan, "filtrr-bencher.html", "bencher-config.js", @filterKey)
1665   end
1666 end
1667
1668 class DSPJSVP8Benchmark
1669   include Benchmark
1670   
1671   def initialize
1672     @name = "route9-vp8"
1673   end
1674   
1675   def weight
1676     5
1677   end
1678   
1679   def emitRunCode(plan)
1680     ensureBenchmarkFiles(DSPJS_ROUTE9_PATH)
1681     emitSelfContainedBenchRunCode(fullname, plan, "route9-bencher.html", "bencher-config.js", "")
1682   end
1683 end
1684
1685 class DSPStarfieldBenchmark
1686   include Benchmark
1687
1688   def initialize
1689     @name = "starfield"
1690   end
1691   
1692   def weight
1693     5
1694   end
1695
1696   def emitRunCode(plan)
1697     ensureBenchmarkFiles(DSPJS_STARFIELD_PATH)
1698     emitSelfContainedBenchRunCode(fullname, plan, "starfield-bencher.html", "bencher-config.js", "")
1699   end
1700 end
1701
1702 class DSPJSJSLinuxBenchmark
1703   include Benchmark
1704   def initialize
1705     @name = "bellard-jslinux"
1706   end
1707
1708   def weight
1709     5
1710   end
1711
1712   def emitRunCode(plan)
1713     ensureBenchmarkFiles(DSPJS_JSLINUX_PATH)
1714     emitSelfContainedBenchRunCode(fullname, plan, "jslinux-bencher.html", "bencher-config.js", "")
1715   end
1716 end
1717
1718 class DSPJSQuake3Benchmark
1719   include Benchmark
1720
1721   def initialize
1722     @name = "zynaps-quake3"
1723   end
1724
1725   def weight
1726     5
1727   end
1728
1729   def emitRunCode(plan)
1730     ensureBenchmarkFiles(DSPJS_QUAKE3_PATH)
1731     emitSelfContainedBenchRunCode(fullname, plan, "quake-bencher.html", "bencher-config.js", "")
1732   end
1733 end
1734
1735 class DSPJSMandelbrotBenchmark
1736   include Benchmark
1737
1738   def initialize
1739     @name = "zynaps-mandelbrot"
1740   end
1741
1742   def weight
1743     5
1744   end
1745
1746   def emitRunCode(plan)
1747     ensureBenchmarkFiles(DSPJS_MANDELBROT_PATH)
1748     emitSelfContainedBenchRunCode(fullname, plan, "mandelbrot-bencher.html", "bencher-config.js", "")
1749   end
1750 end
1751
1752 class DSPJSAmmoJSASMBenchmark
1753   include Benchmark
1754
1755   def initialize
1756     @name = "ammojs-asm-js"
1757   end
1758
1759   def weight
1760     5
1761   end
1762
1763   def emitRunCode(plan)
1764     ensureBenchmarkFiles(DSPJS_AMMOJS_ASMJS_PATH)
1765     emitSelfContainedBenchRunCode(fullname, plan, "ammo-asmjs-bencher.html", "bencher-config.js", "")
1766   end
1767 end
1768
1769 class DSPJSAmmoJSRegularBenchmark
1770   include Benchmark
1771
1772   def initialize
1773     @name = "ammojs-regular-js"
1774   end
1775
1776   def weight
1777     5
1778   end
1779
1780   def emitRunCode(plan)
1781     ensureBenchmarkFiles(DSPJS_AMMOJS_REGULAR_PATH)
1782     emitSelfContainedBenchRunCode(fullname, plan, "ammo-regular-bencher.html", "bencher-config.js", "")
1783   end
1784 end
1785
1786 class BrowsermarkJSBenchmark
1787   include Benchmark
1788     
1789   def initialize(name)
1790     @name = name
1791   end
1792   
1793   def emitRunCode(plan)
1794     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new([ensureFile(name, "#{BROWSERMARK_JS_PATH}/#{name}/test.js"), ensureFile("browsermark-bencher", "#{BROWSERMARK_JS_PATH}/browsermark-bencher.js")], simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, 32))
1795   end
1796 end
1797
1798 class BrowsermarkDOMBenchmark
1799   include Benchmark
1800     
1801   def initialize(name)
1802     @name = name
1803   end
1804   
1805   def emitRunCode(plan)
1806     ensureBenchmarkFiles(BROWSERMARK_PATH)
1807     emitSelfContainedBenchRunCode(fullname, plan, "tests/benchmarks/dom/#{name}/index.html", "bencher-config.js", name)
1808   end
1809 end
1810
1811 class BenchmarkSuite
1812   def initialize(name, preferredMean, decimalShift)
1813     @name = name
1814     @preferredMean = preferredMean
1815     @benchmarks = []
1816     @subSuites = []
1817     @decimalShift = decimalShift
1818   end
1819   
1820   def name
1821     @name
1822   end
1823   
1824   def to_s
1825     @name
1826   end
1827   
1828   def decimalShift
1829     @decimalShift
1830   end
1831   
1832   def addIgnoringPattern(benchmark)
1833     benchmark.benchmarkSuite = self
1834     @benchmarks << benchmark
1835   end
1836   
1837   def add(benchmark)
1838     if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern
1839         addIgnoringPattern(benchmark)
1840     end
1841   end
1842   
1843   def addSubSuite(subSuite)
1844     @subSuites << subSuite
1845   end
1846   
1847   def benchmarks
1848     @benchmarks
1849   end
1850   
1851   def benchmarkForName(name)
1852     result = @benchmarks.select{|v| v.name == name}
1853     raise unless result.length == 1
1854     result[0]
1855   end
1856   
1857   def hasBenchmark(benchmark)
1858     array = @benchmarks.select{|v| v == benchmark}
1859     raise unless array.length == 1 or array.length == 0
1860     array.length == 1
1861   end
1862   
1863   def subSuites
1864     @subSuites
1865   end
1866   
1867   def suites
1868     [self] + @subSuites
1869   end
1870   
1871   def suitesWithBenchmark(benchmark)
1872     result = [self]
1873     @subSuites.each {
1874       | subSuite |
1875       if subSuite.hasBenchmark(benchmark)
1876         result << subSuite
1877       end
1878     }
1879     result
1880   end
1881   
1882   def empty?
1883     @benchmarks.empty?
1884   end
1885   
1886   def retain_if
1887     @benchmarks.delete_if {
1888       | benchmark |
1889       not yield benchmark
1890     }
1891   end
1892   
1893   def preferredMean
1894     @preferredMean
1895   end
1896   
1897   def computeMean(stat)
1898     if stat.ok?
1899       (stat.send @preferredMean) * (10 ** decimalShift)
1900     else
1901       nil
1902     end
1903   end
1904 end
1905
1906 class BenchRunPlan
1907   def initialize(benchmark, vm, iteration)
1908     @benchmark = benchmark
1909     @vm = vm
1910     @iteration = iteration
1911     @environment = {}
1912     if $environment.has_key?(vm.name)
1913       if $environment[vm.name].has_key?(benchmark.benchmarkSuite.name)
1914         if $environment[vm.name][benchmark.benchmarkSuite.name].has_key?(benchmark.name)
1915           @environment = $environment[vm.name][benchmark.benchmarkSuite.name][benchmark.name]
1916         end
1917       end
1918     end
1919   end
1920   
1921   def benchmark
1922     @benchmark
1923   end
1924   
1925   def suite
1926     @benchmark.benchmarkSuite
1927   end
1928   
1929   def vm
1930     @vm
1931   end
1932   
1933   def iteration
1934     @iteration
1935   end
1936  
1937   def environment
1938     @environment
1939   end
1940  
1941   def prefix
1942     "#{@benchmark.fullname}: #{vm.name}: #{iteration}"
1943   end
1944   
1945   def emitRunCode
1946     @benchmark.emitRunCode(self)
1947   end
1948   
1949   def to_s
1950     benchmark.to_s + "/" + vm.to_s
1951   end
1952 end
1953
1954 class BenchmarkOnVM
1955   def initialize(benchmark, suiteOnVM, subSuitesOnVM)
1956     @benchmark = benchmark
1957     @suiteOnVM = suiteOnVM
1958     @subSuitesOnVM = subSuitesOnVM
1959     @stats = Stats.new
1960   end
1961   
1962   def to_s
1963     "#{@benchmark} on #{@suiteOnVM.vm}"
1964   end
1965   
1966   def benchmark
1967     @benchmark
1968   end
1969   
1970   def vm
1971     @suiteOnVM.vm
1972   end
1973   
1974   def vmStats
1975     @suiteOnVM.vmStats
1976   end
1977   
1978   def suite
1979     @benchmark.benchmarkSuite
1980   end
1981   
1982   def suiteOnVM
1983     @suiteOnVM
1984   end
1985   
1986   def subSuitesOnVM
1987     @subSuitesOnVM
1988   end
1989   
1990   def stats
1991     @stats
1992   end
1993   
1994   def parseResult(result)
1995     raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm
1996     raise unless result.benchmark == @benchmark
1997     @stats.add(result.time)
1998   end
1999 end
2000
2001 class NamedStatsAccumulator < StatsAccumulator
2002   def initialize(name)
2003     super()
2004     @name = name
2005   end
2006   
2007   def reportingName
2008     @name
2009   end
2010 end
2011
2012 class SuiteOnVM < StatsAccumulator
2013   def initialize(vm, vmStats, suite)
2014     super()
2015     @vm = vm
2016     @vmStats = vmStats
2017     @suite = suite
2018     
2019     raise unless @vm.is_a? VM
2020     raise unless @vmStats.is_a? StatsAccumulator
2021     raise unless @suite.is_a? BenchmarkSuite
2022   end
2023   
2024   def to_s
2025     "#{@suite} on #{@vm}"
2026   end
2027   
2028   def suite
2029     @suite
2030   end
2031   
2032   def vm
2033     @vm
2034   end
2035
2036   def reportingName
2037     @vm.name
2038   end
2039   
2040   def vmStats
2041     raise unless @vmStats
2042     @vmStats
2043   end
2044 end
2045
2046 class SubSuiteOnVM < StatsAccumulator
2047   def initialize(vm, suite)
2048     super()
2049     @vm = vm
2050     @suite = suite
2051     raise unless @vm.is_a? VM
2052     raise unless @suite.is_a? BenchmarkSuite
2053   end
2054   
2055   def to_s
2056     "#{@suite} on #{@vm}"
2057   end
2058   
2059   def suite
2060     @suite
2061   end
2062   
2063   def vm
2064     @vm
2065   end
2066   
2067   def reportingName
2068     @vm.name
2069   end
2070 end
2071
2072 class BenchPlan
2073   def initialize(benchmarkOnVM, iteration)
2074     @benchmarkOnVM = benchmarkOnVM
2075     @iteration = iteration
2076   end
2077   
2078   def to_s
2079     "#{@benchmarkOnVM} \##{@iteration+1}"
2080   end
2081   
2082   def benchmarkOnVM
2083     @benchmarkOnVM
2084   end
2085   
2086   def benchmark
2087     @benchmarkOnVM.benchmark
2088   end
2089   
2090   def suite
2091     @benchmarkOnVM.suite
2092   end
2093   
2094   def vm
2095     @benchmarkOnVM.vm
2096   end
2097   
2098   def iteration
2099     @iteration
2100   end
2101   
2102   def parseResult(result)
2103     raise unless result.plan == self
2104     @benchmarkOnVM.parseResult(result)
2105     benchmark.weight.times {
2106       @benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time)
2107       @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time)
2108       @benchmarkOnVM.subSuitesOnVM.each {
2109         | subSuite |
2110         subSuite.statsForIteration(@iteration, result.innerIndex).add(result.time)
2111       }
2112     }
2113   end
2114 end
2115
2116 def lpad(str,chars)
2117   if str.length>chars
2118     str
2119   else
2120     "%#{chars}s"%(str)
2121   end
2122 end
2123
2124 def rpad(str,chars)
2125   while str.length < chars
2126     str+=" "
2127   end
2128   str
2129 end
2130
2131 def center(str,chars)
2132   while str.length<chars
2133     str+=" "
2134     if str.length<chars
2135       str=" "+str
2136     end
2137   end
2138   str
2139 end
2140
2141 def statsToStr(stats, decimalShift)
2142   if stats.error?
2143     lpad(center("ERROR", 10+10+2), 12+10+2)
2144   elsif $inner*$outer == 1
2145     string = numToStr(stats.mean, decimalShift)
2146     raise unless string =~ /\./
2147     left = $~.pre_match
2148     right = $~.post_match
2149     lpad(left, 13 - decimalShift) + "." + rpad(right, 10 + decimalShift)
2150   else
2151     lpad(numToStr(stats.mean, decimalShift), 12) + "+-" + rpad(numToStr(stats.confInt, decimalShift), 10)
2152   end
2153 end
2154
2155 def plural(num)
2156   if num == 1
2157     ""
2158   else
2159     "s"
2160   end
2161 end
2162
2163 def wrap(str, columns)
2164   array = str.split
2165   result = ""
2166   curLine = array.shift
2167   array.each {
2168     | curStr |
2169     if (curLine + " " + curStr).size > columns
2170       result += curLine + "\n"
2171       curLine = curStr
2172     else
2173       curLine += " " + curStr
2174     end
2175   }
2176   result + curLine + "\n"
2177 end
2178   
2179 def runAndGetResults
2180   results = nil
2181   Dir.chdir(BENCH_DATA_PATH) {
2182     $stderr.puts ">> sh ./runscript" if $verbosity >= 2
2183     raise "Script did not complete correctly: #{$?}" unless system("sh ./runscript > runlog")
2184     results = IO::read("runlog")
2185   }
2186   raise unless results
2187   results
2188 end
2189
2190 def parseAndDisplayResults(results)
2191   vmStatses = []
2192   $vms.each {
2193     | vm |
2194     vmStatses << NamedStatsAccumulator.new(vm.name)
2195   }
2196   
2197   suitesOnVMs = []
2198   suitesOnVMsForSuite = {}
2199   subSuitesOnVMsForSubSuite = {}
2200   $suites.each {
2201     | suite |
2202     suitesOnVMsForSuite[suite] = []
2203     suite.subSuites.each {
2204       | subSuite |
2205       subSuitesOnVMsForSubSuite[subSuite] = []
2206     }
2207   }
2208   suitesOnVMsForVM = {}
2209   $vms.each {
2210     | vm |
2211     suitesOnVMsForVM[vm] = []
2212   }
2213   
2214   benchmarksOnVMs = []
2215   benchmarksOnVMsForBenchmark = {}
2216   $benchmarks.each {
2217     | benchmark |
2218     benchmarksOnVMsForBenchmark[benchmark] = []
2219   }
2220   
2221   $vms.each_with_index {
2222     | vm, vmIndex |
2223     vmStats = vmStatses[vmIndex]
2224     $suites.each {
2225       | suite |
2226       suiteOnVM = SuiteOnVM.new(vm, vmStats, suite)
2227       subSuitesOnVM = suite.subSuites.map {
2228         | subSuite |
2229         result = SubSuiteOnVM.new(vm, subSuite)
2230         subSuitesOnVMsForSubSuite[subSuite] << result
2231         result
2232       }
2233       suitesOnVMs << suiteOnVM
2234       suitesOnVMsForSuite[suite] << suiteOnVM
2235       suitesOnVMsForVM[vm] << suiteOnVM
2236       suite.benchmarks.each {
2237         | benchmark |
2238         subSuitesOnVMForThisBenchmark = []
2239         subSuitesOnVM.each {
2240           | subSuiteOnVM |
2241           if subSuiteOnVM.suite.hasBenchmark(benchmark)
2242             subSuitesOnVMForThisBenchmark << subSuiteOnVM
2243           end
2244         }
2245         benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM, subSuitesOnVMForThisBenchmark)
2246         benchmarksOnVMs << benchmarkOnVM
2247         benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
2248       }
2249     }
2250   }
2251   
2252   plans = []
2253   benchmarksOnVMs.each {
2254     | benchmarkOnVM |
2255     $outer.times {
2256       | iteration |
2257       plans << BenchPlan.new(benchmarkOnVM, iteration)
2258     }
2259   }
2260
2261   hostname = nil
2262   hwmodel = nil
2263   results.each_line {
2264     | line |
2265     line.chomp!
2266     if line =~ /HOSTNAME:([^.]+)/
2267       hostname = $1
2268     elsif line =~ /HARDWARE:hw\.model: /
2269       hwmodel = $~.post_match.chomp
2270     else
2271       result = ParsedResult.parse(plans, line)
2272       if result
2273         result.plan.parseResult(result)
2274       end
2275     end
2276   }
2277   
2278   # Compute the geomean of the preferred means of results on a SuiteOnVM
2279   overallResults = []
2280   $vms.each {
2281     | vm |
2282     result = Stats.new
2283     $outer.times {
2284       | outerIndex |
2285       $inner.times {
2286         | innerIndex |
2287         curResult = Stats.new
2288         suitesOnVMsForVM[vm].each {
2289           | suiteOnVM |
2290           # For a given iteration, suite, and VM, compute the suite's preferred mean
2291           # over the data collected for all benchmarks in that suite. We'll have one
2292           # sample per benchmark. For example on V8 this will be the geomean of 1
2293           # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
2294           # splay.
2295           curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
2296         }
2297         
2298         # curResult now holds 1 sample for each of the means computed in the above
2299         # loop. Compute the geomean over this, and store it.
2300         if curResult.ok?
2301           result.add(curResult.geometricMean)
2302         else
2303           result.add(nil)
2304         end
2305       }
2306     }
2307
2308     # $overallResults will have a Stats for each VM. That Stats object will hold
2309     # $inner*$outer geomeans, allowing us to compute the arithmetic mean and
2310     # confidence interval of the geomeans of preferred means. Convoluted, but
2311     # useful and probably sound.
2312     overallResults << result
2313   }
2314   
2315   if $verbosity >= 2
2316     benchmarksOnVMs.each {
2317       | benchmarkOnVM |
2318       $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
2319     }
2320     
2321     $vms.each_with_index {
2322       | vm, vmIndex |
2323       vmStats = vmStatses[vmIndex]
2324       $stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}"
2325       $stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}"
2326     }
2327   end
2328
2329   if $outputName
2330     reportName = $outputName
2331   else
2332     reportName =
2333       (if ($vms.collect {
2334              | vm |
2335              vm.nameKind
2336            }.index :auto)
2337          ""
2338        else
2339          text = $vms.collect {
2340            | vm |
2341            vm.to_s
2342          }.join("_") + "_"
2343          if text.size >= 40
2344            ""
2345          else
2346            text
2347          end
2348        end) +
2349       ($suites.collect {
2350          | suite |
2351          suite.to_s
2352        }.join("")) + "_" +
2353       (if hostname
2354          hostname + "_"
2355        else
2356          ""
2357        end)+
2358       (begin
2359          time = Time.now
2360          "%04d%02d%02d_%02d%02d" %
2361            [ time.year, time.month, time.day,
2362              time.hour, time.min ]
2363        end)
2364   end
2365
2366   unless $brief
2367     puts "Generating benchmark report at #{Dir.pwd}/#{reportName}_report.txt"
2368     puts "And raw data at #{Dir.pwd}/#{reportName}.json"
2369   end
2370   
2371   outp = $stdout
2372   json = {}
2373   begin
2374     outp = File.open(reportName + "_report.txt","w")
2375   rescue => e
2376     $stderr.puts "Error: could not save report to #{reportName}_report.txt: #{e}"
2377     $stderr.puts
2378   end
2379   
2380   def createVMsString
2381     result = ""
2382     result += "   " if $allSuites.size > 1
2383     result += rpad("", $benchpad + $weightpad)
2384     result += " "
2385     $vms.size.times {
2386       | index |
2387       if index != 0
2388         result += " "+NoChange.new(0).shortForm
2389       end
2390       result += lpad(center($vms[index].name, 10+10+2), 12+10+2)
2391     }
2392     result += "    "
2393     if $vms.size >= 3
2394       result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
2395     elsif $vms.size >= 2
2396       result += " "*26
2397     end
2398     result
2399   end
2400   
2401   def andJoin(list)
2402     if list.size == 1
2403       list[0].to_s
2404     elsif list.size == 2
2405       "#{list[0]} and #{list[1]}"
2406     else
2407       "#{list[0..-2].join(', ')}, and #{list[-1]}"
2408     end
2409   end
2410   
2411   json["vms"] = $vms.collect{|v| v.name}
2412   json["suites"] = {}
2413   json["runlog"] = results
2414   
2415   columns = [createVMsString.size, 78].max
2416   
2417   outp.print "Benchmark report for "
2418   outp.print andJoin($suites)
2419   if hostname
2420     outp.print " on #{hostname}"
2421   end
2422   if hwmodel
2423     outp.print " (#{hwmodel})"
2424   end
2425   outp.puts "."
2426   outp.puts
2427   
2428   outp.puts "VMs tested:"
2429   $vms.each {
2430     | vm |
2431     outp.print "\"#{vm.name}\" at #{vm.origPath}"
2432     if vm.svnRevision
2433       outp.print " (r#{vm.svnRevision})"
2434     end
2435     outp.puts
2436     vm.extraEnv.each_pair {
2437       | key, val |
2438       outp.puts "    export #{key}=#{val}"
2439     }
2440   }
2441   
2442   outp.puts
2443   
2444   outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
2445                  "with #{$outer} VM invocation#{plural($outer)} per benchmark."+
2446                  (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
2447                                       "total time of those iterations, for each sample.")
2448                   else "" end)+
2449                  (if $measureGC == true then (" No manual garbage collection invocations were "+
2450                                               "emitted.")
2451                   elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
2452                                          "all VMs except #{$measureGC}.")
2453                   else (" Emitted a call to gc() between sample measurements.") end)+
2454                  (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
2455                                         "began with the very first iteration.")
2456                   else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
2457                         "invocation for warm-up.") end)+
2458                  (case $timeMode
2459                   when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
2460                                           "microsecond-level timing.")
2461                   when :date then (" Used the portable Date.now() method to get millisecond-"+
2462                                    "level timing.")
2463                   else raise end)+
2464                  " Reporting benchmark execution times with 95% confidence "+
2465                  "intervals in milliseconds.",
2466                  columns)
2467   
2468   outp.puts
2469   
2470   def printVMs(outp)
2471     outp.puts createVMsString
2472   end
2473   
2474   def summaryStats(outp, json, accumulators, name, decimalShift, &proc)
2475     resultingJson = {}
2476     outp.print "   " if $allSuites.size > 1
2477     outp.print rpad(name, $benchpad + $weightpad)
2478     outp.print " "
2479     accumulators.size.times {
2480       | index |
2481       if index != 0
2482         outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
2483       end
2484       outp.print statsToStr(accumulators[index].stats(&proc), decimalShift)
2485       resultingJson[accumulators[index].reportingName] = accumulators[index].stats(&proc).jsonMap
2486     }
2487     if accumulators.size>=2
2488       outp.print("    "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm)
2489     end
2490     outp.puts
2491     json[name] = resultingJson
2492   end
2493
2494   def allSummaryStats(outp, json, accumulators, preferredMean, decimalShift)
2495     meanLabel = '<' + preferredMean.to_s.sub(/Mean$/, '') + '>'
2496     summaryStats(outp, json, accumulators, meanLabel, decimalShift) {
2497       | stat |
2498       stat.send(preferredMean)
2499     }
2500   end
2501   
2502   $suites.each {
2503     | suite |
2504     suiteJson = {}
2505     subSuiteJsons = {}
2506     suite.subSuites.each {
2507       | subSuite |
2508       subSuiteJsons[subSuite] = {}
2509     }
2510     
2511     printVMs(outp)
2512     if $allSuites.size > 1
2513       outp.puts(andJoin(suite.suites.map{|v| v.name}) + ":")
2514     else
2515       outp.puts
2516     end
2517     suite.benchmarks.each {
2518       | benchmark |
2519       benchmarkJson = {}
2520       outp.print "   " if $allSuites.size > 1
2521       outp.print rpad(benchmark.name, $benchpad) + rpad(benchmark.weightString, $weightpad)
2522       if benchmark.name.size > $benchNameClip
2523         outp.puts
2524         outp.print "   " if $allSuites.size > 1
2525         outp.print((" " * $benchpad) + (" " * $weightpad))
2526       end
2527       outp.print " "
2528       myConfigs = benchmarksOnVMsForBenchmark[benchmark]
2529       myConfigs.size.times {
2530         | index |
2531         if index != 0
2532           outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
2533         end
2534         outp.print statsToStr(myConfigs[index].stats, suite.decimalShift)
2535         benchmarkJson[myConfigs[index].vm.name] = myConfigs[index].stats.jsonMap
2536       }
2537       if $vms.size>=2
2538         outp.print("    "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
2539       end
2540       outp.puts
2541       suiteJson[benchmark.name] = benchmarkJson
2542       suite.subSuites.each {
2543         | subSuite |
2544         if subSuite.hasBenchmark(benchmark)
2545           subSuiteJsons[subSuite][benchmark.name] = benchmarkJson
2546         end
2547       }
2548     }
2549     outp.puts
2550     unless suite.subSuites.empty?
2551       suite.subSuites.each {
2552         | subSuite |
2553         outp.puts "#{subSuite.name}:"
2554         allSummaryStats(outp, subSuiteJsons[subSuite], subSuitesOnVMsForSubSuite[subSuite], subSuite.preferredMean, subSuite.decimalShift)
2555         outp.puts
2556       }
2557       outp.puts "#{suite.name} including #{andJoin(suite.subSuites.map{|v| v.name})}:"
2558     end
2559     allSummaryStats(outp, suiteJson, suitesOnVMsForSuite[suite], suite.preferredMean, suite.decimalShift)
2560     outp.puts if $allSuites.size > 1
2561     
2562     json["suites"][suite.name] = suiteJson
2563     suite.subSuites.each {
2564       | subSuite |
2565       json["suites"][subSuite.name] = subSuiteJsons[subSuite]
2566     }
2567   }
2568   
2569   if $suites.size > 1
2570     scaledResultJson = {}
2571     printVMs(outp)
2572     outp.puts "Geomean of preferred means:"
2573     outp.print "   "
2574     outp.print rpad("<scaled-result>", $benchpad + $weightpad)
2575     outp.print " "
2576     $vms.size.times {
2577       | index |
2578       if index != 0
2579         outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm
2580       end
2581       outp.print statsToStr(overallResults[index], 0)
2582       scaledResultJson[$vms[index].name] = overallResults[index].jsonMap
2583     }
2584     if overallResults.size>=2
2585       outp.print("    "+overallResults[-1].compareTo(overallResults[0]).longForm)
2586     end
2587     outp.puts
2588     
2589     json["<scaled-result>"] = scaledResultJson
2590   end
2591   outp.puts
2592   
2593   if outp != $stdout
2594     outp.close
2595   end
2596   
2597   if outp != $stdout and not $brief
2598     puts
2599     File.open(reportName + "_report.txt") {
2600       | inp |
2601       puts inp.read
2602     }
2603   end
2604   
2605   if $brief
2606     puts(overallResults.collect{|stats| stats.mean}.join("\t"))
2607     puts(overallResults.collect{|stats| stats.confInt}.join("\t"))
2608   end
2609   
2610   File.open(reportName + ".json", "w") {
2611     | outp |
2612     outp.puts json.to_json
2613   }
2614 end
2615
2616 begin
2617   $sawBenchOptions = false
2618   
2619   def resetBenchOptionsIfNecessary
2620     unless $sawBenchOptions
2621       $includeSunSpider = false
2622       $includeLongSpider = false
2623       $includeV8 = false
2624       $includeKraken = false
2625       $includeJSBench = false
2626       $includeJSRegress = false
2627       $includeAsmBench = false
2628       $includeDSPJS = false
2629       $includeBrowsermarkJS = false
2630       $includeBrowsermarkDOM = false
2631       $includeOctane = false
2632       $includeCompressionBench = false
2633       $sawBenchOptions = true
2634     end
2635   end
2636   
2637   GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
2638                  ['--inner', GetoptLong::REQUIRED_ARGUMENT],
2639                  ['--outer', GetoptLong::REQUIRED_ARGUMENT],
2640                  ['--warmup', GetoptLong::REQUIRED_ARGUMENT],
2641                  ['--no-ss-warmup', GetoptLong::NO_ARGUMENT],
2642                  ['--quantum', GetoptLong::REQUIRED_ARGUMENT],
2643                  ['--minimum', GetoptLong::REQUIRED_ARGUMENT],
2644                  ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
2645                  ['--sunspider', GetoptLong::NO_ARGUMENT],
2646                  ['--longspider', GetoptLong::NO_ARGUMENT],
2647                  ['--v8-spider', GetoptLong::NO_ARGUMENT],
2648                  ['--kraken', GetoptLong::NO_ARGUMENT],
2649                  ['--js-bench', GetoptLong::NO_ARGUMENT],
2650                  ['--js-regress', GetoptLong::NO_ARGUMENT],
2651                  ['--asm-bench', GetoptLong::NO_ARGUMENT],
2652                  ['--dsp', GetoptLong::NO_ARGUMENT],
2653                  ['--browsermark-js', GetoptLong::NO_ARGUMENT],
2654                  ['--browsermark-dom', GetoptLong::NO_ARGUMENT],
2655                  ['--octane', GetoptLong::NO_ARGUMENT],
2656                  ['--compression-bench', GetoptLong::NO_ARGUMENT],
2657                  ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
2658                  ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
2659                  ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
2660                  ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
2661                  ['--dont-copy-vms', GetoptLong::NO_ARGUMENT],
2662                  ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
2663                  ['--brief', GetoptLong::NO_ARGUMENT],
2664                  ['--silent', GetoptLong::NO_ARGUMENT],
2665                  ['--remote', GetoptLong::REQUIRED_ARGUMENT],
2666                  ['--local', GetoptLong::NO_ARGUMENT],
2667                  ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
2668                  ['--slave', GetoptLong::NO_ARGUMENT],
2669                  ['--prepare-only', GetoptLong::NO_ARGUMENT],
2670                  ['--analyze', GetoptLong::REQUIRED_ARGUMENT],
2671                  ['--vms', GetoptLong::REQUIRED_ARGUMENT],
2672                  ['--output-name', GetoptLong::REQUIRED_ARGUMENT],
2673                  ['--environment', GetoptLong::REQUIRED_ARGUMENT],
2674                  ['--config', GetoptLong::REQUIRED_ARGUMENT],
2675                  ['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
2676     | opt, arg |
2677     case opt
2678     when '--rerun'
2679       $rerun = intArg(opt,arg,1,nil)
2680     when '--inner'
2681       $inner = intArg(opt,arg,1,nil)
2682     when '--outer'
2683       $outer = intArg(opt,arg,1,nil)
2684     when '--warmup'
2685       $warmup = intArg(opt,arg,0,nil)
2686     when '--no-ss-warmup'
2687       $sunSpiderWarmup = false
2688     when '--quantum'
2689       $quantum = intArg(opt,arg,1,nil)
2690     when '--minimum'
2691       $minimum = intArg(opt,arg,1,nil)
2692     when '--timing-mode'
2693       if arg.upcase == "PRECISETIME"
2694         $timeMode = :preciseTime
2695       elsif arg.upcase == "DATE"
2696         $timeMode = :date
2697       elsif arg.upcase == "AUTO"
2698         $timeMode = :auto
2699       else
2700         quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
2701                   "Invalid argument for command-line option")
2702       end
2703     when '--force-vm-kind'
2704       if arg.upcase == "JSC"
2705         $forceVMKind = :jsc
2706       elsif arg.upcase == "DUMPRENDERTREE"
2707         $forceVMKind = :dumpRenderTree
2708       elsif arg.upcase == "WEBKITTESTRUNNER"
2709         $forceVMKind = :webkitTestRunner
2710       elsif arg.upcase == "AUTO"
2711         $forceVMKind = nil
2712       else
2713         quickFail("Expected 'jsc', 'DumpRenderTree', or 'WebKitTestRunner' for --force-vm-kind, but got '#{arg}'.",
2714                   "Invalid argument for command-line option")
2715       end
2716     when '--force-vm-copy'
2717       $needToCopyVMs = true
2718     when '--dont-copy-vms'
2719       $dontCopyVMs = true
2720     when '--sunspider'
2721       resetBenchOptionsIfNecessary
2722       $includeSunSpider = true
2723     when '--longspider'
2724       resetBenchOptionsIfNecessary
2725       $includeLongSpider = true
2726     when '--v8-spider'
2727       resetBenchOptionsIfNecessary
2728       $includeV8 = true
2729     when '--kraken'
2730       resetBenchOptionsIfNecessary
2731       $includeKraken = true
2732     when '--js-bench'
2733       resetBenchOptionsIfNecessary
2734       $includeJSBench = true
2735     when '--js-regress'
2736       resetBenchOptionsIfNecessary
2737       $includeJSRegress = true
2738     when '--asm-bench'
2739       resetBenchOptionsIfNecessary
2740       $includeAsmBench = true
2741     when '--dsp'
2742       resetBenchOptionsIfNecessary
2743       $includeDSPJS = true
2744     when '--browsermark-js'
2745       resetBenchOptionsIfNecessary
2746       $includeBrowsermarkJS = true
2747     when '--browsermark-dom'
2748       resetBenchOptionsIfNecessary
2749       $includeBrowsermarkDOM = true
2750     when '--octane'
2751       resetBenchOptionsIfNecessary
2752       $includeOctane = true
2753     when '--compression-bench'
2754       resetBenchOptionsIfNecessary
2755       $includeCompressionBench = true
2756     when '--benchmarks'
2757       $benchmarkPattern = Regexp.new(arg)
2758     when '--measure-gc'
2759       if arg == ''
2760         $measureGC = true
2761       else
2762         $measureGC = arg
2763       end
2764     when '--verbose'
2765       $verbosity += 1
2766     when '--brief'
2767       $brief = true
2768     when '--silent'
2769       $silent = true
2770     when '--remote'
2771       $remoteHosts += arg.split(',')
2772       $needToCopyVMs = true
2773     when '--ssh-options'
2774       $sshOptions << arg
2775     when '--local'
2776       $alsoLocal = true
2777     when '--prepare-only'
2778       $run = false
2779     when '--analyze'
2780       $prepare = false
2781       $run = false
2782       $analyze << arg
2783     when '--output-name'
2784       $outputName = arg
2785     when '--vms'
2786       JSON::parse(IO::read(arg)).each {
2787         | vmDescription |
2788         path = Pathname.new(vmDescription["path"]).realpath
2789         if vmDescription["name"]
2790           name = vmDescription["name"]
2791           nameKind = :given
2792         else
2793           name = "Conf\##{$vms.length+1}"
2794           nameKind = :auto
2795         end
2796         vm = VM.new(path, name, nameKind, nil)
2797         if vmDescription["env"]
2798           vmDescription["env"].each_pair {
2799             | key, val |
2800             vm.addExtraEnv(key, val)
2801           }
2802         end
2803         $vms << vm
2804       }
2805     when '--environment'
2806       $environment = JSON::parse(IO::read(arg))
2807     when '--config'
2808       $configPath = Pathname.new(arg)
2809     when '--help'
2810       usage
2811     else
2812       raise "bad option: #{opt}"
2813     end
2814   }
2815   
2816   # Figure out the configuration
2817   if $configPath.file?
2818     config = JSON::parse(IO::read($configPath.to_s))
2819   else
2820     config = {}
2821   end
2822
2823   def pathname_if_exist config
2824     if config
2825       Pathname.new config
2826     else
2827       config
2828     end
2829   end
2830
2831   OCTANE_PATH = pathname_if_exist(config["OctanePath"])
2832   BROWSERMARK_PATH = pathname_if_exist(config["BrowserMarkPath"])
2833   BROWSERMARK_JS_PATH = pathname_if_exist(config["BrowserMarkJSPath"])
2834   BROWSERMARK_DOM_PATH = pathname_if_exist(config["BrowserMarkDOMPath"])
2835   ASMBENCH_PATH = pathname_if_exist(config["AsmBenchPath"])
2836   COMPRESSIONBENCH_PATH = pathname_if_exist(config["CompressionBenchPath"])
2837   DSPJS_FILTRR_PATH = pathname_if_exist(config["DSPJSFiltrrPath"])
2838   DSPJS_ROUTE9_PATH = pathname_if_exist(config["DSPJSRoute9Path"])
2839   DSPJS_STARFIELD_PATH = pathname_if_exist(config["DSPJSStarfieldPath"])
2840   DSPJS_QUAKE3_PATH = pathname_if_exist(config["DSPJSQuake3Path"])
2841   DSPJS_MANDELBROT_PATH = pathname_if_exist(config["DSPJSMandelbrotPath"])
2842   DSPJS_JSLINUX_PATH = pathname_if_exist(config["DSPJSLinuxPath"])
2843   DSPJS_AMMOJS_ASMJS_PATH = pathname_if_exist(config["DSPJSAmmoJSAsmJSPath"])
2844   DSPJS_AMMOJS_REGULAR_PATH = pathname_if_exist(config["DSPJSAmmoJSRegularPath"])
2845   JSBENCH_PATH = pathname_if_exist(config["JSBenchPath"])
2846   KRAKEN_PATH = pathname_if_exist(config["KrakenPath"])
2847   
2848   # If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option.
2849   if $dontCopyVMs
2850     $needToCopyVMs = false
2851   end
2852   
2853   ARGV.each {
2854     | vm |
2855     if vm =~ /([a-zA-Z0-9_ .]+):/
2856       name = $1
2857       nameKind = :given
2858       vm = $~.post_match
2859     else
2860       name = "Conf\##{$vms.length+1}"
2861       nameKind = :auto
2862     end
2863     envs = []
2864     while vm =~ /([a-zA-Z0-9_]+)=([a-zA-Z0-9_:.\/-]+):/
2865       envs << [$1, $2]
2866       vm = $~.post_match
2867     end
2868     $stderr.puts "#{name}: #{vm}" if $verbosity >= 1
2869     vm = VM.new(Pathname.new(vm).realpath, name, nameKind, nil)
2870     envs.each {
2871       | pair |
2872       vm.addExtraEnv(pair[0], pair[1])
2873     }
2874     $vms << vm
2875   }
2876   
2877   if $vms.empty?
2878     quickFail("Please specify at least on configuraiton on the command line.",
2879               "Insufficient arguments")
2880   end
2881   
2882   $vms.each {
2883     | vm |
2884     if vm.vmType == :jsc
2885       $allDRT = false
2886     end
2887   }
2888   
2889   SUNSPIDER = BenchmarkSuite.new("SunSpider", :arithmeticMean, 0)
2890   WARMUP = BenchmarkSuite.new("WARMUP", :arithmeticMean, 0)
2891   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
2892    "access-fannkuch", "access-nbody", "access-nsieve",
2893    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
2894    "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
2895    "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
2896    "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
2897    "string-base64", "string-fasta", "string-tagcloud",
2898    "string-unpack-code", "string-validate-input"].each {
2899     | name |
2900     SUNSPIDER.add SunSpiderBenchmark.new(name)
2901     WARMUP.addIgnoringPattern SunSpiderBenchmark.new(name)
2902   }
2903
2904   LONGSPIDER = BenchmarkSuite.new("LongSpider", :geometricMean, 0)
2905   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
2906    "access-fannkuch", "access-nbody", "access-nsieve",
2907    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-nsieve-bits",
2908    "controlflow-recursive", "crypto-aes", "crypto-md5", "crypto-sha1",
2909    "date-format-tofte", "date-format-xparb", "hash-map", "math-cordic",
2910    "math-partial-sums", "math-spectral-norm", "string-base64",
2911    "string-fasta", "string-tagcloud"].each {
2912     | name |
2913     LONGSPIDER.add LongSpiderBenchmark.new(name)
2914   }
2915
2916   V8 = BenchmarkSuite.new("V8Spider", :geometricMean, 0)
2917   ["crypto", "deltablue", "earley-boyer", "raytrace",
2918    "regexp", "richards", "splay"].each {
2919     | name |
2920     V8.add V8Benchmark.new(name)
2921   }
2922
2923   OCTANE = BenchmarkSuite.new("Octane", :geometricMean, 1)
2924   [[["crypto"], "encrypt", 1, true, false, 32],
2925    [["crypto"], "decrypt", 1, true, false, 32],
2926    [["deltablue"], "deltablue", 2, true, false, 32],
2927    [["earley-boyer"], "earley", 1, true, false, 32],
2928    [["earley-boyer"], "boyer", 1, true, false, 32],
2929    [["navier-stokes"], "navier-stokes", 2, true, false, 16],
2930    [["raytrace"], "raytrace", 2, true, false, 32],
2931    [["richards"], "richards", 2, true, false, 32],
2932    [["splay"], "splay", 2, true, false, 32],
2933    [["regexp"], "regexp", 2, true, false, 16],
2934    [["pdfjs"], "pdfjs", 2, false, false, 4],
2935    [["mandreel"], "mandreel", 2, false, false, 4],
2936    [["gbemu-part1", "gbemu-part2"], "gbemu", 2, false, false, 4],
2937    [["code-load"], "closure", 1, false, false, 16],
2938    [["code-load"], "jquery", 1, false, false, 16],
2939    [["box2d"], "box2d", 2, false, false, 8],
2940    [["zlib", "zlib-data"], "zlib", 2, false, true, 3],
2941    [["typescript", "typescript-input", "typescript-compiler"], "typescript", 2, false, true, 1]].each {
2942     | args |
2943     OCTANE.add OctaneBenchmark.new(*args)
2944   }
2945
2946   KRAKEN = BenchmarkSuite.new("Kraken", :arithmeticMean, -1)
2947   ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
2948    "audio-oscillator", "imaging-darkroom", "imaging-desaturate",
2949    "imaging-gaussian-blur", "json-parse-financial",
2950    "json-stringify-tinderbox", "stanford-crypto-aes",
2951    "stanford-crypto-ccm", "stanford-crypto-pbkdf2",
2952    "stanford-crypto-sha256-iterative"].each {
2953     | name |
2954     KRAKEN.add KrakenBenchmark.new(name)
2955   }
2956   
2957   JSBENCH = BenchmarkSuite.new("JSBench", :arithmeticMean, 0)
2958   [["amazon-chrome", "urem"], ["amazon-chrome-win", "urem"], ["amazon-firefox", "urm"],
2959    ["amazon-firefox-win", "urm"], ["amazon-safari", "urem"], ["facebook-chrome", "urem"],
2960    ["facebook-chrome-win", "urem"], ["facebook-firefox", "urem"],
2961    ["facebook-firefox-win", "urem"], ["facebook-safari", "urem"],
2962    ["google-chrome", "urem"], ["google-chrome-win", "urem"], ["google-firefox", "uem"],
2963    ["google-firefox-win", "urem"], ["google-safari", "urem"], ["twitter-chrome", "urem"],
2964    ["twitter-chrome-win", "rem"], ["twitter-firefox", "urem"],
2965    ["twitter-firefox-win", "urem"], ["twitter-safari", "urem"], ["yahoo-chrome", "urem"],
2966    ["yahoo-chrome-win", "urem"], ["yahoo-firefox", "urem"], ["yahoo-firefox-win", "urem"],
2967    ["yahoo-safari", "urem"]].each {
2968     | nameAndMode |
2969     JSBENCH.add JSBenchBenchmark.new(*nameAndMode)
2970   }
2971   
2972   JSREGRESS = BenchmarkSuite.new("JSRegress", :geometricMean, 0)
2973   Dir.foreach(JSREGRESS_PATH) {
2974     | filename |
2975     if filename =~ /\.js$/
2976       name = $~.pre_match
2977       JSREGRESS.add JSRegressBenchmark.new(name)
2978     end
2979   }
2980   
2981   ASMBENCH = BenchmarkSuite.new("AsmBench", :geometricMean, 0)
2982   if ASMBENCH_PATH
2983     Dir.foreach(ASMBENCH_PATH) {
2984       | filename |
2985       if filename =~ /\.js$/
2986         name = $~.pre_match
2987         ASMBENCH.add AsmBenchBenchmark.new(name)
2988       end
2989     }
2990   end
2991
2992   COMPRESSIONBENCH = BenchmarkSuite.new("CompressionBench", :geometricMean, 0)
2993   [[["huffman", "compression-data"], "huffman", ""],
2994    [["arithmetic", "compression-data"], "arithmetic", "Simple"],
2995    [["arithmetic", "compression-data"], "arithmetic", "Precise"],
2996    [["arithmetic", "compression-data"], "arithmetic", "Complex Precise"],
2997    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 0"],
2998    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 1"],
2999    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 2"],
3000    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 1"],
3001    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 2"],
3002    [["lz-string", "compression-data"], "lz-string", ""]
3003   ].each {
3004     | args |
3005     COMPRESSIONBENCH.add CompressionBenchBenchmark.new(*args)
3006   }
3007
3008   DSPJS = BenchmarkSuite.new("DSP", :geometricMean, 0)
3009   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-posterize-tint", "e2")
3010   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-contrast-sat-bright", "e5")
3011   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-sat-adj-contr-mult", "e7")
3012   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-blur-overlay-sat-contr", "e8")
3013   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sat-blur-mult-sharpen-contr", "e9")
3014   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sepia-bias", "e10")
3015   DSPJS.add DSPJSVP8Benchmark.new
3016   DSPJS.add DSPStarfieldBenchmark.new
3017   DSPJS.add DSPJSJSLinuxBenchmark.new
3018   DSPJS.add DSPJSQuake3Benchmark.new
3019   DSPJS.add DSPJSMandelbrotBenchmark.new
3020   DSPJS.add DSPJSAmmoJSASMBenchmark.new
3021   DSPJS.add DSPJSAmmoJSRegularBenchmark.new
3022
3023   BROWSERMARK_JS = BenchmarkSuite.new("BrowsermarkJS", :geometricMean, 1)
3024   ["array_blur", "array_weighted", "string_chat", "string_filter", "string_weighted"].each {
3025     | name |
3026       BROWSERMARK_JS.add BrowsermarkJSBenchmark.new(name)
3027   }
3028   
3029   BROWSERMARK_DOM = BenchmarkSuite.new("BrowsermarkDOM", :geometricMean, 1)
3030   ["advanced_search", "create_source", "dynamic_create", "search"].each {
3031     | name |
3032       BROWSERMARK_DOM.add BrowsermarkDOMBenchmark.new(name)
3033   }
3034
3035   $suites = []
3036   
3037   if $includeSunSpider and not SUNSPIDER.empty?
3038     $suites << SUNSPIDER
3039   end
3040   
3041   if $includeLongSpider and not LONGSPIDER.empty?
3042     $suites << LONGSPIDER
3043   end
3044   
3045   if $includeV8 and not V8.empty?
3046     $suites << V8
3047   end
3048   
3049   if $includeOctane and not OCTANE.empty?
3050     if OCTANE_PATH
3051       $suites << OCTANE
3052     else
3053       $stderr.puts "Warning: refusing to run Octane because \"OctanePath\" isn't set in #{$configPath}."
3054     end
3055   end
3056   
3057   if $includeKraken and not KRAKEN.empty?
3058     if KRAKEN_PATH
3059       $suites << KRAKEN
3060     else
3061       $stderr.puts "Warning: refusing to run Kraken because \"KrakenPath\" isn't set in #{$configPath}."
3062     end
3063   end
3064   
3065   if $includeJSBench and not JSBENCH.empty?
3066     if $allDRT
3067       if JSBENCH_PATH
3068         $suites << JSBENCH
3069       else
3070         $stderr.puts "Warning: refusing to run JSBench because \"JSBenchPath\" isn't set in #{$configPath}"
3071       end
3072     else
3073       $stderr.puts "Warning: refusing to run JSBench because not all VMs are DumpRenderTree or WebKitTestRunner."
3074     end
3075   end
3076   
3077   if $includeJSRegress and not JSREGRESS.empty?
3078     $suites << JSREGRESS
3079   end
3080   
3081   if $includeAsmBench and not ASMBENCH.empty?
3082     if ASMBENCH_PATH
3083       $suites << ASMBENCH
3084     else
3085       $stderr.puts "Warning: refusing to run AsmBench because \"AsmBenchPath\" isn't set in #{$configPath}."
3086     end
3087   end
3088   
3089   if $includeDSPJS and not DSPJS.empty?
3090     if $allDRT
3091       if DSPJS_FILTRR_PATH and DSPJS_ROUTE9_PATH and DSPJS_STARFIELD_PATH and DSPJS_QUAKE3_PATH and DSPJS_MANDELBROT_PATH and DSPJS_JSLINUX_PATH and DSPJS_AMMOJS_ASMJS_PATH and DSPJS_AMMOJS_REGULAR_PATH
3092         $suites << DSPJS
3093       else
3094         $stderr.puts "Warning: refusing to run DSPJS because one of the following isn't set in #{$configPath}: \"DSPJSFiltrrPath\", \"DSPJSRoute9Path\", \"DSPJSStarfieldPath\", \"DSPJSQuake3Path\", \"DSPJSMandelbrotPath\", \"DSPJSLinuxPath\", \"DSPJSAmmoJSAsmJSPath\", \"DSPJSAmmoJSRegularPath\"."
3095       end
3096     else
3097       $stderr.puts "Warning: refusing to run DSPJS because not all VMs are DumpRenderTree or WebKitTestRunner."
3098     end
3099   end
3100   
3101   if $includeBrowsermarkJS and not BROWSERMARK_JS.empty?
3102     if BROWSERMARK_PATH and BROWSERMARK_JS_PATH
3103       $suites << BROWSERMARK_JS
3104     else
3105       $stderr.puts "Warning: refusing to run Browsermark-JS because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\" or \"BrowserMarkJSPath\"."
3106     end
3107   end
3108
3109   if $includeBrowsermarkDOM and not BROWSERMARK_DOM.empty?
3110     if $allDRT
3111       if BROWSERMARK_PATH and BROWSERMARK_JS_PATH and BROWSERMARK_DOM_PATH
3112         $suites << BROWSERMARK_DOM
3113       else
3114         $stderr.puts "Warning: refusing to run Browsermark-DOM because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\", \"BrowserMarkJSPath\", or \"BrowserMarkDOMPath\"."
3115       end
3116     else
3117       $stderr.puts "Warning: refusing to run Browsermark-DOM because not all VMs are DumpRenderTree or WebKitTestRunner."
3118     end
3119   end
3120
3121   if $includeCompressionBench and not COMPRESSIONBENCH.empty?
3122     if COMPRESSIONBENCH_PATH
3123       $suites << COMPRESSIONBENCH
3124     else
3125       $stderr.puts "Warning: refusing to run CompressionBench because \"CompressionBenchPath\" isn't set in #{$configPath}"
3126     end
3127   end
3128
3129   $allSuites = $suites.map{|v| v.suites}.flatten(1)
3130   
3131   $benchmarks = []
3132   $suites.each {
3133     | suite |
3134     $benchmarks += suite.benchmarks
3135   }
3136   
3137   if $suites.empty? or $benchmarks.empty?
3138     $stderr.puts "No benchmarks found.  Bailing out."
3139     exit 1
3140   end
3141   
3142   if $outer*$inner == 1
3143     $stderr.puts "Warning: will only collect one sample per benchmark/VM.  Confidence interval calculation will fail."
3144   end
3145   
3146   $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
3147   
3148   $runPlans = []
3149   $vms.each {
3150     | vm |
3151     $benchmarks.each {
3152       | benchmark |
3153       $outer.times {
3154         | iteration |
3155         $runPlans << BenchRunPlan.new(benchmark, vm, iteration)
3156       }
3157     }
3158   }
3159   
3160   $runPlans.shuffle!
3161   
3162   if $sunSpiderWarmup
3163     warmupPlans = []
3164     $vms.each {
3165       | vm |
3166       WARMUP.benchmarks.each {
3167         | benchmark |
3168         warmupPlans << BenchRunPlan.new(benchmark, vm, 0)
3169       }
3170     }
3171     
3172     $runPlans = warmupPlans.shuffle + $runPlans
3173   end
3174   
3175   $suitepad = $suites.collect {
3176     | suite |
3177     suite.to_s.size
3178   }.max + 1
3179   
3180   $planpad = $runPlans.collect {
3181     | plan |
3182     plan.to_s.size
3183   }.max + 1
3184   
3185   maxBenchNameLength =
3186     ($benchmarks + ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
3187     | benchmark |
3188     if benchmark.respond_to? :name
3189       benchmark.name.size
3190     else
3191       benchmark.size
3192     end
3193   }.max
3194   $benchNameClip = 40
3195   $benchpad = [maxBenchNameLength, $benchNameClip].min + 1
3196   
3197   $weightpad = $benchmarks.collect {
3198     | benchmark |
3199     benchmark.weightString.size
3200   }.max
3201
3202   $vmpad = $vms.collect {
3203     | vm |
3204     vm.to_s.size
3205   }.max + 1
3206   
3207   $analyze.each_with_index {
3208     | filename, index |
3209     if index >= 1
3210       puts
3211     end
3212     parseAndDisplayResults(IO::read(filename))
3213   }
3214   
3215   if not $prepare and not $run
3216     exit 0
3217   end
3218   
3219   if FileTest.exist? BENCH_DATA_PATH
3220     cmd = "rm -rf #{BENCH_DATA_PATH}"
3221     $stderr.puts ">> #{cmd}" if $verbosity >= 2
3222     raise unless system cmd
3223   end
3224   
3225   Dir.mkdir BENCH_DATA_PATH
3226   
3227   if $needToCopyVMs
3228     canCopyIntoBenchPath = true
3229     $vms.each {
3230       | vm |
3231       canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath
3232     }
3233     
3234     if canCopyIntoBenchPath
3235       $vms.each {
3236         | vm |
3237         $stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..."
3238         vm.copyIntoBenchPath
3239       }
3240       $stderr.puts "All VMs are in place."
3241     else
3242       $stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it."
3243     end
3244   end
3245   
3246   if $measureGC and $measureGC != true
3247     found = false
3248     $vms.each {
3249       | vm |
3250       if vm.name == $measureGC
3251         found = true
3252       end
3253     }
3254     unless found
3255       $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
3256     end
3257   end
3258   
3259   if $prepare
3260     File.open("#{BENCH_DATA_PATH}/runscript", "w") {
3261       | file |
3262       file.puts "echo \"HOSTNAME:\\c\""
3263       file.puts "hostname"
3264       file.puts "echo"
3265       file.puts "echo \"HARDWARE:\\c\""
3266       file.puts "/usr/sbin/sysctl hw.model"
3267       file.puts "echo"
3268       file.puts "set -e"
3269       $script = file
3270       $runPlans.each_with_index {
3271         | plan, idx |
3272         if $verbosity == 0 and not $silent
3273           text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s
3274           text2 = plan.to_s
3275           file.puts("echo " + Shellwords.shellescape("\r#{text1} #{rpad(text2,$planpad)}") + "\"\\c\" 1>&2")
3276           file.puts("echo " + Shellwords.shellescape("\r#{text1} #{text2}") + "\"\\c\" 1>&2")
3277         end
3278         plan.emitRunCode
3279       }
3280       if $verbosity == 0 and not $silent
3281         file.puts("echo " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}") + "\"\\c\" 1>&2")
3282         file.puts("echo " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size}") + " 1>&2")
3283       end
3284     }
3285   end
3286   
3287   if $run
3288     unless $remoteHosts.empty?
3289       $stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0
3290       Dir.chdir(TEMP_PATH) {
3291         cmd = "tar -czf payload.tar.gz benchdata"
3292         $stderr.puts ">> #{cmd}" if $verbosity>=2
3293         raise unless system(cmd)
3294       }
3295       
3296       def grokHost(host)
3297         if host =~ /:([0-9]+)$/
3298           "-p " + $1 + " " + Shellwords.shellescape($~.pre_match)
3299         else
3300           Shellwords.shellescape(host)
3301         end
3302       end
3303       
3304       def sshRead(host, command)
3305         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3306         $stderr.puts ">> #{cmd}" if $verbosity>=2
3307         result = ""
3308         IO.popen(cmd, "r") {
3309           | inp |
3310           inp.each_line {
3311             | line |
3312             $stderr.puts "#{host}: #{line}" if $verbosity>=2
3313             result += line
3314           }
3315         }
3316         raise "#{$?}" unless $?.success?
3317         result
3318       end
3319       
3320       def sshWrite(host, command, data)
3321         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3322         $stderr.puts ">> #{cmd}" if $verbosity>=2
3323         IO.popen(cmd, "w") {
3324           | outp |
3325           outp.write(data)
3326         }
3327         raise "#{$?}" unless $?.success?
3328       end
3329       
3330       $remoteHosts.each {
3331         | host |
3332         $stderr.puts "Sending benchmark payload to #{host}..." if $verbosity==0
3333         
3334         remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"]
3335         raise unless remoteTempPath
3336         
3337         sshWrite(host, "cd #{Shellwords.shellescape(remoteTempPath)} && rm -rf benchdata && tar -xz", IO::read("#{TEMP_PATH}/payload.tar.gz"))
3338         
3339         $stderr.puts "Running on #{host}..." if $verbosity==0
3340         
3341         parseAndDisplayResults(sshRead(host, "cd #{Shellwords.shellescape(remoteTempPath + '/benchdata')} && sh runscript"))
3342       }
3343     end
3344     
3345     if not $remoteHosts.empty? and $alsoLocal
3346       $stderr.puts "Running locally..."
3347     end
3348     
3349     if $remoteHosts.empty? or $alsoLocal
3350       parseAndDisplayResults(runAndGetResults)
3351     end
3352   end
3353   
3354   if $prepare and not $run and $analyze.empty?
3355     puts wrap("Benchmarking script and data are in #{BENCH_DATA_PATH}. You can run "+
3356               "the benchmarks and get the results by doing:", 78)
3357     puts
3358     puts "cd #{BENCH_DATA_PATH}"
3359     puts "sh runscript > results.txt"
3360     puts
3361     puts wrap("Then you can analyze the results by running bencher with the same arguments "+
3362               "as now, but replacing --prepare-only with --analyze results.txt.", 78)
3363   end
3364 rescue => e
3365   fail(e)
3366 end
3367