It should be fun and easy to run every possible JavaScript benchmark from the command...
[WebKit-https.git] / Tools / Scripts / run-jsc-benchmarks
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2011, 2012, 2013, 2014 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 # 1. Redistributions of source code must retain the above copyright
9 #    notice, this list of conditions and the following disclaimer.
10 # 2. Redistributions in binary form must reproduce the above copyright
11 #    notice, this list of conditions and the following disclaimer in the
12 #    documentation and/or other materials provided with the distribution.
13 #
14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24 # THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'rubygems'
27
28 require 'getoptlong'
29 require 'pathname'
30 require 'shellwords'
31 require 'socket'
32
33 begin
34   require 'json'
35 rescue LoadError => e
36   $stderr.puts "It does not appear that you have the 'json' package installed.  Try running 'sudo gem install json'."
37   exit 1
38 end
39
40 SCRIPT_PATH = Pathname.new(__FILE__).realpath
41
42 raise unless SCRIPT_PATH.dirname.basename.to_s == "Scripts"
43 raise unless SCRIPT_PATH.dirname.dirname.basename.to_s == "Tools"
44
45 OPENSOURCE_PATH = SCRIPT_PATH.dirname.dirname.dirname
46
47 SUNSPIDER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "SunSpider" + "tests" + "sunspider-1.0"
48 LONGSPIDER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "LongSpider"
49 V8_PATH = OPENSOURCE_PATH + "PerformanceTests" + "SunSpider" + "tests" + "v8-v6"
50 JSREGRESS_PATH = OPENSOURCE_PATH + "LayoutTests" + "js" + "regress" + "script-tests"
51 OCTANE_WRAPPER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "Octane" + "wrappers"
52
53 TEMP_PATH = OPENSOURCE_PATH + "BenchmarkTemp"
54
55 if TEMP_PATH.exist?
56   raise unless TEMP_PATH.directory?
57 else
58   Dir.mkdir(TEMP_PATH)
59 end
60
61 BENCH_DATA_PATH = TEMP_PATH + "benchdata"
62
63 IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933, 
64             0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684, 
65             0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227, 
66             0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674, 
67             0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652, 
68             0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604, 
69             0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338, 
70             0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538, 
71             0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185, 
72             0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694, 
73             0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244, 
74             0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067, 
75             0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323, 
76             0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129, 
77             0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574, 
78             0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722, 
79             0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624, 
80             0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321, 
81             0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843, 
82             0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217, 
83             0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464, 
84             0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599, 
85             0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638, 
86             0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592, 
87             0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471, 
88             0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283, 
89             0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037, 
90             0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738, 
91             0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481, 
92             0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085, 
93             0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651, 
94             0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183, 
95             0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683, 
96             0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154, 
97             0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599, 
98             0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019, 
99             0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417, 
100             0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795, 
101             0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153, 
102             0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494, 
103             0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818, 
104             0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128, 
105             0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423, 
106             0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704, 
107             0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974, 
108             0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232, 
109             0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479, 
110             0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716, 
111             0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943, 
112             0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192, 
113             0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401, 
114             0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963, 
115             0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822, 
116             0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007, 
117             0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186, 
118             0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358, 
119             0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525, 
120             0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686, 
121             0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841, 
122             0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991, 
123             0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137, 
124             0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278, 
125             0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414, 
126             0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547, 
127             0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675, 
128             0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799, 
129             0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192, 
130             0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037, 
131             0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151, 
132             0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262, 
133             0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369, 
134             0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489, 
135             0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259, 
136             0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689, 
137             0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785, 
138             0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879, 
139             0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297, 
140             0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059, 
141             0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145, 
142             0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242, 
143             0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324, 
144             0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404, 
145             0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483, 
146             0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559, 
147             0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634, 
148             0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707, 
149             0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778, 
150             0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848, 
151             0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916, 
152             0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983, 
153             0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057, 
154             0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121, 
155             0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183, 
156             0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244, 
157             0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304, 
158             0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363, 
159             0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442, 
160             0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476, 
161             0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539, 
162             0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946, 
163             0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653, 
164             0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712, 
165             0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762, 
166             0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812, 
167             0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867, 
168             0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915, 
169             0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962, 
170             0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007, 
171             0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059, 
172             0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103, 
173             0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146, 
174             0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189, 
175             0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231, 
176             0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272, 
177             0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312, 
178             0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352, 
179             0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391, 
180             0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429, 
181             0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467, 
182             0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504, 
183             0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546, 
184             0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582, 
185             0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617, 
186             0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652, 
187             0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686, 
188             0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724, 
189             0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757, 
190             0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579, 
191             0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822, 
192             0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853, 
193             0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885, 
194             0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915, 
195             0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595, 
196             0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979, 
197             0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013, 
198             0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041, 
199             0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074, 
200             0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102, 
201             0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129, 
202             0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156, 
203             0.99616, 0.996164]
204
205 # Run-time configuration parameters (can be set with command-line options)
206
207 $rerun=1
208 $inner=1
209 $warmup=1
210 $outer=4
211 $quantum=1000
212 $includeSunSpider=true
213 $includeLongSpider=true
214 $includeV8=true
215 $includeKraken=true
216 $includeJSBench=true
217 $includeJSRegress=true
218 $includeAsmBench=true
219 $includeDSPJS=true
220 $includeBrowsermarkJS=false
221 $includeBrowsermarkDOM=false
222 $includeOctane=true
223 $includeCompressionBench = true
224 $measureGC=false
225 $benchmarkPattern=nil
226 $verbosity=0
227 $timeMode=:preciseTime
228 $forceVMKind=nil
229 $brief=false
230 $silent=false
231 $remoteHosts=[]
232 $alsoLocal=false
233 $sshOptions=[]
234 $vms = []
235 $environment = {}
236 $needToCopyVMs = false
237 $dontCopyVMs = false
238 $allDRT = true
239 $outputName = nil
240 $sunSpiderWarmup = true
241 $configPath = Pathname.new(ENV["HOME"]) + ".run-jsc-benchmarks"
242
243 $prepare = true
244 $run = true
245 $analyze = []
246
247 # Helpful functions and classes
248
249 def smallUsage
250   puts "Use the --help option to get basic usage information."
251   exit 1
252 end
253
254 def usage
255   puts "run-jsc-benchmarks [options] <vm1> [<vm2> ...]"
256   puts
257   puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
258   puts "benchmarks, and reports detailed statistics.  What makes run-jsc-benchmarks"
259   puts "special is that each benchmark/VM configuration is run in a single VM invocation,"
260   puts "and the invocations are run in random order.  This minimizes systematics due to"
261   puts "one benchmark polluting the running time of another.  The fine-grained"
262   puts "interleaving of VM invocations further minimizes systematics due to changes in"
263   puts "the performance or behavior of your machine."
264   puts 
265   puts "Run-jsc-benchmarks is highly configurable.  You can compare as many VMs as you"
266   puts "like.  You can change the amount of warm-up iterations, number of iterations"
267   puts "executed per VM invocation, and the number of VM invocations per benchmark."
268   puts
269   puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
270   puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
271   puts "the executable and <name> is the name that you would like to give the"
272   puts "configuration for the purposeof reporting.  If no name is given, a generic name"
273   puts "of the form Conf#<n> will be ascribed to the configuration automatically."
274   puts
275   puts "It's also possible to specify per-VM environment variables. For example, you"
276   puts "might specify a VM like Foo:JSC_useJIT=false:/path/to/jsc, in which case the"
277   puts "harness will set the JSC_useJIT environment variable to false just before running"
278   puts "the given VM. Note that the harness will not unset the environment variable, so"
279   puts "you must ensure that your other VMs will use the opposite setting"
280   puts "(JSC_useJIT=true in this case)."
281   puts
282   puts "Options:"
283   puts "--rerun <n>          Set the number of iterations of the benchmark that"
284   puts "                     contribute to the measured run time.  Default is #{$rerun}."
285   puts "--inner <n>          Set the number of inner (per-runtime-invocation)"
286   puts "                     iterations.  Default is #{$inner}."
287   puts "--outer <n>          Set the number of runtime invocations for each benchmark."
288   puts "                     Default is #{$outer}."
289   puts "--warmup <n>         Set the number of warm-up runs per invocation.  Default"
290   puts "                     is #{$warmup}. This has a different effect on different kinds"
291   puts "                     benchmarks. Some benchmarks have no notion of warm-up."
292   puts "--no-ss-warmup       Disable SunSpider-based warm-up runs."
293   puts "--quantum <n>        Set the duration in milliseconds for which an iteration of"
294   puts "                     a throughput benchmark should be run.  Default is #{$quantum}."
295   puts "--timing-mode        Set the way that time is measured.  Possible values"
296   puts "                     are 'preciseTime' and 'date'.  Default is 'preciseTime'."
297   puts "--force-vm-kind      Turn off auto-detection of VM kind, and assume that it is"
298   puts "                     the one specified.  Valid arguments are 'jsc', "
299   puts "                     'DumpRenderTree', or 'WebKitTestRunner'."
300   puts "--force-vm-copy      Force VM builds to be copied to the working directory."
301   puts "                     This may reduce pathologies resulting from path names."
302   puts "--dont-copy-vms      Don't copy VMs even when doing a remote benchmarking run;"
303   puts "                     instead assume that they are already there."
304   puts "--sunspider          Only run SunSpider."
305   puts "--v8-spider          Only run V8."
306   puts "--kraken             Only run Kraken."
307   puts "--js-bench           Only run JSBench."
308   puts "--js-regress         Only run JSRegress."
309   puts "--dsp                Only run DSP."
310   puts "--asm-bench          Only run AsmBench."
311   puts "--browsermark-js     Only run browsermark-js."
312   puts "--browsermark-dom    Only run browsermark-dom."
313   puts "--octane             Only run Octane."
314   puts "--compression-bench  Only run compression bench"
315   puts "                     The default is to run all benchmarks. The above options can"
316   puts "                     be combined to run any subset (so --sunspider --dsp will run"
317   puts "                     both SunSpider and DSP)."
318   puts "--benchmarks         Only run benchmarks matching the given regular expression."
319   puts "--measure-gc         Turn off manual calls to gc(), so that GC time is measured."
320   puts "                     Works best with large values of --inner.  You can also say"
321   puts "                     --measure-gc <conf>, which turns this on for one"
322   puts "                     configuration only."
323   puts "--verbose or -v      Print more stuff."
324   puts "--brief              Print only the final result for each VM."
325   puts "--silent             Don't print progress. This might slightly reduce some"
326   puts "                     performance perturbation."
327   puts "--remote <sshhosts>  Perform performance measurements remotely, on the given"
328   puts "                     SSH host(s). Easiest way to use this is to specify the SSH"
329   puts "                     user@host string. However, you can also supply a comma-"
330   puts "                     separated list of SSH hosts. Alternatively, you can use this"
331   puts "                     option multiple times to specify multiple hosts. This"
332   puts "                     automatically copies the WebKit release builds of the VMs"
333   puts "                     you specified to all of the hosts."
334   puts "--ssh-options        Pass additional options to SSH."
335   puts "--local              Also do a local benchmark run even when doing --remote."
336   puts "--vms                Use a JSON file to specify which VMs to run, as opposed to"
337   puts "                     specifying them on the command line."
338   puts "--prepare-only       Only prepare the runscript (a shell script that"
339   puts "                     invokes the VMs to run benchmarks) but don't run it."
340   puts "--analyze            Only read the output of the runscript but don't do anything"
341   puts "                     else. This requires passing the same arguments that you"
342   puts "                     passed when running --prepare-only."
343   puts "--output-name        Base of the filenames to put results into. Will write a file"
344   puts "                     called <base>_report.txt and <base>.json.  By default this"
345   puts "                     name is automatically synthesized from the machine name,"
346   puts "                     date, set of benchmarks run, and set of configurations."
347   puts "--environment        JSON file that specifies the environment variables that should"
348   puts "                     be used for particular VMs and benchmarks."
349   puts "--config <path>      Specify the path of the configuration file. Defaults to"
350   puts "                     ~/.run-jsc-benchmarks"
351   puts "--help or -h         Display this message."
352   puts
353   puts "Example:"
354   puts "run-jsc-benchmarks TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
355   exit 1
356 end
357
358 def fail(reason)
359   if reason.respond_to? :backtrace
360     puts "FAILED: #{reason.inspect}"
361     puts "Stack trace:"
362     puts reason.backtrace.join("\n")
363   else
364     puts "FAILED: #{reason.inspect}"
365   end
366   smallUsage
367 end
368
369 def quickFail(r1,r2)
370   $stderr.puts "#{$0}: #{r1}"
371   puts
372   fail(r2)
373 end
374
375 def intArg(argName,arg,min,max)
376   result=arg.to_i
377   unless result.to_s == arg
378     quickFail("Expected an integer value for #{argName}, but got #{arg}.",
379               "Invalid argument for command-line option")
380   end
381   if min and result<min
382     quickFail("Argument for #{argName} cannot be smaller than #{min}.",
383               "Invalid argument for command-line option")
384   end
385   if max and result>max
386     quickFail("Argument for #{argName} cannot be greater than #{max}.",
387               "Invalid argument for command-line option")
388   end
389   result
390 end
391
392 def computeMean(array)
393   sum=0.0
394   array.each {
395     | value |
396     sum += value
397   }
398   sum/array.length
399 end
400
401 def computeGeometricMean(array)
402   sum = 0.0
403   array.each {
404     | value |
405     sum += Math.log(value)
406   }
407   Math.exp(sum * (1.0/array.length))
408 end
409
410 def computeHarmonicMean(array)
411   1.0 / computeMean(array.collect{ | value | 1.0 / value })
412 end
413
414 def computeStdDev(array)
415   case array.length
416   when 0
417     0.0/0.0
418   when 1
419     0.0
420   else
421     begin
422       mean=computeMean(array)
423       sum=0.0
424       array.each {
425         | value |
426         sum += (value-mean)**2
427       }
428       Math.sqrt(sum/(array.length-1))
429     rescue
430       0.0/0.0
431     end
432   end
433 end
434
435 class Array
436   def shuffle!
437     size.downto(1) { |n| push delete_at(rand(n)) }
438     self
439   end
440 end
441
442 def inverseBetaRegularized(n)
443   IBR_LOOKUP[n-1]
444 end
445
446 def numToStr(num, decimalShift)
447   ("%." + (4 + decimalShift).to_s + "f") % (num.to_f)
448 end
449
450 class CantSay
451   def initialize
452   end
453   
454   def shortForm
455     " "
456   end
457   
458   def longForm
459     ""
460   end
461   
462   def to_s
463     ""
464   end
465 end
466   
467 class NoChange
468   attr_reader :amountFaster
469   
470   def initialize(amountFaster)
471     @amountFaster = amountFaster
472   end
473   
474   def shortForm
475     " "
476   end
477   
478   def longForm
479     "  might be #{numToStr(@amountFaster, 0)}x faster"
480   end
481   
482   def to_s
483     if @amountFaster < 1.01
484       ""
485     else
486       longForm
487     end
488   end
489 end
490
491 class Faster
492   attr_reader :amountFaster
493   
494   def initialize(amountFaster)
495     @amountFaster = amountFaster
496   end
497   
498   def shortForm
499     "^"
500   end
501   
502   def longForm
503     "^ definitely #{numToStr(@amountFaster, 0)}x faster"
504   end
505   
506   def to_s
507     longForm
508   end
509 end
510
511 class Slower
512   attr_reader :amountSlower
513   
514   def initialize(amountSlower)
515     @amountSlower = amountSlower
516   end
517   
518   def shortForm
519     "!"
520   end
521   
522   def longForm
523     "! definitely #{numToStr(@amountSlower, 0)}x slower"
524   end
525   
526   def to_s
527     longForm
528   end
529 end
530
531 class MayBeSlower
532   attr_reader :amountSlower
533   
534   def initialize(amountSlower)
535     @amountSlower = amountSlower
536   end
537   
538   def shortForm
539     "?"
540   end
541   
542   def longForm
543     "? might be #{numToStr(@amountSlower, 0)}x slower"
544   end
545   
546   def to_s
547     if @amountSlower < 1.01
548       "?"
549     else
550       longForm
551     end
552   end
553 end
554
555 def jsonSanitize(value)
556   if value.is_a? Fixnum
557     value
558   elsif value.is_a? Float
559     if value.nan? or value.infinite?
560       value.to_s
561     else
562       value
563     end
564   elsif value.is_a? Array
565     value.map{|v| jsonSanitize(v)}
566   elsif value.nil?
567     value
568   else
569     raise "Unrecognized value #{value.inspect}"
570   end
571 end
572
573 class Stats
574   def initialize
575     @array = []
576   end
577   
578   def add(value)
579     if not value or not @array
580       @array = nil
581     elsif value.is_a? Float
582       if value.nan? or value.infinite?
583         @array = nil
584       else
585         @array << value
586       end
587     elsif value.is_a? Stats
588       add(value.array)
589     elsif value.respond_to? :each
590       value.each {
591         | v |
592         add(v)
593       }
594     else
595       @array << value.to_f
596     end
597   end
598   
599   def status
600     if @array
601       :ok
602     else
603       :error
604     end
605   end
606   
607   def error?
608     # TODO: We're probably still not handling this case correctly. 
609     not @array or @array.empty?
610   end
611   
612   def ok?
613     not not @array
614   end
615     
616   def array
617     @array
618   end
619   
620   def sum
621     result=0
622     @array.each {
623       | value |
624       result += value
625     }
626     result
627   end
628   
629   def min
630     @array.min
631   end
632   
633   def max
634     @array.max
635   end
636   
637   def size
638     @array.length
639   end
640   
641   def mean
642     computeMean(array)
643   end
644   
645   def arithmeticMean
646     mean
647   end
648   
649   def stdDev
650     computeStdDev(array)
651   end
652
653   def stdErr
654     stdDev/Math.sqrt(size)
655   end
656   
657   # Computes a 95% Student's t distribution confidence interval
658   def confInt
659     if size < 2
660       0.0/0.0
661     else
662       raise if size > 1000
663       Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
664     end
665   end
666   
667   def lower
668     mean-confInt
669   end
670   
671   def upper
672     mean+confInt
673   end
674   
675   def geometricMean
676     computeGeometricMean(array)
677   end
678   
679   def harmonicMean
680     computeHarmonicMean(array)
681   end
682   
683   def compareTo(other)
684     return CantSay.new unless ok? and other.ok?
685     
686     if upper < other.lower
687       Faster.new(other.mean/mean)
688     elsif lower > other.upper
689       Slower.new(mean/other.mean)
690     elsif mean > other.mean
691       MayBeSlower.new(mean/other.mean)
692     else
693       NoChange.new(other.mean/mean)
694     end
695   end
696   
697   def to_s
698     "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
699   end
700   
701   def jsonMap
702     if ok?
703       {"data"=>jsonSanitize(@array), "mean"=>jsonSanitize(mean), "confInt"=>jsonSanitize(confInt)}
704     else
705       "ERROR"
706     end
707   end
708 end
709
710 def doublePuts(out1,out2,msg)
711   out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
712   out2.puts msg
713 end
714
715 class Benchfile < File
716   @@counter = 0
717   
718   attr_reader :filename, :basename
719   
720   def initialize(name)
721     @basename, @filename = Benchfile.uniqueFilename(name)
722     super(@filename, "w")
723   end
724   
725   def self.uniqueFilename(name)
726     if name.is_a? Array
727       basename = name[0] + @@counter.to_s + name[1]
728     else
729       basename = name + @@counter.to_s
730     end
731     filename = BENCH_DATA_PATH + basename
732     @@counter += 1
733     raise "Benchfile #{filename} already exists" if FileTest.exist?(filename)
734     [basename, filename]
735   end
736   
737   def self.create(name)
738     file = Benchfile.new(name)
739     yield file
740     file.close
741     file.basename
742   end
743 end
744
745 $dataFiles={}
746 def ensureFile(key, filename)
747   unless $dataFiles[key]
748     $dataFiles[key] = Benchfile.create(key) {
749       | outp |
750       doublePuts($stderr,outp,IO::read(filename))
751     }
752   end
753   $dataFiles[key]
754 end
755
756 # Helper for files that cannot be renamed.
757 $absoluteFiles={}
758 def ensureAbsoluteFile(filename, basedir=nil)
759   return if $absoluteFiles[filename]
760   filename = Pathname.new(filename)
761
762   directory = Pathname.new('')
763   if basedir and filename.dirname != basedir
764     remainingPath = filename.dirname
765     while remainingPath != basedir
766       directory = remainingPath.basename + directory
767       remainingPath = remainingPath.dirname
768     end
769     if not $absoluteFiles[directory]
770       cmd = "mkdir -p #{Shellwords.shellescape((BENCH_DATA_PATH + directory).to_s)}"
771       $stderr.puts ">> #{cmd}" if $verbosity >= 2
772       raise unless system(cmd)
773       intermediateDirectory = Pathname.new(directory)
774       while intermediateDirectory.basename.to_s != "."
775         $absoluteFiles[intermediateDirectory] = true
776         intermediateDirectory = intermediateDirectory.dirname
777       end
778     end
779   end
780   
781   cmd = "cp #{Shellwords.shellescape(filename.to_s)} #{Shellwords.shellescape((BENCH_DATA_PATH + directory + filename.basename).to_s)}"
782   $stderr.puts ">> #{cmd}" if $verbosity >= 2
783   raise unless system(cmd)
784   $absoluteFiles[filename] = true
785 end
786
787 # Helper for large benchmarks with lots of files and directories.
788 def ensureBenchmarkFiles(rootdir)
789     toProcess = [rootdir]
790     while not toProcess.empty?
791       currdir = toProcess.pop
792       Dir.foreach(currdir.to_s) {
793         | filename |
794         path = currdir + filename
795         next if filename.match(/^\./)
796         toProcess.push(path) if File.directory?(path.to_s)
797         ensureAbsoluteFile(path, rootdir) if File.file?(path.to_s)
798       }
799     end
800 end
801
802 class JSCommand
803   attr_reader :js, :html
804   def initialize(js, html)
805     @js = js
806     @html = html
807   end
808 end
809
810 def loadCommandForFile(key, filename)
811   file = ensureFile(key, filename)
812   JSCommand.new("load(#{file.inspect});", "<script src=#{file.inspect}></script>")
813 end
814
815 def simpleCommand(command)
816   JSCommand.new(command, "<script type=\"text/javascript\">#{command}</script>")
817 end
818
819 # Benchmark that consists of a single file and must be loaded in its own global object each
820 # time (i.e. run()).
821 class SingleFileTimedBenchmarkParameters
822   attr_reader :benchPath
823   
824   def initialize(benchPath)
825     @benchPath = benchPath
826   end
827   
828   def kind
829     :singleFileTimedBenchmark
830   end
831 end
832
833 # Benchmark that consists of one or more data files that should be loaded globally, followed
834 # by a command to run the benchmark.
835 class MultiFileTimedBenchmarkParameters
836   attr_reader :dataPaths, :command
837
838   def initialize(dataPaths, command)
839     @dataPaths = dataPaths
840     @command = command
841   end
842   
843   def kind
844     :multiFileTimedBenchmark
845   end
846 end
847
848 # Benchmark that consists of one or more data files that should be loaded globally, followed
849 # by a command to run a short tick of the benchmark. The benchmark should be run for as many
850 # ticks as possible, for one quantum (quantum is 1000ms by default).
851 class ThroughputBenchmarkParameters
852   attr_reader :dataPaths, :setUpCommand, :command, :tearDownCommand, :doWarmup, :deterministic, :minimumIterations
853
854   def initialize(dataPaths, setUpCommand, command, tearDownCommand, doWarmup, deterministic, minimumIterations)
855     @dataPaths = dataPaths
856     @setUpCommand = setUpCommand
857     @command = command
858     @tearDownCommand = tearDownCommand
859     @doWarmup = doWarmup
860     @deterministic = deterministic
861     @minimumIterations = minimumIterations
862   end
863   
864   def kind
865     :throughputBenchmark
866   end
867 end
868
869 # Benchmark that can only run in DumpRenderTree or WebKitTestRunner, that has its own callback for reporting
870 # results. Other than that it's just like SingleFileTimedBenchmark.
871 class SingleFileTimedCallbackBenchmarkParameters
872   attr_reader :callbackDecl, :benchPath
873   
874   def initialize(callbackDecl, benchPath)
875     @callbackDecl = callbackDecl
876     @benchPath = benchPath
877   end
878   
879   def kind
880     :singleFileTimedCallbackBenchmark
881   end
882 end
883
884 def emitTimerFunctionCode(file)
885   case $timeMode
886   when :preciseTime
887     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
888     doublePuts($stderr,file,"   return preciseTime()*1000")
889     doublePuts($stderr,file,"}")
890   when :date
891     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
892     doublePuts($stderr,file,"   return Date.now()")
893     doublePuts($stderr,file,"}")
894   else
895     raise
896   end
897 end
898
899 def emitBenchRunCodeFile(name, plan, benchParams)
900   case plan.vm.vmType
901   when :jsc
902     Benchfile.create("bencher") {
903       | file |
904       emitTimerFunctionCode(file)
905       
906       if benchParams.kind == :multiFileTimedBenchmark
907         benchParams.dataPaths.each {
908           | path |
909           doublePuts($stderr,file,"load(#{path.inspect});")
910         }
911         doublePuts($stderr,file,"gc();")
912         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
913         doublePuts($stderr,file,"   var __before = __bencher_curTimeMS();")
914         $rerun.times {
915           doublePuts($stderr,file,"   #{benchParams.command.js}")
916         }
917         doublePuts($stderr,file,"   var __after = __bencher_curTimeMS();")
918         doublePuts($stderr,file,"   if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(__after-__before));");
919         doublePuts($stderr,file,"   gc();") unless plan.vm.shouldMeasureGC
920         doublePuts($stderr,file,"}")
921       elsif benchParams.kind == :throughputBenchmark
922         emitTimerFunctionCode(file)
923         benchParams.dataPaths.each {
924           | path |
925           doublePuts($stderr,file,"load(#{path.inspect});")
926         }
927         doublePuts($stderr,file,"#{benchParams.setUpCommand.js}")
928         if benchParams.doWarmup
929           warmup = $warmup
930         else
931           warmup = 0
932         end
933         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{warmup + $inner}; __bencher_index++) {")
934         doublePuts($stderr,file,"    var __before = __bencher_curTimeMS();")
935         doublePuts($stderr,file,"    var __after = __before;")
936         doublePuts($stderr,file,"    var __runs = 0;")
937         doublePuts($stderr,file,"    var __expected = #{$quantum};")
938         doublePuts($stderr,file,"    while (true) {")
939         $rerun.times {
940           doublePuts($stderr,file,"       #{benchParams.command.js}")
941         }
942         doublePuts($stderr,file,"       __runs++;")
943         doublePuts($stderr,file,"       __after = __bencher_curTimeMS();")
944         if benchParams.deterministic
945           doublePuts($stderr,file,"       if (true) {")
946         else
947           doublePuts($stderr,file,"       if (__after - __before >= __expected) {")
948         end
949         doublePuts($stderr,file,"           if (__runs >= #{benchParams.minimumIterations} || __bencher_index < #{warmup})")
950         doublePuts($stderr,file,"               break;")
951         doublePuts($stderr,file,"           __expected += #{$quantum}")
952         doublePuts($stderr,file,"       }")
953         doublePuts($stderr,file,"    }")
954         doublePuts($stderr,file,"    if (__bencher_index >= #{warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{warmup}) + \": Time: \"+((__after-__before)/__runs));")
955         doublePuts($stderr,file,"}")
956         doublePuts($stderr,file,"#{benchParams.tearDownCommand.js}")
957       else
958         raise unless benchParams.kind == :singleFileTimedBenchmark
959         doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
960         doublePuts($stderr,file,"   var __bencher_before = __bencher_curTimeMS();")
961         $rerun.times {
962           doublePuts($stderr,file,"   run(__bencher_what);")
963         }
964         doublePuts($stderr,file,"   var __bencher_after = __bencher_curTimeMS();")
965         doublePuts($stderr,file,"   return __bencher_after - __bencher_before;")
966         doublePuts($stderr,file,"}")
967         $warmup.times {
968           doublePuts($stderr,file,"__bencher_run(#{benchParams.benchPath.inspect})")
969           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
970         }
971         $inner.times {
972           | innerIndex |
973           doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchParams.benchPath.inspect}));")
974           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
975         }
976       end
977     }
978   when :dumpRenderTree, :webkitTestRunner
979     case $timeMode
980     when :preciseTime
981       curTime = "(testRunner.preciseTime()*1000)"
982     when :date
983       curTime = "(Date.now())"
984     else
985       raise
986     end
987
988     mainCode = Benchfile.create("bencher") {
989       | file |
990       doublePuts($stderr,file,"__bencher_count = 0;")
991       doublePuts($stderr,file,"function __bencher_doNext(result) {")
992       doublePuts($stderr,file,"    if (__bencher_count >= #{$warmup})")
993       doublePuts($stderr,file,"        debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);")
994       doublePuts($stderr,file,"    __bencher_count++;")
995       doublePuts($stderr,file,"    if (__bencher_count < #{$inner+$warmup})")
996       doublePuts($stderr,file,"        __bencher_runImpl(__bencher_doNext);")
997       doublePuts($stderr,file,"    else")
998       doublePuts($stderr,file,"        quit();")
999       doublePuts($stderr,file,"}")
1000       doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);")
1001     }
1002     
1003     cssCode = Benchfile.create("bencher-css") {
1004       | file |
1005       doublePuts($stderr,file,".pass {\n    font-weight: bold;\n    color: green;\n}\n.fail {\n    font-weight: bold;\n    color: red;\n}\n\#console {\n    white-space: pre-wrap;\n    font-family: monospace;\n}")
1006     }
1007     
1008     preCode = Benchfile.create("bencher-pre") {
1009       | file |
1010       doublePuts($stderr,file,"if (window.testRunner) {")
1011       doublePuts($stderr,file,"    testRunner.dumpAsText(window.enablePixelTesting);")
1012       doublePuts($stderr,file,"    testRunner.waitUntilDone();")
1013       doublePuts($stderr,file,"}")
1014       doublePuts($stderr,file,"")
1015       doublePuts($stderr,file,"function debug(msg)")
1016       doublePuts($stderr,file,"{")
1017       doublePuts($stderr,file,"    var span = document.createElement(\"span\");")
1018       doublePuts($stderr,file,"    document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace")
1019       doublePuts($stderr,file,"    span.innerHTML = msg + '<br />';")
1020       doublePuts($stderr,file,"}")
1021       doublePuts($stderr,file,"")
1022       doublePuts($stderr,file,"function quit() {")
1023       doublePuts($stderr,file,"    testRunner.notifyDone();")
1024       doublePuts($stderr,file,"}")
1025       doublePuts($stderr,file,"")
1026       doublePuts($stderr,file,"__bencher_continuation=null;")
1027       doublePuts($stderr,file,"")
1028       doublePuts($stderr,file,"function reportResult(result) {")
1029       doublePuts($stderr,file,"    __bencher_continuation(result);")
1030       doublePuts($stderr,file,"}")
1031       if benchParams.kind == :singleFileTimedCallbackBenchmark
1032         doublePuts($stderr,file,"")
1033         doublePuts($stderr,file,benchParams.callbackDecl)
1034       end
1035       doublePuts($stderr,file,"")
1036       doublePuts($stderr,file,"function __bencher_runImpl(continuation) {")
1037       doublePuts($stderr,file,"    function doit() {")
1038       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"\";")
1039       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";")
1040       doublePuts($stderr,file,"        var testFrame = document.getElementById(\"testframe\");")
1041       doublePuts($stderr,file,"        testFrame.contentDocument.open();")
1042       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");")
1043       if benchParams.kind == :throughputBenchmark or benchParams.kind == :multiFileTimedBenchmark
1044         benchParams.dataPaths.each {
1045           | path |
1046           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{path.inspect.inspect[1..-2]}></script>\");")
1047         }
1048       end
1049       if benchParams.kind == :throughputBenchmark
1050         if benchParams.doWarmup
1051           warmup = $warmup
1052         else
1053           warmup = 0
1054         end
1055         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">\");")
1056         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.setUpCommand.js}\");")
1057         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_before = #{curTime};\");")
1058         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_after = __bencher_before;\");")
1059         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_expected = #{$quantum};\");")
1060         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_runs = 0;\");")
1061         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"while (true) {\");")
1062         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    #{benchParams.command.js}\");")
1063         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_runs++;\");")
1064         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_after = #{curTime};\");")
1065         if benchParams.deterministic
1066           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (true) {\");")
1067         else
1068           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (__bencher_after - __bencher_before >= __bencher_expected) {\");")
1069         end
1070         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        if (__bencher_runs >= #{benchParams.minimumIterations} || window.parent.__bencher_count < #{warmup})\");")
1071         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"            break;\");")
1072         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        __bencher_expected += #{$quantum}\");")
1073         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    }\");")
1074         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"}\");")
1075         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.tearDownCommand.js}\");")
1076         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"window.parent.reportResult((__bencher_after - __bencher_before) / __bencher_runs);\");")
1077         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</script>\");")
1078       else
1079         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">var __bencher_before = #{curTime};</script>\");")
1080         if benchParams.kind == :multiFileTimedBenchmark
1081           doublePuts($stderr,file,"        testFrame.contentDocument.write(#{benchParams.command.html.inspect});")
1082         else
1083           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{benchParams.benchPath.inspect.inspect[1..-2]}></script>\");")
1084         end
1085         unless benchParams.kind == :singleFileTimedCallbackBenchmark
1086           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">window.parent.reportResult(#{curTime} - __bencher_before);</script>\");")
1087         end
1088       end
1089       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</body></html>\");")
1090       doublePuts($stderr,file,"        testFrame.contentDocument.close();")
1091       doublePuts($stderr,file,"    }")
1092       doublePuts($stderr,file,"    __bencher_continuation = continuation;")
1093       doublePuts($stderr,file,"    window.setTimeout(doit, 10);")
1094       doublePuts($stderr,file,"}")
1095     }
1096
1097     Benchfile.create(["bencher-htmldoc",".html"]) {
1098       | file |
1099       doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>")
1100     }
1101   else
1102     raise
1103   end
1104 end
1105
1106 def emitBenchRunCode(name, plan, benchParams)
1107   plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchParams), plan)
1108 end
1109
1110 class FileCreator
1111   def initialize(filename)
1112     @filename = filename
1113     @state = :empty
1114   end
1115   
1116   def puts(text)
1117     $script.print "echo #{Shellwords.shellescape(text)}"
1118     if @state == :empty
1119       $script.print " > "
1120       @state = :nonEmpty
1121     else
1122       $script.print " >> "
1123     end
1124     $script.puts "#{Shellwords.shellescape(@filename)}"
1125   end
1126   
1127   def close
1128     if @state == :empty
1129       $script.puts "rm -f #{Shellwords.shellescape(text)}"
1130       $script.puts "touch #{Shellwords.shellescape(text)}"
1131     end
1132   end
1133   
1134   def self.open(filename)
1135     outp = FileCreator.new(filename)
1136     yield outp
1137     outp.close
1138   end
1139 end
1140
1141 def emitSelfContainedBenchRunCode(name, plan, targetFile, configFile, benchmark)
1142   FileCreator.open(configFile) {
1143     | outp |
1144     outp.puts "__bencher_message = \"#{name}: #{plan.vm}: #{plan.iteration}: \";"
1145     outp.puts "__bencher_warmup = #{$warmup};"
1146     outp.puts "__bencher_inner = #{$inner};"
1147     outp.puts "__bencher_benchmark = #{benchmark.to_json};"
1148     case $timeMode
1149     when :preciseTime
1150       outp.puts "__bencher_curTime = (function(){ return testRunner.preciseTime() * 1000; });"
1151     when :date
1152       outp.puts "__bencher_curTime = (function(){ return Date.now(); });"
1153     else
1154       raise
1155     end
1156   }
1157   
1158   plan.vm.emitRunCode(targetFile, plan)
1159 end
1160
1161 def planForDescription(string, plans, benchFullname, vmName, iteration)
1162   raise "Unexpected benchmark full name: #{benchFullname.inspect}, string: #{string.inspect}" unless benchFullname =~ /\//
1163   suiteName = $~.pre_match
1164   return nil if suiteName == "WARMUP"
1165   benchName = $~.post_match
1166   result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration}
1167   raise "Unexpected result dimensions: #{result.inspect}, string: #{string.inspect}" unless result.size == 1
1168   result[0]
1169 end
1170
1171 class ParsedResult
1172   attr_reader :plan, :innerIndex, :time, :result
1173   
1174   def initialize(plan, innerIndex, time)
1175     @plan = plan
1176     @innerIndex = innerIndex
1177     if time == :crashed
1178       @result = :error
1179     else
1180       @time = time
1181       @result = :success
1182     end
1183     
1184     raise unless @plan.is_a? BenchPlan
1185     raise unless @innerIndex.is_a? Integer
1186     raise unless @time.is_a? Numeric or @result == :error
1187   end
1188   
1189   def benchmark
1190     plan.benchmark
1191   end
1192   
1193   def suite
1194     plan.suite
1195   end
1196   
1197   def vm
1198     plan.vm
1199   end
1200   
1201   def outerIndex
1202     plan.iteration
1203   end
1204   
1205   def self.create(plan, innerIndex, time)
1206     if plan
1207       ParsedResult.new(plan, innerIndex, time)
1208     else
1209       nil
1210     end
1211   end
1212   
1213   def self.parse(plans, string)
1214     if string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): Time: /
1215       benchFullname = $1
1216       vmName = $2
1217       outerIndex = $3.to_i
1218       innerIndex = $4.to_i
1219       time = $~.post_match.to_f
1220       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, time)
1221     elsif string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): CRASHED/
1222       benchFullname = $1
1223       vmName = $2
1224       outerIndex = $3.to_i
1225       innerIndex = $4.to_i
1226       time = $~.post_match.to_f
1227       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, :crashed)
1228     else
1229       nil
1230     end
1231   end
1232 end
1233
1234 class VM
1235   @@extraEnvSet = {}
1236   
1237   def initialize(origPath, name, nameKind, svnRevision)
1238     @origPath = origPath.to_s
1239     @path = origPath.to_s
1240     @name = name
1241     @nameKind = nameKind
1242     @extraEnv = {}
1243     
1244     if $forceVMKind
1245       @vmType = $forceVMKind
1246     else
1247       if @origPath =~ /DumpRenderTree$/
1248         @vmType = :dumpRenderTree
1249       elsif @origPath =~ /WebKitTestRunner$/
1250         @vmType = :webkitTestRunner
1251       else
1252         @vmType = :jsc
1253       end
1254     end
1255     
1256     @svnRevision = svnRevision
1257     
1258     # Try to detect information about the VM.
1259     if path =~ /\/WebKitBuild\/(Release|Debug)+\/([a-zA-Z]+)$/
1260       @checkoutPath = $~.pre_match
1261       # FIXME: Use some variant of this: 
1262       # <bdash>   def retrieve_revision
1263       # <bdash>     `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i
1264       # <bdash>   end
1265       unless @svnRevision
1266         begin
1267           Dir.chdir(@checkoutPath) {
1268             $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
1269             IO.popen("svn info", "r") {
1270               | inp |
1271               inp.each_line {
1272                 | line |
1273                 if line =~ /Revision: ([0-9]+)/
1274                   @svnRevision = $1
1275                 end
1276               }
1277             }
1278           }
1279           unless @svnRevision
1280             $stderr.puts "Warning: running svn info for #{name} silently failed."
1281           end
1282         rescue => e
1283           # Failed to detect svn revision.
1284           $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
1285         end
1286       end
1287     else
1288       $stderr.puts "Warning: could not identify checkout location for #{name}"
1289     end
1290     
1291     if @path =~ /\/Release\/([a-zA-Z]+)$/
1292       @libPath, @relativeBinPath = [$~.pre_match+"/Release"], "./#{$1}"
1293     elsif @path =~ /\/Debug\/([a-zA-Z]+)$/
1294       @libPath, @relativeBinPath = [$~.pre_match+"/Debug"], "./#{$1}"
1295     elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/
1296       @libPath = [$~.pre_match + "/Contents/Resources", $~.pre_match + "/Contents/Frameworks"]
1297     elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/
1298       @libPath, @relativeBinPath = [$~.pre_match], $&[1..-1]
1299     end
1300   end
1301   
1302   def canCopyIntoBenchPath
1303     if @libPath and @relativeBinPath
1304       true
1305     else
1306       false
1307     end
1308   end
1309   
1310   def addExtraEnv(key, val)
1311     @extraEnv[key] = val
1312     @@extraEnvSet[key] = true
1313   end
1314   
1315   def copyIntoBenchPath
1316     raise unless canCopyIntoBenchPath
1317     basename, filename = Benchfile.uniqueFilename("vm")
1318     raise unless Dir.mkdir(filename)
1319     @libPath.each {
1320       | libPathPart |
1321       cmd = "cp -a #{Shellwords.shellescape(libPathPart)}/* #{Shellwords.shellescape(filename.to_s)}"
1322       $stderr.puts ">> #{cmd}" if $verbosity>=2
1323       raise unless system(cmd)
1324     }
1325     @path = "#{basename}/#{@relativeBinPath}"
1326     @libPath = [basename]
1327   end
1328   
1329   def to_s
1330     @name
1331   end
1332   
1333   def name
1334     @name
1335   end
1336   
1337   def shouldMeasureGC
1338     $measureGC == true or ($measureGC == name)
1339   end
1340   
1341   def origPath
1342     @origPath
1343   end
1344   
1345   def path
1346     @path
1347   end
1348   
1349   def nameKind
1350     @nameKind
1351   end
1352   
1353   def vmType
1354     @vmType
1355   end
1356   
1357   def checkoutPath
1358     @checkoutPath
1359   end
1360   
1361   def svnRevision
1362     @svnRevision
1363   end
1364   
1365   def extraEnv
1366     @extraEnv
1367   end
1368   
1369   def printFunction
1370     case @vmType
1371     when :jsc
1372       "print"
1373     when :dumpRenderTree, :webkitTestRunner
1374       "debug"
1375     else
1376       raise @vmType
1377     end
1378   end
1379   
1380   def emitRunCode(fileToRun, plan)
1381     myLibPath = @libPath
1382     myLibPath = [] unless myLibPath
1383     @@extraEnvSet.keys.each {
1384       | key |
1385       $script.puts "unset #{Shellwords.shellescape(key)}"
1386     }
1387     $script.puts "export DYLD_LIBRARY_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1388     $script.puts "export DYLD_FRAMEWORK_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1389     @extraEnv.each_pair {
1390       | key, val |
1391       $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1392     }
1393     plan.environment.each_pair {
1394         | key, val |
1395         $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1396     }
1397     $script.puts "#{path} #{fileToRun} 2>&1 || {"
1398     $script.puts "    echo " + Shellwords.shellescape("#{name} failed to run!") + " 1>&2"
1399     $inner.times {
1400       | iteration |
1401       $script.puts "    echo " + Shellwords.shellescape("#{plan.prefix}: #{iteration}: CRASHED")
1402     }
1403     $script.puts "}"
1404     plan.environment.keys.each {
1405         | key |
1406         $script.puts "export #{Shellwords.shellescape(key)}="
1407     }
1408   end
1409 end
1410
1411 class StatsAccumulator
1412   def initialize
1413     @stats = []
1414     ($outer*$inner).times {
1415       @stats << Stats.new
1416     }
1417   end
1418   
1419   def statsForIteration(outerIteration, innerIteration)
1420     @stats[outerIteration*$inner + innerIteration]
1421   end
1422   
1423   def stats
1424     result = Stats.new
1425     @stats.each {
1426       | stat |
1427       if stat.ok?
1428         result.add(yield stat)
1429       else
1430         result.add(nil)
1431       end
1432     }
1433     result
1434   end
1435   
1436   def geometricMeanStats
1437     stats {
1438       | stat |
1439       stat.geometricMean
1440     }
1441   end
1442   
1443   def arithmeticMeanStats
1444     stats {
1445       | stat |
1446       stat.arithmeticMean
1447     }
1448   end
1449 end
1450
1451 module Benchmark
1452   attr_accessor :benchmarkSuite
1453   attr_reader :name
1454   
1455   def fullname
1456     benchmarkSuite.name + "/" + name
1457   end
1458   
1459   def to_s
1460     fullname
1461   end
1462   
1463   def weight
1464     1
1465   end
1466   
1467   def weightString
1468     raise unless weight.is_a? Fixnum
1469     raise unless weight >= 1
1470     if weight == 1
1471       ""
1472     else
1473       "x#{weight} "
1474     end
1475   end
1476 end
1477
1478 class SunSpiderBenchmark
1479   include Benchmark
1480   
1481   def initialize(name)
1482     @name = name
1483   end
1484   
1485   def emitRunCode(plan)
1486     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js")))
1487   end
1488 end
1489
1490 class LongSpiderBenchmark
1491   include Benchmark
1492   
1493   def initialize(name)
1494     @name = name
1495   end
1496   
1497   def emitRunCode(plan)
1498     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("LongSpider-#{@name}", "#{LONGSPIDER_PATH}/#{@name}.js")))
1499   end
1500 end
1501
1502 class V8Benchmark
1503   include Benchmark
1504   
1505   def initialize(name)
1506     @name = name
1507   end
1508   
1509   def emitRunCode(plan)
1510     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js")))
1511   end
1512 end
1513
1514 class V8RealBenchmark
1515   include Benchmark
1516   
1517   attr_reader :v8SuiteName
1518   
1519   def initialize(v8SuiteName, name, weight, minimumIterations)
1520     @v8SuiteName = v8SuiteName
1521     @name = name
1522     @weight = weight
1523     @minimumIterations = minimumIterations
1524   end
1525   
1526   def weight
1527     @weight
1528   end
1529   
1530   def emitRunCode(plan)
1531     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(["base", @v8SuiteName, "jsc-#{@name}"].collect{|v| ensureFile("V8Real-#{v}", "#{V8_REAL_PATH}/#{v}.js")}, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, false, @minimumIterations))
1532   end
1533 end
1534
1535 class OctaneBenchmark
1536   include Benchmark
1537   
1538   def initialize(files, name, weight, doWarmup, deterministic, minimumIterations)
1539     @files = files
1540     @name = name
1541     @weight = weight
1542     @doWarmup = doWarmup
1543     @deterministic = deterministic
1544     @minimumIterations = minimumIterations
1545   end
1546   
1547   def weight
1548     @weight
1549   end
1550   
1551   def emitRunCode(plan)
1552     files = []
1553     files += (["base"] + @files).collect {
1554       | v |
1555       ensureFile("Octane-#{v}", "#{OCTANE_PATH}/#{v}.js")
1556     }
1557     files += ["jsc-#{@name}"].collect {
1558       | v |
1559       ensureFile("Octane-#{v}", "#{OCTANE_WRAPPER_PATH}/#{v}.js")
1560     }
1561     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(files, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1562   end
1563 end
1564
1565 class KrakenBenchmark
1566   include Benchmark
1567   
1568   def initialize(name)
1569     @name = name
1570   end
1571   
1572   def emitRunCode(plan)
1573     emitBenchRunCode(fullname, plan, MultiFileTimedBenchmarkParameters.new([ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js")], loadCommandForFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js")))
1574   end
1575 end
1576
1577 class JSBenchBenchmark
1578   include Benchmark
1579   
1580   attr_reader :jsBenchMode
1581   
1582   def initialize(name, jsBenchMode)
1583     @name = name
1584     @jsBenchMode = jsBenchMode
1585   end
1586   
1587   def emitRunCode(plan)
1588     callbackDecl  = "function JSBNG_handleResult(result) {\n"
1589     callbackDecl += "    if (result.error) {\n"
1590     callbackDecl += "        console.log(\"Did not run benchmark correctly!\");\n"
1591     callbackDecl += "        quit();\n"
1592     callbackDecl += "    }\n"
1593     callbackDecl += "    reportResult(result.time);\n"
1594     callbackDecl += "}\n"
1595     emitBenchRunCode(fullname, plan, SingleFileTimedCallbackBenchmarkParameters.new(callbackDecl, ensureFile("JSBench-#{@name}", "#{JSBENCH_PATH}/#{@name}/#{@jsBenchMode}.js")))
1596   end
1597 end
1598
1599 class JSRegressBenchmark
1600   include Benchmark
1601   
1602   def initialize(name)
1603     @name = name
1604   end
1605   
1606   def emitRunCode(plan)
1607     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("JSRegress-#{@name}", "#{JSREGRESS_PATH}/#{@name}.js")))
1608   end
1609 end
1610
1611 class AsmBenchBenchmark
1612   include Benchmark
1613   
1614   def initialize(name)
1615     @name = name
1616   end
1617   
1618   def emitRunCode(plan)
1619     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("AsmBench-#{@name}", "#{ASMBENCH_PATH}/#{@name}.js")))
1620   end
1621 end
1622
1623
1624 class CompressionBenchBenchmark
1625   include Benchmark
1626   
1627   def initialize(files, name, model)
1628     @files = files
1629     @name = name;
1630     @name = name + "-" + model if !model.empty?
1631     @name = @name.gsub(" ", "-").downcase
1632     @scriptName = name
1633     @weight = 1
1634     @doWarmup = true
1635     @deterministic = true
1636     @minimumIterations = 1
1637     @model = model
1638   end
1639   
1640   def weight
1641     @weight
1642   end
1643   
1644   def emitRunCode(plan)
1645     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new((["base"] + @files + ["jsc-#{@scriptName}"]).collect{|v| ensureFile("Compression-#{v}", "#{COMPRESSIONBENCH_PATH}/#{v}.js")}, simpleCommand("jscSetUp('#{@model}');"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1646   end
1647 end
1648
1649 class DSPJSFiltrrBenchmark
1650   include Benchmark
1651   
1652   def initialize(name, filterKey)
1653     @name = name
1654     @filterKey = filterKey
1655   end
1656   
1657   def emitRunCode(plan)
1658     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr.js")
1659     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr_back.jpg")
1660     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-jquery.min.js")
1661     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-bencher.html")
1662     emitSelfContainedBenchRunCode(fullname, plan, "filtrr-bencher.html", "bencher-config.js", @filterKey)
1663   end
1664 end
1665
1666 class DSPJSVP8Benchmark
1667   include Benchmark
1668   
1669   def initialize
1670     @name = "route9-vp8"
1671   end
1672   
1673   def weight
1674     5
1675   end
1676   
1677   def emitRunCode(plan)
1678     ensureBenchmarkFiles(DSPJS_ROUTE9_PATH)
1679     emitSelfContainedBenchRunCode(fullname, plan, "route9-bencher.html", "bencher-config.js", "")
1680   end
1681 end
1682
1683 class DSPStarfieldBenchmark
1684   include Benchmark
1685
1686   def initialize
1687     @name = "starfield"
1688   end
1689   
1690   def weight
1691     5
1692   end
1693
1694   def emitRunCode(plan)
1695     ensureBenchmarkFiles(DSPJS_STARFIELD_PATH)
1696     emitSelfContainedBenchRunCode(fullname, plan, "starfield-bencher.html", "bencher-config.js", "")
1697   end
1698 end
1699
1700 class DSPJSJSLinuxBenchmark
1701   include Benchmark
1702   def initialize
1703     @name = "bellard-jslinux"
1704   end
1705
1706   def weight
1707     5
1708   end
1709
1710   def emitRunCode(plan)
1711     ensureBenchmarkFiles(DSPJS_JSLINUX_PATH)
1712     emitSelfContainedBenchRunCode(fullname, plan, "jslinux-bencher.html", "bencher-config.js", "")
1713   end
1714 end
1715
1716 class DSPJSQuake3Benchmark
1717   include Benchmark
1718
1719   def initialize
1720     @name = "zynaps-quake3"
1721   end
1722
1723   def weight
1724     5
1725   end
1726
1727   def emitRunCode(plan)
1728     ensureBenchmarkFiles(DSPJS_QUAKE3_PATH)
1729     emitSelfContainedBenchRunCode(fullname, plan, "quake-bencher.html", "bencher-config.js", "")
1730   end
1731 end
1732
1733 class DSPJSMandelbrotBenchmark
1734   include Benchmark
1735
1736   def initialize
1737     @name = "zynaps-mandelbrot"
1738   end
1739
1740   def weight
1741     5
1742   end
1743
1744   def emitRunCode(plan)
1745     ensureBenchmarkFiles(DSPJS_MANDELBROT_PATH)
1746     emitSelfContainedBenchRunCode(fullname, plan, "mandelbrot-bencher.html", "bencher-config.js", "")
1747   end
1748 end
1749
1750 class DSPJSAmmoJSASMBenchmark
1751   include Benchmark
1752
1753   def initialize
1754     @name = "ammojs-asm-js"
1755   end
1756
1757   def weight
1758     5
1759   end
1760
1761   def emitRunCode(plan)
1762     ensureBenchmarkFiles(DSPJS_AMMOJS_ASMJS_PATH)
1763     emitSelfContainedBenchRunCode(fullname, plan, "ammo-asmjs-bencher.html", "bencher-config.js", "")
1764   end
1765 end
1766
1767 class DSPJSAmmoJSRegularBenchmark
1768   include Benchmark
1769
1770   def initialize
1771     @name = "ammojs-regular-js"
1772   end
1773
1774   def weight
1775     5
1776   end
1777
1778   def emitRunCode(plan)
1779     ensureBenchmarkFiles(DSPJS_AMMOJS_REGULAR_PATH)
1780     emitSelfContainedBenchRunCode(fullname, plan, "ammo-regular-bencher.html", "bencher-config.js", "")
1781   end
1782 end
1783
1784 class BrowsermarkJSBenchmark
1785   include Benchmark
1786     
1787   def initialize(name)
1788     @name = name
1789   end
1790   
1791   def emitRunCode(plan)
1792     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new([ensureFile(name, "#{BROWSERMARK_JS_PATH}/#{name}/test.js"), ensureFile("browsermark-bencher", "#{BROWSERMARK_JS_PATH}/browsermark-bencher.js")], simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, 32))
1793   end
1794 end
1795
1796 class BrowsermarkDOMBenchmark
1797   include Benchmark
1798     
1799   def initialize(name)
1800     @name = name
1801   end
1802   
1803   def emitRunCode(plan)
1804     ensureBenchmarkFiles(BROWSERMARK_PATH)
1805     emitSelfContainedBenchRunCode(fullname, plan, "tests/benchmarks/dom/#{name}/index.html", "bencher-config.js", name)
1806   end
1807 end
1808
1809 class BenchmarkSuite
1810   def initialize(name, preferredMean, decimalShift)
1811     @name = name
1812     @preferredMean = preferredMean
1813     @benchmarks = []
1814     @subSuites = []
1815     @decimalShift = decimalShift
1816   end
1817   
1818   def name
1819     @name
1820   end
1821   
1822   def to_s
1823     @name
1824   end
1825   
1826   def decimalShift
1827     @decimalShift
1828   end
1829   
1830   def addIgnoringPattern(benchmark)
1831     benchmark.benchmarkSuite = self
1832     @benchmarks << benchmark
1833   end
1834   
1835   def add(benchmark)
1836     if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern
1837         addIgnoringPattern(benchmark)
1838     end
1839   end
1840   
1841   def addSubSuite(subSuite)
1842     @subSuites << subSuite
1843   end
1844   
1845   def benchmarks
1846     @benchmarks
1847   end
1848   
1849   def benchmarkForName(name)
1850     result = @benchmarks.select{|v| v.name == name}
1851     raise unless result.length == 1
1852     result[0]
1853   end
1854   
1855   def hasBenchmark(benchmark)
1856     array = @benchmarks.select{|v| v == benchmark}
1857     raise unless array.length == 1 or array.length == 0
1858     array.length == 1
1859   end
1860   
1861   def subSuites
1862     @subSuites
1863   end
1864   
1865   def suites
1866     [self] + @subSuites
1867   end
1868   
1869   def suitesWithBenchmark(benchmark)
1870     result = [self]
1871     @subSuites.each {
1872       | subSuite |
1873       if subSuite.hasBenchmark(benchmark)
1874         result << subSuite
1875       end
1876     }
1877     result
1878   end
1879   
1880   def empty?
1881     @benchmarks.empty?
1882   end
1883   
1884   def retain_if
1885     @benchmarks.delete_if {
1886       | benchmark |
1887       not yield benchmark
1888     }
1889   end
1890   
1891   def preferredMean
1892     @preferredMean
1893   end
1894   
1895   def computeMean(stat)
1896     if stat.ok?
1897       (stat.send @preferredMean) * (10 ** decimalShift)
1898     else
1899       nil
1900     end
1901   end
1902 end
1903
1904 class BenchRunPlan
1905   def initialize(benchmark, vm, iteration)
1906     @benchmark = benchmark
1907     @vm = vm
1908     @iteration = iteration
1909     @environment = {}
1910     if $environment.has_key?(vm.name)
1911       if $environment[vm.name].has_key?(benchmark.benchmarkSuite.name)
1912         if $environment[vm.name][benchmark.benchmarkSuite.name].has_key?(benchmark.name)
1913           @environment = $environment[vm.name][benchmark.benchmarkSuite.name][benchmark.name]
1914         end
1915       end
1916     end
1917   end
1918   
1919   def benchmark
1920     @benchmark
1921   end
1922   
1923   def suite
1924     @benchmark.benchmarkSuite
1925   end
1926   
1927   def vm
1928     @vm
1929   end
1930   
1931   def iteration
1932     @iteration
1933   end
1934  
1935   def environment
1936     @environment
1937   end
1938  
1939   def prefix
1940     "#{@benchmark.fullname}: #{vm.name}: #{iteration}"
1941   end
1942   
1943   def emitRunCode
1944     @benchmark.emitRunCode(self)
1945   end
1946   
1947   def to_s
1948     benchmark.to_s + "/" + vm.to_s
1949   end
1950 end
1951
1952 class BenchmarkOnVM
1953   def initialize(benchmark, suiteOnVM, subSuitesOnVM)
1954     @benchmark = benchmark
1955     @suiteOnVM = suiteOnVM
1956     @subSuitesOnVM = subSuitesOnVM
1957     @stats = Stats.new
1958   end
1959   
1960   def to_s
1961     "#{@benchmark} on #{@suiteOnVM.vm}"
1962   end
1963   
1964   def benchmark
1965     @benchmark
1966   end
1967   
1968   def vm
1969     @suiteOnVM.vm
1970   end
1971   
1972   def vmStats
1973     @suiteOnVM.vmStats
1974   end
1975   
1976   def suite
1977     @benchmark.benchmarkSuite
1978   end
1979   
1980   def suiteOnVM
1981     @suiteOnVM
1982   end
1983   
1984   def subSuitesOnVM
1985     @subSuitesOnVM
1986   end
1987   
1988   def stats
1989     @stats
1990   end
1991   
1992   def parseResult(result)
1993     raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm
1994     raise unless result.benchmark == @benchmark
1995     @stats.add(result.time)
1996   end
1997 end
1998
1999 class NamedStatsAccumulator < StatsAccumulator
2000   def initialize(name)
2001     super()
2002     @name = name
2003   end
2004   
2005   def reportingName
2006     @name
2007   end
2008 end
2009
2010 class SuiteOnVM < StatsAccumulator
2011   def initialize(vm, vmStats, suite)
2012     super()
2013     @vm = vm
2014     @vmStats = vmStats
2015     @suite = suite
2016     
2017     raise unless @vm.is_a? VM
2018     raise unless @vmStats.is_a? StatsAccumulator
2019     raise unless @suite.is_a? BenchmarkSuite
2020   end
2021   
2022   def to_s
2023     "#{@suite} on #{@vm}"
2024   end
2025   
2026   def suite
2027     @suite
2028   end
2029   
2030   def vm
2031     @vm
2032   end
2033
2034   def reportingName
2035     @vm.name
2036   end
2037   
2038   def vmStats
2039     raise unless @vmStats
2040     @vmStats
2041   end
2042 end
2043
2044 class SubSuiteOnVM < StatsAccumulator
2045   def initialize(vm, suite)
2046     super()
2047     @vm = vm
2048     @suite = suite
2049     raise unless @vm.is_a? VM
2050     raise unless @suite.is_a? BenchmarkSuite
2051   end
2052   
2053   def to_s
2054     "#{@suite} on #{@vm}"
2055   end
2056   
2057   def suite
2058     @suite
2059   end
2060   
2061   def vm
2062     @vm
2063   end
2064   
2065   def reportingName
2066     @vm.name
2067   end
2068 end
2069
2070 class BenchPlan
2071   def initialize(benchmarkOnVM, iteration)
2072     @benchmarkOnVM = benchmarkOnVM
2073     @iteration = iteration
2074   end
2075   
2076   def to_s
2077     "#{@benchmarkOnVM} \##{@iteration+1}"
2078   end
2079   
2080   def benchmarkOnVM
2081     @benchmarkOnVM
2082   end
2083   
2084   def benchmark
2085     @benchmarkOnVM.benchmark
2086   end
2087   
2088   def suite
2089     @benchmarkOnVM.suite
2090   end
2091   
2092   def vm
2093     @benchmarkOnVM.vm
2094   end
2095   
2096   def iteration
2097     @iteration
2098   end
2099   
2100   def parseResult(result)
2101     raise unless result.plan == self
2102     @benchmarkOnVM.parseResult(result)
2103     benchmark.weight.times {
2104       @benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time)
2105       @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time)
2106       @benchmarkOnVM.subSuitesOnVM.each {
2107         | subSuite |
2108         subSuite.statsForIteration(@iteration, result.innerIndex).add(result.time)
2109       }
2110     }
2111   end
2112 end
2113
2114 def lpad(str,chars)
2115   if str.length>chars
2116     str
2117   else
2118     "%#{chars}s"%(str)
2119   end
2120 end
2121
2122 def rpad(str,chars)
2123   while str.length < chars
2124     str+=" "
2125   end
2126   str
2127 end
2128
2129 def center(str,chars)
2130   while str.length<chars
2131     str+=" "
2132     if str.length<chars
2133       str=" "+str
2134     end
2135   end
2136   str
2137 end
2138
2139 def statsToStr(stats, decimalShift)
2140   if stats.error?
2141     lpad(center("ERROR", 10+10+2), 12+10+2)
2142   elsif $inner*$outer == 1
2143     string = numToStr(stats.mean, decimalShift)
2144     raise unless string =~ /\./
2145     left = $~.pre_match
2146     right = $~.post_match
2147     lpad(left, 13 - decimalShift) + "." + rpad(right, 10 + decimalShift)
2148   else
2149     lpad(numToStr(stats.mean, decimalShift), 12) + "+-" + rpad(numToStr(stats.confInt, decimalShift), 10)
2150   end
2151 end
2152
2153 def plural(num)
2154   if num == 1
2155     ""
2156   else
2157     "s"
2158   end
2159 end
2160
2161 def wrap(str, columns)
2162   array = str.split
2163   result = ""
2164   curLine = array.shift
2165   array.each {
2166     | curStr |
2167     if (curLine + " " + curStr).size > columns
2168       result += curLine + "\n"
2169       curLine = curStr
2170     else
2171       curLine += " " + curStr
2172     end
2173   }
2174   result + curLine + "\n"
2175 end
2176   
2177 def runAndGetResults
2178   results = nil
2179   Dir.chdir(BENCH_DATA_PATH) {
2180     $stderr.puts ">> sh ./runscript" if $verbosity >= 2
2181     raise "Script did not complete correctly: #{$?}" unless system("sh ./runscript > runlog")
2182     results = IO::read("runlog")
2183   }
2184   raise unless results
2185   results
2186 end
2187
2188 def parseAndDisplayResults(results)
2189   vmStatses = []
2190   $vms.each {
2191     | vm |
2192     vmStatses << NamedStatsAccumulator.new(vm.name)
2193   }
2194   
2195   suitesOnVMs = []
2196   suitesOnVMsForSuite = {}
2197   subSuitesOnVMsForSubSuite = {}
2198   $suites.each {
2199     | suite |
2200     suitesOnVMsForSuite[suite] = []
2201     suite.subSuites.each {
2202       | subSuite |
2203       subSuitesOnVMsForSubSuite[subSuite] = []
2204     }
2205   }
2206   suitesOnVMsForVM = {}
2207   $vms.each {
2208     | vm |
2209     suitesOnVMsForVM[vm] = []
2210   }
2211   
2212   benchmarksOnVMs = []
2213   benchmarksOnVMsForBenchmark = {}
2214   $benchmarks.each {
2215     | benchmark |
2216     benchmarksOnVMsForBenchmark[benchmark] = []
2217   }
2218   
2219   $vms.each_with_index {
2220     | vm, vmIndex |
2221     vmStats = vmStatses[vmIndex]
2222     $suites.each {
2223       | suite |
2224       suiteOnVM = SuiteOnVM.new(vm, vmStats, suite)
2225       subSuitesOnVM = suite.subSuites.map {
2226         | subSuite |
2227         result = SubSuiteOnVM.new(vm, subSuite)
2228         subSuitesOnVMsForSubSuite[subSuite] << result
2229         result
2230       }
2231       suitesOnVMs << suiteOnVM
2232       suitesOnVMsForSuite[suite] << suiteOnVM
2233       suitesOnVMsForVM[vm] << suiteOnVM
2234       suite.benchmarks.each {
2235         | benchmark |
2236         subSuitesOnVMForThisBenchmark = []
2237         subSuitesOnVM.each {
2238           | subSuiteOnVM |
2239           if subSuiteOnVM.suite.hasBenchmark(benchmark)
2240             subSuitesOnVMForThisBenchmark << subSuiteOnVM
2241           end
2242         }
2243         benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM, subSuitesOnVMForThisBenchmark)
2244         benchmarksOnVMs << benchmarkOnVM
2245         benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
2246       }
2247     }
2248   }
2249   
2250   plans = []
2251   benchmarksOnVMs.each {
2252     | benchmarkOnVM |
2253     $outer.times {
2254       | iteration |
2255       plans << BenchPlan.new(benchmarkOnVM, iteration)
2256     }
2257   }
2258
2259   hostname = nil
2260   hwmodel = nil
2261   results.each_line {
2262     | line |
2263     line.chomp!
2264     if line =~ /HOSTNAME:([^.]+)/
2265       hostname = $1
2266     elsif line =~ /HARDWARE:hw\.model: /
2267       hwmodel = $~.post_match.chomp
2268     else
2269       result = ParsedResult.parse(plans, line)
2270       if result
2271         result.plan.parseResult(result)
2272       end
2273     end
2274   }
2275   
2276   # Compute the geomean of the preferred means of results on a SuiteOnVM
2277   overallResults = []
2278   $vms.each {
2279     | vm |
2280     result = Stats.new
2281     $outer.times {
2282       | outerIndex |
2283       $inner.times {
2284         | innerIndex |
2285         curResult = Stats.new
2286         suitesOnVMsForVM[vm].each {
2287           | suiteOnVM |
2288           # For a given iteration, suite, and VM, compute the suite's preferred mean
2289           # over the data collected for all benchmarks in that suite. We'll have one
2290           # sample per benchmark. For example on V8 this will be the geomean of 1
2291           # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
2292           # splay.
2293           curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
2294         }
2295         
2296         # curResult now holds 1 sample for each of the means computed in the above
2297         # loop. Compute the geomean over this, and store it.
2298         if curResult.ok?
2299           result.add(curResult.geometricMean)
2300         else
2301           result.add(nil)
2302         end
2303       }
2304     }
2305
2306     # $overallResults will have a Stats for each VM. That Stats object will hold
2307     # $inner*$outer geomeans, allowing us to compute the arithmetic mean and
2308     # confidence interval of the geomeans of preferred means. Convoluted, but
2309     # useful and probably sound.
2310     overallResults << result
2311   }
2312   
2313   if $verbosity >= 2
2314     benchmarksOnVMs.each {
2315       | benchmarkOnVM |
2316       $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
2317     }
2318     
2319     $vms.each_with_index {
2320       | vm, vmIndex |
2321       vmStats = vmStatses[vmIndex]
2322       $stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}"
2323       $stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}"
2324     }
2325   end
2326
2327   if $outputName
2328     reportName = $outputName
2329   else
2330     reportName =
2331       (if ($vms.collect {
2332              | vm |
2333              vm.nameKind
2334            }.index :auto)
2335          ""
2336        else
2337          text = $vms.collect {
2338            | vm |
2339            vm.to_s
2340          }.join("_") + "_"
2341          if text.size >= 40
2342            ""
2343          else
2344            text
2345          end
2346        end) +
2347       ($suites.collect {
2348          | suite |
2349          suite.to_s
2350        }.join("")) + "_" +
2351       (if hostname
2352          hostname + "_"
2353        else
2354          ""
2355        end)+
2356       (begin
2357          time = Time.now
2358          "%04d%02d%02d_%02d%02d" %
2359            [ time.year, time.month, time.day,
2360              time.hour, time.min ]
2361        end)
2362   end
2363
2364   unless $brief
2365     puts "Generating benchmark report at #{Dir.pwd}/#{reportName}_report.txt"
2366     puts "And raw data at #{Dir.pwd}/#{reportName}.json"
2367   end
2368   
2369   outp = $stdout
2370   json = {}
2371   begin
2372     outp = File.open(reportName + "_report.txt","w")
2373   rescue => e
2374     $stderr.puts "Error: could not save report to #{reportName}_report.txt: #{e}"
2375     $stderr.puts
2376   end
2377   
2378   def createVMsString
2379     result = ""
2380     result += "   " if $allSuites.size > 1
2381     result += rpad("", $benchpad + $weightpad)
2382     result += " "
2383     $vms.size.times {
2384       | index |
2385       if index != 0
2386         result += " "+NoChange.new(0).shortForm
2387       end
2388       result += lpad(center($vms[index].name, 10+10+2), 12+10+2)
2389     }
2390     result += "    "
2391     if $vms.size >= 3
2392       result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
2393     elsif $vms.size >= 2
2394       result += " "*26
2395     end
2396     result
2397   end
2398   
2399   def andJoin(list)
2400     if list.size == 1
2401       list[0].to_s
2402     elsif list.size == 2
2403       "#{list[0]} and #{list[1]}"
2404     else
2405       "#{list[0..-2].join(', ')}, and #{list[-1]}"
2406     end
2407   end
2408   
2409   json["vms"] = $vms.collect{|v| v.name}
2410   json["suites"] = {}
2411   json["runlog"] = results
2412   
2413   columns = [createVMsString.size, 78].max
2414   
2415   outp.print "Benchmark report for "
2416   outp.print andJoin($suites)
2417   if hostname
2418     outp.print " on #{hostname}"
2419   end
2420   if hwmodel
2421     outp.print " (#{hwmodel})"
2422   end
2423   outp.puts "."
2424   outp.puts
2425   
2426   outp.puts "VMs tested:"
2427   $vms.each {
2428     | vm |
2429     outp.print "\"#{vm.name}\" at #{vm.origPath}"
2430     if vm.svnRevision
2431       outp.print " (r#{vm.svnRevision})"
2432     end
2433     outp.puts
2434     vm.extraEnv.each_pair {
2435       | key, val |
2436       outp.puts "    export #{key}=#{val}"
2437     }
2438   }
2439   
2440   outp.puts
2441   
2442   outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
2443                  "with #{$outer} VM invocation#{plural($outer)} per benchmark."+
2444                  (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
2445                                       "total time of those iterations, for each sample.")
2446                   else "" end)+
2447                  (if $measureGC == true then (" No manual garbage collection invocations were "+
2448                                               "emitted.")
2449                   elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
2450                                          "all VMs except #{$measureGC}.")
2451                   else (" Emitted a call to gc() between sample measurements.") end)+
2452                  (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
2453                                         "began with the very first iteration.")
2454                   else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
2455                         "invocation for warm-up.") end)+
2456                  (case $timeMode
2457                   when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
2458                                           "microsecond-level timing.")
2459                   when :date then (" Used the portable Date.now() method to get millisecond-"+
2460                                    "level timing.")
2461                   else raise end)+
2462                  " Reporting benchmark execution times with 95% confidence "+
2463                  "intervals in milliseconds.",
2464                  columns)
2465   
2466   outp.puts
2467   
2468   def printVMs(outp)
2469     outp.puts createVMsString
2470   end
2471   
2472   def summaryStats(outp, json, jsonKey, accumulators, name, decimalShift, &proc)
2473     resultingJson = {}
2474     outp.print "   " if $allSuites.size > 1
2475     outp.print rpad(name, $benchpad + $weightpad)
2476     outp.print " "
2477     accumulators.size.times {
2478       | index |
2479       if index != 0
2480         outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
2481       end
2482       outp.print statsToStr(accumulators[index].stats(&proc), decimalShift)
2483       resultingJson[accumulators[index].reportingName] = accumulators[index].stats(&proc).jsonMap
2484     }
2485     if accumulators.size>=2
2486       outp.print("    "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm)
2487     end
2488     outp.puts
2489     json[jsonKey] = resultingJson
2490   end
2491   
2492   def meanName(currentMean, preferredMean)
2493     result = "<#{currentMean}>"
2494     if "#{currentMean}Mean" == preferredMean.to_s
2495       result += " *"
2496     end
2497     result
2498   end
2499   
2500   def allSummaryStats(outp, json, accumulators, preferredMean, decimalShift)
2501     summaryStats(outp, json, "<arithmetic>", accumulators, meanName("arithmetic", preferredMean), decimalShift) {
2502       | stat |
2503       stat.arithmeticMean
2504     }
2505     
2506     summaryStats(outp, json, "<geometric>", accumulators, meanName("geometric", preferredMean), decimalShift) {
2507       | stat |
2508       stat.geometricMean
2509     }
2510     
2511     summaryStats(outp, json, "<harmonic>", accumulators, meanName("harmonic", preferredMean), decimalShift) {
2512       | stat |
2513       stat.harmonicMean
2514     }
2515   end
2516   
2517   $suites.each {
2518     | suite |
2519     suiteJson = {}
2520     subSuiteJsons = {}
2521     suite.subSuites.each {
2522       | subSuite |
2523       subSuiteJsons[subSuite] = {}
2524     }
2525     
2526     printVMs(outp)
2527     if $allSuites.size > 1
2528       outp.puts(andJoin(suite.suites.map{|v| v.name}) + ":")
2529     else
2530       outp.puts
2531     end
2532     suite.benchmarks.each {
2533       | benchmark |
2534       benchmarkJson = {}
2535       outp.print "   " if $allSuites.size > 1
2536       outp.print rpad(benchmark.name, $benchpad) + rpad(benchmark.weightString, $weightpad)
2537       if benchmark.name.size > $benchNameClip
2538         outp.puts
2539         outp.print "   " if $allSuites.size > 1
2540         outp.print((" " * $benchpad) + (" " * $weightpad))
2541       end
2542       outp.print " "
2543       myConfigs = benchmarksOnVMsForBenchmark[benchmark]
2544       myConfigs.size.times {
2545         | index |
2546         if index != 0
2547           outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
2548         end
2549         outp.print statsToStr(myConfigs[index].stats, suite.decimalShift)
2550         benchmarkJson[myConfigs[index].vm.name] = myConfigs[index].stats.jsonMap
2551       }
2552       if $vms.size>=2
2553         outp.print("    "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
2554       end
2555       outp.puts
2556       suiteJson[benchmark.name] = benchmarkJson
2557       suite.subSuites.each {
2558         | subSuite |
2559         if subSuite.hasBenchmark(benchmark)
2560           subSuiteJsons[subSuite][benchmark.name] = benchmarkJson
2561         end
2562       }
2563     }
2564     outp.puts
2565     unless suite.subSuites.empty?
2566       suite.subSuites.each {
2567         | subSuite |
2568         outp.puts "#{subSuite.name}:"
2569         allSummaryStats(outp, subSuiteJsons[subSuite], subSuitesOnVMsForSubSuite[subSuite], subSuite.preferredMean, subSuite.decimalShift)
2570         outp.puts
2571       }
2572       outp.puts "#{suite.name} including #{andJoin(suite.subSuites.map{|v| v.name})}:"
2573     end
2574     allSummaryStats(outp, suiteJson, suitesOnVMsForSuite[suite], suite.preferredMean, suite.decimalShift)
2575     outp.puts if $allSuites.size > 1
2576     
2577     json["suites"][suite.name] = suiteJson
2578     suite.subSuites.each {
2579       | subSuite |
2580       json["suites"][subSuite.name] = subSuiteJsons[subSuite]
2581     }
2582   }
2583   
2584   if $suites.size > 1
2585     printVMs(outp)
2586     outp.puts "All benchmarks:"
2587     allSummaryStats(outp, json, vmStatses, nil, 0)
2588     
2589     scaledResultJson = {}
2590     
2591     outp.puts
2592     printVMs(outp)
2593     outp.puts "Geomean of preferred means:"
2594     outp.print "   "
2595     outp.print rpad("<scaled-result>", $benchpad + $weightpad)
2596     outp.print " "
2597     $vms.size.times {
2598       | index |
2599       if index != 0
2600         outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm
2601       end
2602       outp.print statsToStr(overallResults[index], 0)
2603       scaledResultJson[$vms[index].name] = overallResults[index].jsonMap
2604     }
2605     if overallResults.size>=2
2606       outp.print("    "+overallResults[-1].compareTo(overallResults[0]).longForm)
2607     end
2608     outp.puts
2609     
2610     json["<scaled-result>"] = scaledResultJson
2611   end
2612   outp.puts
2613   
2614   if outp != $stdout
2615     outp.close
2616   end
2617   
2618   if outp != $stdout and not $brief
2619     puts
2620     File.open(reportName + "_report.txt") {
2621       | inp |
2622       puts inp.read
2623     }
2624   end
2625   
2626   if $brief
2627     puts(overallResults.collect{|stats| stats.mean}.join("\t"))
2628     puts(overallResults.collect{|stats| stats.confInt}.join("\t"))
2629   end
2630   
2631   File.open(reportName + ".json", "w") {
2632     | outp |
2633     outp.puts json.to_json
2634   }
2635 end
2636
2637 begin
2638   $sawBenchOptions = false
2639   
2640   def resetBenchOptionsIfNecessary
2641     unless $sawBenchOptions
2642       $includeSunSpider = false
2643       $includeLongSpider = false
2644       $includeV8 = false
2645       $includeKraken = false
2646       $includeJSBench = false
2647       $includeJSRegress = false
2648       $includeAsmBench = false
2649       $includeDSPJS = false
2650       $includeBrowsermarkJS = false
2651       $includeBrowsermarkDOM = false
2652       $includeOctane = false
2653       $includeCompressionBench = false
2654       $sawBenchOptions = true
2655     end
2656   end
2657   
2658   GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
2659                  ['--inner', GetoptLong::REQUIRED_ARGUMENT],
2660                  ['--outer', GetoptLong::REQUIRED_ARGUMENT],
2661                  ['--warmup', GetoptLong::REQUIRED_ARGUMENT],
2662                  ['--no-ss-warmup', GetoptLong::NO_ARGUMENT],
2663                  ['--quantum', GetoptLong::REQUIRED_ARGUMENT],
2664                  ['--minimum', GetoptLong::REQUIRED_ARGUMENT],
2665                  ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
2666                  ['--sunspider', GetoptLong::NO_ARGUMENT],
2667                  ['--longspider', GetoptLong::NO_ARGUMENT],
2668                  ['--v8-spider', GetoptLong::NO_ARGUMENT],
2669                  ['--kraken', GetoptLong::NO_ARGUMENT],
2670                  ['--js-bench', GetoptLong::NO_ARGUMENT],
2671                  ['--js-regress', GetoptLong::NO_ARGUMENT],
2672                  ['--asm-bench', GetoptLong::NO_ARGUMENT],
2673                  ['--dsp', GetoptLong::NO_ARGUMENT],
2674                  ['--browsermark-js', GetoptLong::NO_ARGUMENT],
2675                  ['--browsermark-dom', GetoptLong::NO_ARGUMENT],
2676                  ['--octane', GetoptLong::NO_ARGUMENT],
2677                  ['--compression-bench', GetoptLong::NO_ARGUMENT],
2678                  ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
2679                  ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
2680                  ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
2681                  ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
2682                  ['--dont-copy-vms', GetoptLong::NO_ARGUMENT],
2683                  ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
2684                  ['--brief', GetoptLong::NO_ARGUMENT],
2685                  ['--silent', GetoptLong::NO_ARGUMENT],
2686                  ['--remote', GetoptLong::REQUIRED_ARGUMENT],
2687                  ['--local', GetoptLong::NO_ARGUMENT],
2688                  ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
2689                  ['--slave', GetoptLong::NO_ARGUMENT],
2690                  ['--prepare-only', GetoptLong::NO_ARGUMENT],
2691                  ['--analyze', GetoptLong::REQUIRED_ARGUMENT],
2692                  ['--vms', GetoptLong::REQUIRED_ARGUMENT],
2693                  ['--output-name', GetoptLong::REQUIRED_ARGUMENT],
2694                  ['--environment', GetoptLong::REQUIRED_ARGUMENT],
2695                  ['--config', GetoptLong::REQUIRED_ARGUMENT],
2696                  ['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
2697     | opt, arg |
2698     case opt
2699     when '--rerun'
2700       $rerun = intArg(opt,arg,1,nil)
2701     when '--inner'
2702       $inner = intArg(opt,arg,1,nil)
2703     when '--outer'
2704       $outer = intArg(opt,arg,1,nil)
2705     when '--warmup'
2706       $warmup = intArg(opt,arg,0,nil)
2707     when '--no-ss-warmup'
2708       $sunSpiderWarmup = false
2709     when '--quantum'
2710       $quantum = intArg(opt,arg,1,nil)
2711     when '--minimum'
2712       $minimum = intArg(opt,arg,1,nil)
2713     when '--timing-mode'
2714       if arg.upcase == "PRECISETIME"
2715         $timeMode = :preciseTime
2716       elsif arg.upcase == "DATE"
2717         $timeMode = :date
2718       elsif arg.upcase == "AUTO"
2719         $timeMode = :auto
2720       else
2721         quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
2722                   "Invalid argument for command-line option")
2723       end
2724     when '--force-vm-kind'
2725       if arg.upcase == "JSC"
2726         $forceVMKind = :jsc
2727       elsif arg.upcase == "DUMPRENDERTREE"
2728         $forceVMKind = :dumpRenderTree
2729       elsif arg.upcase == "WEBKITTESTRUNNER"
2730         $forceVMKind = :webkitTestRunner
2731       elsif arg.upcase == "AUTO"
2732         $forceVMKind = nil
2733       else
2734         quickFail("Expected 'jsc', 'DumpRenderTree', or 'WebKitTestRunner' for --force-vm-kind, but got '#{arg}'.",
2735                   "Invalid argument for command-line option")
2736       end
2737     when '--force-vm-copy'
2738       $needToCopyVMs = true
2739     when '--dont-copy-vms'
2740       $dontCopyVMs = true
2741     when '--sunspider'
2742       resetBenchOptionsIfNecessary
2743       $includeSunSpider = true
2744     when '--longspider'
2745       resetBenchOptionsIfNecessary
2746       $includeLongSpider = true
2747     when '--v8-spider'
2748       resetBenchOptionsIfNecessary
2749       $includeV8 = true
2750     when '--kraken'
2751       resetBenchOptionsIfNecessary
2752       $includeKraken = true
2753     when '--js-bench'
2754       resetBenchOptionsIfNecessary
2755       $includeJSBench = true
2756     when '--js-regress'
2757       resetBenchOptionsIfNecessary
2758       $includeJSRegress = true
2759     when '--asm-bench'
2760       resetBenchOptionsIfNecessary
2761       $includeAsmBench = true
2762     when '--dsp'
2763       resetBenchOptionsIfNecessary
2764       $includeDSPJS = true
2765     when '--browsermark-js'
2766       resetBenchOptionsIfNecessary
2767       $includeBrowsermarkJS = true
2768     when '--browsermark-dom'
2769       resetBenchOptionsIfNecessary
2770       $includeBrowsermarkDOM = true
2771     when '--octane'
2772       resetBenchOptionsIfNecessary
2773       $includeOctane = true
2774     when '--compression-bench'
2775       resetBenchOptionsIfNecessary
2776       $includeCompressionBench = true
2777     when '--benchmarks'
2778       $benchmarkPattern = Regexp.new(arg)
2779     when '--measure-gc'
2780       if arg == ''
2781         $measureGC = true
2782       else
2783         $measureGC = arg
2784       end
2785     when '--verbose'
2786       $verbosity += 1
2787     when '--brief'
2788       $brief = true
2789     when '--silent'
2790       $silent = true
2791     when '--remote'
2792       $remoteHosts += arg.split(',')
2793       $needToCopyVMs = true
2794     when '--ssh-options'
2795       $sshOptions << arg
2796     when '--local'
2797       $alsoLocal = true
2798     when '--prepare-only'
2799       $run = false
2800     when '--analyze'
2801       $prepare = false
2802       $run = false
2803       $analyze << arg
2804     when '--output-name'
2805       $outputName = arg
2806     when '--vms'
2807       JSON::parse(IO::read(arg)).each {
2808         | vmDescription |
2809         path = Pathname.new(vmDescription["path"]).realpath
2810         if vmDescription["name"]
2811           name = vmDescription["name"]
2812           nameKind = :given
2813         else
2814           name = "Conf\##{$vms.length+1}"
2815           nameKind = :auto
2816         end
2817         vm = VM.new(path, name, nameKind, nil)
2818         if vmDescription["env"]
2819           vmDescription["env"].each_pair {
2820             | key, val |
2821             vm.addExtraEnv(key, val)
2822           }
2823         end
2824         $vms << vm
2825       }
2826     when '--environment'
2827       $environment = JSON::parse(IO::read(arg))
2828     when '--config'
2829       $configPath = Pathname.new(arg)
2830     when '--help'
2831       usage
2832     else
2833       raise "bad option: #{opt}"
2834     end
2835   }
2836   
2837   # Figure out the configuration
2838   if $configPath.file?
2839     config = JSON::parse(IO::read($configPath.to_s))
2840   else
2841     config = {}
2842   end
2843   OCTANE_PATH = config["OctanePath"]
2844   BROWSERMARK_PATH = config["BrowserMarkPath"]
2845   BROWSERMARK_JS_PATH = config["BrowserMarkJSPath"]
2846   BROWSERMARK_DOM_PATH = config["BrowserMarkDOMPath"]
2847   ASMBENCH_PATH = config["AsmBenchPath"]
2848   COMPRESSIONBENCH_PATH = config["CompressionBenchPath"]
2849   DSPJS_FILTRR_PATH = config["DSPJSFiltrrPath"]
2850   DSPJS_ROUTE9_PATH = config["DSPJSRoute9Path"]
2851   DSPJS_STARFIELD_PATH = config["DSPJSStarfieldPath"]
2852   DSPJS_QUAKE3_PATH = config["DSPJSQuake3Path"]
2853   DSPJS_MANDELBROT_PATH = config["DSPJSMandelbrotPath"]
2854   DSPJS_JSLINUX_PATH = config["DSPJSLinuxPath"]
2855   DSPJS_AMMOJS_ASMJS_PATH = config["DSPJSAmmoJSAsmJSPath"]
2856   DSPJS_AMMOJS_REGULAR_PATH = config["DSPJSAmmoJSRegularPath"]
2857   JSBENCH_PATH = config["JSBenchPath"]
2858   KRAKEN_PATH = config["KrakenPath"]
2859   
2860   # If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option.
2861   if $dontCopyVMs
2862     $needToCopyVMs = false
2863   end
2864   
2865   ARGV.each {
2866     | vm |
2867     if vm =~ /([a-zA-Z0-9_ ]+):/
2868       name = $1
2869       nameKind = :given
2870       vm = $~.post_match
2871     else
2872       name = "Conf\##{$vms.length+1}"
2873       nameKind = :auto
2874     end
2875     envs = []
2876     while vm =~ /([a-zA-Z0-9_]+)=([a-zA-Z0-9_:]+):/
2877       envs << [$1, $2]
2878       vm = $~.post_match
2879     end
2880     $stderr.puts "#{name}: #{vm}" if $verbosity >= 1
2881     vm = VM.new(Pathname.new(vm).realpath, name, nameKind, nil)
2882     envs.each {
2883       | pair |
2884       vm.addExtraEnv(pair[0], pair[1])
2885     }
2886     $vms << vm
2887   }
2888   
2889   if $vms.empty?
2890     quickFail("Please specify at least on configuraiton on the command line.",
2891               "Insufficient arguments")
2892   end
2893   
2894   $vms.each {
2895     | vm |
2896     if vm.vmType == :jsc
2897       $allDRT = false
2898     end
2899   }
2900   
2901   SUNSPIDER = BenchmarkSuite.new("SunSpider", :arithmeticMean, 0)
2902   WARMUP = BenchmarkSuite.new("WARMUP", :arithmeticMean, 0)
2903   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
2904    "access-fannkuch", "access-nbody", "access-nsieve",
2905    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
2906    "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
2907    "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
2908    "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
2909    "string-base64", "string-fasta", "string-tagcloud",
2910    "string-unpack-code", "string-validate-input"].each {
2911     | name |
2912     SUNSPIDER.add SunSpiderBenchmark.new(name)
2913     WARMUP.addIgnoringPattern SunSpiderBenchmark.new(name)
2914   }
2915
2916   LONGSPIDER = BenchmarkSuite.new("LongSpider", :geometricMean, 0)
2917   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
2918    "access-fannkuch", "access-nbody", "access-nsieve",
2919    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-nsieve-bits",
2920    "controlflow-recursive", "crypto-aes", "crypto-md5", "crypto-sha1",
2921    "date-format-tofte", "date-format-xparb", "math-cordic",
2922    "math-partial-sums", "math-spectral-norm", "string-base64",
2923    "string-fasta", "string-tagcloud"].each {
2924     | name |
2925     LONGSPIDER.add LongSpiderBenchmark.new(name)
2926   }
2927
2928   V8 = BenchmarkSuite.new("V8Spider", :geometricMean, 0)
2929   ["crypto", "deltablue", "earley-boyer", "raytrace",
2930    "regexp", "richards", "splay"].each {
2931     | name |
2932     V8.add V8Benchmark.new(name)
2933   }
2934
2935   OCTANE = BenchmarkSuite.new("Octane", :geometricMean, 1)
2936   [[["crypto"], "encrypt", 1, true, false, 32],
2937    [["crypto"], "decrypt", 1, true, false, 32],
2938    [["deltablue"], "deltablue", 2, true, false, 32],
2939    [["earley-boyer"], "earley", 1, true, false, 32],
2940    [["earley-boyer"], "boyer", 1, true, false, 32],
2941    [["navier-stokes"], "navier-stokes", 2, true, false, 16],
2942    [["raytrace"], "raytrace", 2, true, false, 32],
2943    [["richards"], "richards", 2, true, false, 32],
2944    [["splay"], "splay", 2, true, false, 32],
2945    [["regexp"], "regexp", 2, false, false, 16],
2946    [["pdfjs"], "pdfjs", 2, false, false, 4],
2947    [["mandreel"], "mandreel", 2, false, false, 4],
2948    [["gbemu-part1", "gbemu-part2"], "gbemu", 2, false, false, 4],
2949    [["code-load"], "closure", 1, false, false, 16],
2950    [["code-load"], "jquery", 1, false, false, 16],
2951    [["box2d"], "box2d", 2, false, false, 8],
2952    [["zlib", "zlib-data"], "zlib", 2, false, true, 3],
2953    [["typescript", "typescript-input", "typescript-compiler"], "typescript", 2, false, true, 1]].each {
2954     | args |
2955     OCTANE.add OctaneBenchmark.new(*args)
2956   }
2957
2958   KRAKEN = BenchmarkSuite.new("Kraken", :arithmeticMean, -1)
2959   ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
2960    "audio-oscillator", "imaging-darkroom", "imaging-desaturate",
2961    "imaging-gaussian-blur", "json-parse-financial",
2962    "json-stringify-tinderbox", "stanford-crypto-aes",
2963    "stanford-crypto-ccm", "stanford-crypto-pbkdf2",
2964    "stanford-crypto-sha256-iterative"].each {
2965     | name |
2966     KRAKEN.add KrakenBenchmark.new(name)
2967   }
2968   
2969   JSBENCH = BenchmarkSuite.new("JSBench", :arithmeticMean, 0)
2970   [["amazon", "urm"], ["facebook", "urem"], ["google", "urem"], ["twitter", "urem"],
2971    ["yahoo", "urem"]].each {
2972     | nameAndMode |
2973     JSBENCH.add JSBenchBenchmark.new(*nameAndMode)
2974   }
2975   
2976   JSREGRESS = BenchmarkSuite.new("JSRegress", :geometricMean, 0)
2977   Dir.foreach(JSREGRESS_PATH) {
2978     | filename |
2979     if filename =~ /\.js$/
2980       name = $~.pre_match
2981       JSREGRESS.add JSRegressBenchmark.new(name)
2982     end
2983   }
2984   
2985   ASMBENCH = BenchmarkSuite.new("AsmBench", :geometricMean, 0)
2986   if JSBENCH_PATH
2987     Dir.foreach(ASMBENCH_PATH) {
2988       | filename |
2989       if filename =~ /\.js$/
2990         name = $~.pre_match
2991         ASMBENCH.add AsmBenchBenchmark.new(name)
2992       end
2993     }
2994   end
2995
2996   COMPRESSIONBENCH = BenchmarkSuite.new("CompressionBench", :geometricMean, 0)
2997   [[["huffman", "compression-data"], "huffman", ""],
2998    [["arithmetic", "compression-data"], "arithmetic", "Simple"],
2999    [["arithmetic", "compression-data"], "arithmetic", "Precise"],
3000    [["arithmetic", "compression-data"], "arithmetic", "Complex Precise"],
3001    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 0"],
3002    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 1"],
3003    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 2"],
3004    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 1"],
3005    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 2"],
3006    [["lz-string", "compression-data"], "lz-string", ""]
3007   ].each {
3008     | args |
3009     COMPRESSIONBENCH.add CompressionBenchBenchmark.new(*args)
3010   }
3011
3012   DSPJS = BenchmarkSuite.new("DSP", :geometricMean, 0)
3013   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-posterize-tint", "e2")
3014   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-contrast-sat-bright", "e5")
3015   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-sat-adj-contr-mult", "e7")
3016   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-blur-overlay-sat-contr", "e8")
3017   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sat-blur-mult-sharpen-contr", "e9")
3018   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sepia-bias", "e10")
3019   DSPJS.add DSPJSVP8Benchmark.new
3020   DSPJS.add DSPStarfieldBenchmark.new
3021   DSPJS.add DSPJSJSLinuxBenchmark.new
3022   DSPJS.add DSPJSQuake3Benchmark.new
3023   DSPJS.add DSPJSMandelbrotBenchmark.new
3024   DSPJS.add DSPJSAmmoJSASMBenchmark.new
3025   DSPJS.add DSPJSAmmoJSRegularBenchmark.new
3026
3027   BROWSERMARK_JS = BenchmarkSuite.new("BrowsermarkJS", :geometricMean, 1)
3028   ["array_blur", "array_weighted", "string_chat", "string_filter", "string_weighted"].each {
3029     | name |
3030       BROWSERMARK_JS.add BrowsermarkJSBenchmark.new(name)
3031   }
3032   
3033   BROWSERMARK_DOM = BenchmarkSuite.new("BrowsermarkDOM", :geometricMean, 1)
3034   ["advanced_search", "create_source", "dynamic_create", "search"].each {
3035     | name |
3036       BROWSERMARK_DOM.add BrowsermarkDOMBenchmark.new(name)
3037   }
3038
3039   $suites = []
3040   
3041   if $includeSunSpider and not SUNSPIDER.empty?
3042     $suites << SUNSPIDER
3043   end
3044   
3045   if $includeLongSpider and not LONGSPIDER.empty?
3046     $suites << LONGSPIDER
3047   end
3048   
3049   if $includeV8 and not V8.empty?
3050     $suites << V8
3051   end
3052   
3053   if $includeOctane and not OCTANE.empty?
3054     if OCTANE_PATH
3055       $suites << OCTANE
3056     else
3057       $stderr.puts "Warning: refusing to run Octane because \"OctanePath\" isn't set in #{$configPath}."
3058     end
3059   end
3060   
3061   if $includeKraken and not KRAKEN.empty?
3062     if KRAKEN_PATH
3063       $suites << KRAKEN
3064     else
3065       $stderr.puts "Warning: refusing to run Kraken because \"KrakenPath\" isn't set in #{$configPath}."
3066     end
3067   end
3068   
3069   if $includeJSBench and not JSBENCH.empty?
3070     if $allDRT
3071       if JSBENCH_PATH
3072         $suites << JSBENCH
3073       else
3074         $stderr.puts "Warning: refusing to run JSBench because \"JSBenchPath\" isn't set in #{$configPath}"
3075       end
3076     else
3077       $stderr.puts "Warning: refusing to run JSBench because not all VMs are DumpRenderTree or WebKitTestRunner."
3078     end
3079   end
3080   
3081   if $includeJSRegress and not JSREGRESS.empty?
3082     $suites << JSREGRESS
3083   end
3084   
3085   if $includeAsmBench and not ASMBENCH.empty?
3086     if ASMBENCH_PATH
3087       $suites << ASMBENCH
3088     else
3089       $stderr.puts "Warning: refusing to run AsmBench because \"AsmBenchPath\" isn't set in #{$configPath}."
3090     end
3091   end
3092   
3093   if $includeDSPJS and not DSPJS.empty?
3094     if $allDRT
3095       if DSPJS_FILTRR_PATH and DSPJS_ROUTE9_PATH and DSPJS_STARFIELD_PATH and DSPJS_QUAKE3_PATH and DSPJS_MANDELBROT_PATH and DSPJS_JSLINUX_PATH and DSPJS_AMMOJS_ASMJS_PATH and DSPJS_AMMOJS_REGULAR_PATH
3096         $suites << DSPJS
3097       else
3098         $stderr.puts "Warning: refusing to run DSPJS because one of the following isn't set in #{$configPath}: \"DSPJSFiltrrPath\", \"DSPJSRoute9Path\", \"DSPJSStarfieldPath\", \"DSPJSQuake3Path\", \"DSPJSMandelbrotPath\", \"DSPJSLinuxPath\", \"DSPJSAmmoJSAsmJSPath\", \"DSPJSAmmoJSRegularPath\"."
3099       end
3100     else
3101       $stderr.puts "Warning: refusing to run DSPJS because not all VMs are DumpRenderTree or WebKitTestRunner."
3102     end
3103   end
3104   
3105   if $includeBrowsermarkJS and not BROWSERMARK_JS.empty?
3106     if BROWSERMARK_PATH and BROWSERMARK_JS_PATH
3107       $suites << BROWSERMARK_JS
3108     else
3109       $stderr.puts "Warning: refusing to run Browsermark-JS because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\" or \"BrowserMarkJSPath\"."
3110     end
3111   end
3112
3113   if $includeBrowsermarkDOM and not BROWSERMARK_DOM.empty?
3114     if $allDRT
3115       if BROWSERMARK_PATH and BROWSERMARK_JS_PATH and BROWSERMARK_DOM_PATH
3116         $suites << BROWSERMARK_DOM
3117       else
3118         $stderr.puts "Warning: refusing to run Browsermark-DOM because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\", \"BrowserMarkJSPath\", or \"BrowserMarkDOMPath\"."
3119       end
3120     else
3121       $stderr.puts "Warning: refusing to run Browsermark-DOM because not all VMs are DumpRenderTree or WebKitTestRunner."
3122     end
3123   end
3124
3125   if $includeCompressionBench and not COMPRESSIONBENCH.empty?
3126     if COMPRESSIONBENCH_PATH
3127       $suites << COMPRESSIONBENCH
3128     else
3129       $stderr.puts "Warning: refusing to run CompressionBench because \"CompressionBenchPath\" isn't set in #{$configPath}"
3130     end
3131   end
3132
3133   $allSuites = $suites.map{|v| v.suites}.flatten(1)
3134   
3135   $benchmarks = []
3136   $suites.each {
3137     | suite |
3138     $benchmarks += suite.benchmarks
3139   }
3140   
3141   if $suites.empty? or $benchmarks.empty?
3142     $stderr.puts "No benchmarks found.  Bailing out."
3143     exit 1
3144   end
3145   
3146   if $outer*$inner == 1
3147     $stderr.puts "Warning: will only collect one sample per benchmark/VM.  Confidence interval calculation will fail."
3148   end
3149   
3150   $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
3151   
3152   $runPlans = []
3153   $vms.each {
3154     | vm |
3155     $benchmarks.each {
3156       | benchmark |
3157       $outer.times {
3158         | iteration |
3159         $runPlans << BenchRunPlan.new(benchmark, vm, iteration)
3160       }
3161     }
3162   }
3163   
3164   $runPlans.shuffle!
3165   
3166   if $sunSpiderWarmup
3167     warmupPlans = []
3168     $vms.each {
3169       | vm |
3170       WARMUP.benchmarks.each {
3171         | benchmark |
3172         warmupPlans << BenchRunPlan.new(benchmark, vm, 0)
3173       }
3174     }
3175     
3176     $runPlans = warmupPlans.shuffle + $runPlans
3177   end
3178   
3179   $suitepad = $suites.collect {
3180     | suite |
3181     suite.to_s.size
3182   }.max + 1
3183   
3184   $planpad = $runPlans.collect {
3185     | plan |
3186     plan.to_s.size
3187   }.max + 1
3188   
3189   maxBenchNameLength =
3190     ($benchmarks + ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
3191     | benchmark |
3192     if benchmark.respond_to? :name
3193       benchmark.name.size
3194     else
3195       benchmark.size
3196     end
3197   }.max
3198   $benchNameClip = 40
3199   $benchpad = [maxBenchNameLength, $benchNameClip].min + 1
3200   
3201   $weightpad = $benchmarks.collect {
3202     | benchmark |
3203     benchmark.weightString.size
3204   }.max
3205
3206   $vmpad = $vms.collect {
3207     | vm |
3208     vm.to_s.size
3209   }.max + 1
3210   
3211   $analyze.each_with_index {
3212     | filename, index |
3213     if index >= 1
3214       puts
3215     end
3216     parseAndDisplayResults(IO::read(filename))
3217   }
3218   
3219   if not $prepare and not $run
3220     exit 0
3221   end
3222   
3223   if FileTest.exist? BENCH_DATA_PATH
3224     cmd = "rm -rf #{BENCH_DATA_PATH}"
3225     $stderr.puts ">> #{cmd}" if $verbosity >= 2
3226     raise unless system cmd
3227   end
3228   
3229   Dir.mkdir BENCH_DATA_PATH
3230   
3231   if $needToCopyVMs
3232     canCopyIntoBenchPath = true
3233     $vms.each {
3234       | vm |
3235       canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath
3236     }
3237     
3238     if canCopyIntoBenchPath
3239       $vms.each {
3240         | vm |
3241         $stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..."
3242         vm.copyIntoBenchPath
3243       }
3244       $stderr.puts "All VMs are in place."
3245     else
3246       $stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it."
3247     end
3248   end
3249   
3250   if $measureGC and $measureGC != true
3251     found = false
3252     $vms.each {
3253       | vm |
3254       if vm.name == $measureGC
3255         found = true
3256       end
3257     }
3258     unless found
3259       $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
3260     end
3261   end
3262   
3263   if $prepare
3264     File.open("#{BENCH_DATA_PATH}/runscript", "w") {
3265       | file |
3266       file.puts "echo \"HOSTNAME:\\c\""
3267       file.puts "hostname"
3268       file.puts "echo"
3269       file.puts "echo \"HARDWARE:\\c\""
3270       file.puts "/usr/sbin/sysctl hw.model"
3271       file.puts "echo"
3272       file.puts "set -e"
3273       $script = file
3274       $runPlans.each_with_index {
3275         | plan, idx |
3276         if $verbosity == 0 and not $silent
3277           text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s
3278           text2 = plan.to_s
3279           file.puts("echo " + Shellwords.shellescape("\r#{text1} #{rpad(text2,$planpad)}") + "\"\\c\" 1>&2")
3280           file.puts("echo " + Shellwords.shellescape("\r#{text1} #{text2}") + "\"\\c\" 1>&2")
3281         end
3282         plan.emitRunCode
3283       }
3284       if $verbosity == 0 and not $silent
3285         file.puts("echo " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}") + "\"\\c\" 1>&2")
3286         file.puts("echo " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size}") + " 1>&2")
3287       end
3288     }
3289   end
3290   
3291   if $run
3292     unless $remoteHosts.empty?
3293       $stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0
3294       Dir.chdir(TEMP_PATH) {
3295         cmd = "tar -czf payload.tar.gz benchdata"
3296         $stderr.puts ">> #{cmd}" if $verbosity>=2
3297         raise unless system(cmd)
3298       }
3299       
3300       def grokHost(host)
3301         if host =~ /:([0-9]+)$/
3302           "-p " + $1 + " " + Shellwords.shellescape($~.pre_match)
3303         else
3304           Shellwords.shellescape(host)
3305         end
3306       end
3307       
3308       def sshRead(host, command)
3309         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3310         $stderr.puts ">> #{cmd}" if $verbosity>=2
3311         result = ""
3312         IO.popen(cmd, "r") {
3313           | inp |
3314           inp.each_line {
3315             | line |
3316             $stderr.puts "#{host}: #{line}" if $verbosity>=2
3317             result += line
3318           }
3319         }
3320         raise "#{$?}" unless $?.success?
3321         result
3322       end
3323       
3324       def sshWrite(host, command, data)
3325         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3326         $stderr.puts ">> #{cmd}" if $verbosity>=2
3327         IO.popen(cmd, "w") {
3328           | outp |
3329           outp.write(data)
3330         }
3331         raise "#{$?}" unless $?.success?
3332       end
3333       
3334       $remoteHosts.each {
3335         | host |
3336         $stderr.puts "Sending benchmark payload to #{host}..." if $verbosity==0
3337         
3338         remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"]
3339         raise unless remoteTempPath
3340         
3341         sshWrite(host, "cd #{Shellwords.shellescape(remoteTempPath)} && rm -rf benchdata && tar -xz", IO::read("#{TEMP_PATH}/payload.tar.gz"))
3342         
3343         $stderr.puts "Running on #{host}..." if $verbosity==0
3344         
3345         parseAndDisplayResults(sshRead(host, "cd #{Shellwords.shellescape(remoteTempPath + '/benchdata')} && sh runscript"))
3346       }
3347     end
3348     
3349     if not $remoteHosts.empty? and $alsoLocal
3350       $stderr.puts "Running locally..."
3351     end
3352     
3353     if $remoteHosts.empty? or $alsoLocal
3354       parseAndDisplayResults(runAndGetResults)
3355     end
3356   end
3357   
3358   if $prepare and not $run and $analyze.empty?
3359     puts wrap("Benchmarking script and data are in #{BENCH_DATA_PATH}. You can run "+
3360               "the benchmarks and get the results by doing:", 78)
3361     puts
3362     puts "cd #{BENCH_DATA_PATH}"
3363     puts "sh runscript > results.txt"
3364     puts
3365     puts wrap("Then you can analyze the results by running bencher with the same arguments "+
3366               "as now, but replacing --prepare-only with --analyze results.txt.", 78)
3367   end
3368 rescue => e
3369   fail(e)
3370 end
3371