Unreviewed build fix. Don't try to report stats for all benchmarks.
[WebKit-https.git] / Tools / Scripts / run-jsc-benchmarks
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2011, 2012, 2013, 2014 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 # 1. Redistributions of source code must retain the above copyright
9 #    notice, this list of conditions and the following disclaimer.
10 # 2. Redistributions in binary form must reproduce the above copyright
11 #    notice, this list of conditions and the following disclaimer in the
12 #    documentation and/or other materials provided with the distribution.
13 #
14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24 # THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'rubygems'
27
28 require 'getoptlong'
29 require 'pathname'
30 require 'shellwords'
31 require 'socket'
32
33 begin
34   require 'json'
35 rescue LoadError => e
36   $stderr.puts "It does not appear that you have the 'json' package installed.  Try running 'sudo gem install json'."
37   exit 1
38 end
39
40 SCRIPT_PATH = Pathname.new(__FILE__).realpath
41
42 raise unless SCRIPT_PATH.dirname.basename.to_s == "Scripts"
43 raise unless SCRIPT_PATH.dirname.dirname.basename.to_s == "Tools"
44
45 OPENSOURCE_PATH = SCRIPT_PATH.dirname.dirname.dirname
46
47 SUNSPIDER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "SunSpider" + "tests" + "sunspider-1.0"
48 LONGSPIDER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "LongSpider"
49 V8_PATH = OPENSOURCE_PATH + "PerformanceTests" + "SunSpider" + "tests" + "v8-v6"
50 JSREGRESS_PATH = OPENSOURCE_PATH + "LayoutTests" + "js" + "regress" + "script-tests"
51 OCTANE_WRAPPER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "Octane" + "wrappers"
52
53 TEMP_PATH = OPENSOURCE_PATH + "BenchmarkTemp"
54
55 if TEMP_PATH.exist?
56   raise unless TEMP_PATH.directory?
57 else
58   Dir.mkdir(TEMP_PATH)
59 end
60
61 BENCH_DATA_PATH = TEMP_PATH + "benchdata"
62
63 IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933, 
64             0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684, 
65             0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227, 
66             0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674, 
67             0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652, 
68             0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604, 
69             0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338, 
70             0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538, 
71             0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185, 
72             0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694, 
73             0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244, 
74             0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067, 
75             0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323, 
76             0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129, 
77             0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574, 
78             0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722, 
79             0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624, 
80             0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321, 
81             0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843, 
82             0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217, 
83             0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464, 
84             0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599, 
85             0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638, 
86             0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592, 
87             0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471, 
88             0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283, 
89             0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037, 
90             0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738, 
91             0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481, 
92             0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085, 
93             0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651, 
94             0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183, 
95             0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683, 
96             0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154, 
97             0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599, 
98             0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019, 
99             0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417, 
100             0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795, 
101             0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153, 
102             0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494, 
103             0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818, 
104             0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128, 
105             0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423, 
106             0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704, 
107             0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974, 
108             0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232, 
109             0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479, 
110             0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716, 
111             0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943, 
112             0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192, 
113             0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401, 
114             0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963, 
115             0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822, 
116             0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007, 
117             0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186, 
118             0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358, 
119             0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525, 
120             0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686, 
121             0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841, 
122             0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991, 
123             0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137, 
124             0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278, 
125             0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414, 
126             0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547, 
127             0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675, 
128             0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799, 
129             0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192, 
130             0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037, 
131             0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151, 
132             0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262, 
133             0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369, 
134             0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489, 
135             0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259, 
136             0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689, 
137             0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785, 
138             0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879, 
139             0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297, 
140             0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059, 
141             0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145, 
142             0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242, 
143             0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324, 
144             0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404, 
145             0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483, 
146             0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559, 
147             0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634, 
148             0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707, 
149             0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778, 
150             0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848, 
151             0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916, 
152             0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983, 
153             0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057, 
154             0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121, 
155             0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183, 
156             0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244, 
157             0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304, 
158             0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363, 
159             0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442, 
160             0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476, 
161             0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539, 
162             0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946, 
163             0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653, 
164             0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712, 
165             0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762, 
166             0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812, 
167             0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867, 
168             0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915, 
169             0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962, 
170             0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007, 
171             0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059, 
172             0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103, 
173             0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146, 
174             0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189, 
175             0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231, 
176             0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272, 
177             0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312, 
178             0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352, 
179             0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391, 
180             0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429, 
181             0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467, 
182             0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504, 
183             0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546, 
184             0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582, 
185             0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617, 
186             0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652, 
187             0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686, 
188             0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724, 
189             0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757, 
190             0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579, 
191             0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822, 
192             0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853, 
193             0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885, 
194             0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915, 
195             0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595, 
196             0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979, 
197             0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013, 
198             0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041, 
199             0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074, 
200             0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102, 
201             0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129, 
202             0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156, 
203             0.99616, 0.996164]
204
205 # Run-time configuration parameters (can be set with command-line options)
206
207 $rerun=1
208 $inner=1
209 $warmup=1
210 $outer=4
211 $quantum=1000
212 $includeSunSpider=true
213 $includeLongSpider=true
214 $includeV8=true
215 $includeKraken=true
216 $includeJSBench=true
217 $includeJSRegress=true
218 $includeAsmBench=true
219 $includeDSPJS=true
220 $includeBrowsermarkJS=false
221 $includeBrowsermarkDOM=false
222 $includeOctane=true
223 $includeCompressionBench = true
224 $measureGC=false
225 $benchmarkPattern=nil
226 $verbosity=0
227 $timeMode=:preciseTime
228 $forceVMKind=nil
229 $brief=false
230 $silent=false
231 $remoteHosts=[]
232 $alsoLocal=false
233 $sshOptions=[]
234 $vms = []
235 $environment = {}
236 $needToCopyVMs = false
237 $dontCopyVMs = false
238 $allDRT = true
239 $outputName = nil
240 $sunSpiderWarmup = true
241 $configPath = Pathname.new(ENV["HOME"]) + ".run-jsc-benchmarks"
242
243 $prepare = true
244 $run = true
245 $analyze = []
246
247 # Helpful functions and classes
248
249 def smallUsage
250   puts "Use the --help option to get basic usage information."
251   exit 1
252 end
253
254 def usage
255   puts "run-jsc-benchmarks [options] <vm1> [<vm2> ...]"
256   puts
257   puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
258   puts "benchmarks, and reports detailed statistics.  What makes run-jsc-benchmarks"
259   puts "special is that each benchmark/VM configuration is run in a single VM invocation,"
260   puts "and the invocations are run in random order.  This minimizes systematics due to"
261   puts "one benchmark polluting the running time of another.  The fine-grained"
262   puts "interleaving of VM invocations further minimizes systematics due to changes in"
263   puts "the performance or behavior of your machine."
264   puts 
265   puts "Run-jsc-benchmarks is highly configurable.  You can compare as many VMs as you"
266   puts "like.  You can change the amount of warm-up iterations, number of iterations"
267   puts "executed per VM invocation, and the number of VM invocations per benchmark."
268   puts
269   puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
270   puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
271   puts "the executable and <name> is the name that you would like to give the"
272   puts "configuration for the purposeof reporting.  If no name is given, a generic name"
273   puts "of the form Conf#<n> will be ascribed to the configuration automatically."
274   puts
275   puts "It's also possible to specify per-VM environment variables. For example, you"
276   puts "might specify a VM like Foo:JSC_useJIT=false:/path/to/jsc, in which case the"
277   puts "harness will set the JSC_useJIT environment variable to false just before running"
278   puts "the given VM. Note that the harness will not unset the environment variable, so"
279   puts "you must ensure that your other VMs will use the opposite setting"
280   puts "(JSC_useJIT=true in this case)."
281   puts
282   puts "Options:"
283   puts "--rerun <n>          Set the number of iterations of the benchmark that"
284   puts "                     contribute to the measured run time.  Default is #{$rerun}."
285   puts "--inner <n>          Set the number of inner (per-runtime-invocation)"
286   puts "                     iterations.  Default is #{$inner}."
287   puts "--outer <n>          Set the number of runtime invocations for each benchmark."
288   puts "                     Default is #{$outer}."
289   puts "--warmup <n>         Set the number of warm-up runs per invocation.  Default"
290   puts "                     is #{$warmup}. This has a different effect on different kinds"
291   puts "                     benchmarks. Some benchmarks have no notion of warm-up."
292   puts "--no-ss-warmup       Disable SunSpider-based warm-up runs."
293   puts "--quantum <n>        Set the duration in milliseconds for which an iteration of"
294   puts "                     a throughput benchmark should be run.  Default is #{$quantum}."
295   puts "--timing-mode        Set the way that time is measured.  Possible values"
296   puts "                     are 'preciseTime' and 'date'.  Default is 'preciseTime'."
297   puts "--force-vm-kind      Turn off auto-detection of VM kind, and assume that it is"
298   puts "                     the one specified.  Valid arguments are 'jsc', "
299   puts "                     'DumpRenderTree', or 'WebKitTestRunner'."
300   puts "--force-vm-copy      Force VM builds to be copied to the working directory."
301   puts "                     This may reduce pathologies resulting from path names."
302   puts "--dont-copy-vms      Don't copy VMs even when doing a remote benchmarking run;"
303   puts "                     instead assume that they are already there."
304   puts "--sunspider          Only run SunSpider."
305   puts "--v8-spider          Only run V8."
306   puts "--kraken             Only run Kraken."
307   puts "--js-bench           Only run JSBench."
308   puts "--js-regress         Only run JSRegress."
309   puts "--dsp                Only run DSP."
310   puts "--asm-bench          Only run AsmBench."
311   puts "--browsermark-js     Only run browsermark-js."
312   puts "--browsermark-dom    Only run browsermark-dom."
313   puts "--octane             Only run Octane."
314   puts "--compression-bench  Only run compression bench"
315   puts "                     The default is to run all benchmarks. The above options can"
316   puts "                     be combined to run any subset (so --sunspider --dsp will run"
317   puts "                     both SunSpider and DSP)."
318   puts "--benchmarks         Only run benchmarks matching the given regular expression."
319   puts "--measure-gc         Turn off manual calls to gc(), so that GC time is measured."
320   puts "                     Works best with large values of --inner.  You can also say"
321   puts "                     --measure-gc <conf>, which turns this on for one"
322   puts "                     configuration only."
323   puts "--verbose or -v      Print more stuff."
324   puts "--brief              Print only the final result for each VM."
325   puts "--silent             Don't print progress. This might slightly reduce some"
326   puts "                     performance perturbation."
327   puts "--remote <sshhosts>  Perform performance measurements remotely, on the given"
328   puts "                     SSH host(s). Easiest way to use this is to specify the SSH"
329   puts "                     user@host string. However, you can also supply a comma-"
330   puts "                     separated list of SSH hosts. Alternatively, you can use this"
331   puts "                     option multiple times to specify multiple hosts. This"
332   puts "                     automatically copies the WebKit release builds of the VMs"
333   puts "                     you specified to all of the hosts."
334   puts "--ssh-options        Pass additional options to SSH."
335   puts "--local              Also do a local benchmark run even when doing --remote."
336   puts "--vms                Use a JSON file to specify which VMs to run, as opposed to"
337   puts "                     specifying them on the command line."
338   puts "--prepare-only       Only prepare the runscript (a shell script that"
339   puts "                     invokes the VMs to run benchmarks) but don't run it."
340   puts "--analyze            Only read the output of the runscript but don't do anything"
341   puts "                     else. This requires passing the same arguments that you"
342   puts "                     passed when running --prepare-only."
343   puts "--output-name        Base of the filenames to put results into. Will write a file"
344   puts "                     called <base>_report.txt and <base>.json.  By default this"
345   puts "                     name is automatically synthesized from the machine name,"
346   puts "                     date, set of benchmarks run, and set of configurations."
347   puts "--environment        JSON file that specifies the environment variables that should"
348   puts "                     be used for particular VMs and benchmarks."
349   puts "--config <path>      Specify the path of the configuration file. Defaults to"
350   puts "                     ~/.run-jsc-benchmarks"
351   puts "--help or -h         Display this message."
352   puts
353   puts "Example:"
354   puts "run-jsc-benchmarks TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
355   exit 1
356 end
357
358 def fail(reason)
359   if reason.respond_to? :backtrace
360     puts "FAILED: #{reason.inspect}"
361     puts "Stack trace:"
362     puts reason.backtrace.join("\n")
363   else
364     puts "FAILED: #{reason.inspect}"
365   end
366   smallUsage
367 end
368
369 def quickFail(r1,r2)
370   $stderr.puts "#{$0}: #{r1}"
371   puts
372   fail(r2)
373 end
374
375 def intArg(argName,arg,min,max)
376   result=arg.to_i
377   unless result.to_s == arg
378     quickFail("Expected an integer value for #{argName}, but got #{arg}.",
379               "Invalid argument for command-line option")
380   end
381   if min and result<min
382     quickFail("Argument for #{argName} cannot be smaller than #{min}.",
383               "Invalid argument for command-line option")
384   end
385   if max and result>max
386     quickFail("Argument for #{argName} cannot be greater than #{max}.",
387               "Invalid argument for command-line option")
388   end
389   result
390 end
391
392 def computeMean(array)
393   sum=0.0
394   array.each {
395     | value |
396     sum += value
397   }
398   sum/array.length
399 end
400
401 def computeGeometricMean(array)
402   sum = 0.0
403   array.each {
404     | value |
405     sum += Math.log(value)
406   }
407   Math.exp(sum * (1.0/array.length))
408 end
409
410 def computeHarmonicMean(array)
411   1.0 / computeMean(array.collect{ | value | 1.0 / value })
412 end
413
414 def computeStdDev(array)
415   case array.length
416   when 0
417     0.0/0.0
418   when 1
419     0.0
420   else
421     begin
422       mean=computeMean(array)
423       sum=0.0
424       array.each {
425         | value |
426         sum += (value-mean)**2
427       }
428       Math.sqrt(sum/(array.length-1))
429     rescue
430       0.0/0.0
431     end
432   end
433 end
434
435 class Array
436   def shuffle!
437     size.downto(1) { |n| push delete_at(rand(n)) }
438     self
439   end
440 end
441
442 def inverseBetaRegularized(n)
443   IBR_LOOKUP[n-1]
444 end
445
446 def numToStr(num, decimalShift)
447   ("%." + (4 + decimalShift).to_s + "f") % (num.to_f)
448 end
449
450 class CantSay
451   def initialize
452   end
453   
454   def shortForm
455     " "
456   end
457   
458   def longForm
459     ""
460   end
461   
462   def to_s
463     ""
464   end
465 end
466   
467 class NoChange
468   attr_reader :amountFaster
469   
470   def initialize(amountFaster)
471     @amountFaster = amountFaster
472   end
473   
474   def shortForm
475     " "
476   end
477   
478   def longForm
479     "  might be #{numToStr(@amountFaster, 0)}x faster"
480   end
481   
482   def to_s
483     if @amountFaster < 1.01
484       ""
485     else
486       longForm
487     end
488   end
489 end
490
491 class Faster
492   attr_reader :amountFaster
493   
494   def initialize(amountFaster)
495     @amountFaster = amountFaster
496   end
497   
498   def shortForm
499     "^"
500   end
501   
502   def longForm
503     "^ definitely #{numToStr(@amountFaster, 0)}x faster"
504   end
505   
506   def to_s
507     longForm
508   end
509 end
510
511 class Slower
512   attr_reader :amountSlower
513   
514   def initialize(amountSlower)
515     @amountSlower = amountSlower
516   end
517   
518   def shortForm
519     "!"
520   end
521   
522   def longForm
523     "! definitely #{numToStr(@amountSlower, 0)}x slower"
524   end
525   
526   def to_s
527     longForm
528   end
529 end
530
531 class MayBeSlower
532   attr_reader :amountSlower
533   
534   def initialize(amountSlower)
535     @amountSlower = amountSlower
536   end
537   
538   def shortForm
539     "?"
540   end
541   
542   def longForm
543     "? might be #{numToStr(@amountSlower, 0)}x slower"
544   end
545   
546   def to_s
547     if @amountSlower < 1.01
548       "?"
549     else
550       longForm
551     end
552   end
553 end
554
555 def jsonSanitize(value)
556   if value.is_a? Fixnum
557     value
558   elsif value.is_a? Float
559     if value.nan? or value.infinite?
560       value.to_s
561     else
562       value
563     end
564   elsif value.is_a? Array
565     value.map{|v| jsonSanitize(v)}
566   elsif value.nil?
567     value
568   else
569     raise "Unrecognized value #{value.inspect}"
570   end
571 end
572
573 class Stats
574   def initialize
575     @array = []
576   end
577   
578   def add(value)
579     if not value or not @array
580       @array = nil
581     elsif value.is_a? Float
582       if value.nan? or value.infinite?
583         @array = nil
584       else
585         @array << value
586       end
587     elsif value.is_a? Stats
588       add(value.array)
589     elsif value.respond_to? :each
590       value.each {
591         | v |
592         add(v)
593       }
594     else
595       @array << value.to_f
596     end
597   end
598   
599   def status
600     if @array
601       :ok
602     else
603       :error
604     end
605   end
606   
607   def error?
608     # TODO: We're probably still not handling this case correctly. 
609     not @array or @array.empty?
610   end
611   
612   def ok?
613     not not @array
614   end
615     
616   def array
617     @array
618   end
619   
620   def sum
621     result=0
622     @array.each {
623       | value |
624       result += value
625     }
626     result
627   end
628   
629   def min
630     @array.min
631   end
632   
633   def max
634     @array.max
635   end
636   
637   def size
638     @array.length
639   end
640   
641   def mean
642     computeMean(array)
643   end
644   
645   def arithmeticMean
646     mean
647   end
648   
649   def stdDev
650     computeStdDev(array)
651   end
652
653   def stdErr
654     stdDev/Math.sqrt(size)
655   end
656   
657   # Computes a 95% Student's t distribution confidence interval
658   def confInt
659     if size < 2
660       0.0/0.0
661     else
662       raise if size > 1000
663       Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
664     end
665   end
666   
667   def lower
668     mean-confInt
669   end
670   
671   def upper
672     mean+confInt
673   end
674   
675   def geometricMean
676     computeGeometricMean(array)
677   end
678   
679   def harmonicMean
680     computeHarmonicMean(array)
681   end
682   
683   def compareTo(other)
684     return CantSay.new unless ok? and other.ok?
685     
686     if upper < other.lower
687       Faster.new(other.mean/mean)
688     elsif lower > other.upper
689       Slower.new(mean/other.mean)
690     elsif mean > other.mean
691       MayBeSlower.new(mean/other.mean)
692     else
693       NoChange.new(other.mean/mean)
694     end
695   end
696   
697   def to_s
698     "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
699   end
700   
701   def jsonMap
702     if ok?
703       {"data"=>jsonSanitize(@array), "mean"=>jsonSanitize(mean), "confInt"=>jsonSanitize(confInt)}
704     else
705       "ERROR"
706     end
707   end
708 end
709
710 def doublePuts(out1,out2,msg)
711   out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
712   out2.puts msg
713 end
714
715 class Benchfile < File
716   @@counter = 0
717   
718   attr_reader :filename, :basename
719   
720   def initialize(name)
721     @basename, @filename = Benchfile.uniqueFilename(name)
722     super(@filename, "w")
723   end
724   
725   def self.uniqueFilename(name)
726     if name.is_a? Array
727       basename = name[0] + @@counter.to_s + name[1]
728     else
729       basename = name + @@counter.to_s
730     end
731     filename = BENCH_DATA_PATH + basename
732     @@counter += 1
733     raise "Benchfile #{filename} already exists" if FileTest.exist?(filename)
734     [basename, filename]
735   end
736   
737   def self.create(name)
738     file = Benchfile.new(name)
739     yield file
740     file.close
741     file.basename
742   end
743 end
744
745 $dataFiles={}
746 def ensureFile(key, filename)
747   unless $dataFiles[key]
748     $dataFiles[key] = Benchfile.create(key) {
749       | outp |
750       doublePuts($stderr,outp,IO::read(filename))
751     }
752   end
753   $dataFiles[key]
754 end
755
756 # Helper for files that cannot be renamed.
757 $absoluteFiles={}
758 def ensureAbsoluteFile(filename, basedir=nil)
759   return if $absoluteFiles[filename]
760   filename = Pathname.new(filename)
761
762   directory = Pathname.new('')
763   if basedir and filename.dirname != basedir
764     remainingPath = filename.dirname
765     while remainingPath != basedir
766       directory = remainingPath.basename + directory
767       remainingPath = remainingPath.dirname
768     end
769     if not $absoluteFiles[directory]
770       cmd = "mkdir -p #{Shellwords.shellescape((BENCH_DATA_PATH + directory).to_s)}"
771       $stderr.puts ">> #{cmd}" if $verbosity >= 2
772       raise unless system(cmd)
773       intermediateDirectory = Pathname.new(directory)
774       while intermediateDirectory.basename.to_s != "."
775         $absoluteFiles[intermediateDirectory] = true
776         intermediateDirectory = intermediateDirectory.dirname
777       end
778     end
779   end
780   
781   cmd = "cp #{Shellwords.shellescape(filename.to_s)} #{Shellwords.shellescape((BENCH_DATA_PATH + directory + filename.basename).to_s)}"
782   $stderr.puts ">> #{cmd}" if $verbosity >= 2
783   raise unless system(cmd)
784   $absoluteFiles[filename] = true
785 end
786
787 # Helper for large benchmarks with lots of files and directories.
788 def ensureBenchmarkFiles(rootdir)
789     toProcess = [rootdir]
790     while not toProcess.empty?
791       currdir = toProcess.pop
792       Dir.foreach(currdir.to_s) {
793         | filename |
794         path = currdir + filename
795         next if filename.match(/^\./)
796         toProcess.push(path) if File.directory?(path.to_s)
797         ensureAbsoluteFile(path, rootdir) if File.file?(path.to_s)
798       }
799     end
800 end
801
802 class JSCommand
803   attr_reader :js, :html
804   def initialize(js, html)
805     @js = js
806     @html = html
807   end
808 end
809
810 def loadCommandForFile(key, filename)
811   file = ensureFile(key, filename)
812   JSCommand.new("load(#{file.inspect});", "<script src=#{file.inspect}></script>")
813 end
814
815 def simpleCommand(command)
816   JSCommand.new(command, "<script type=\"text/javascript\">#{command}</script>")
817 end
818
819 # Benchmark that consists of a single file and must be loaded in its own global object each
820 # time (i.e. run()).
821 class SingleFileTimedBenchmarkParameters
822   attr_reader :benchPath
823   
824   def initialize(benchPath)
825     @benchPath = benchPath
826   end
827   
828   def kind
829     :singleFileTimedBenchmark
830   end
831 end
832
833 # Benchmark that consists of one or more data files that should be loaded globally, followed
834 # by a command to run the benchmark.
835 class MultiFileTimedBenchmarkParameters
836   attr_reader :dataPaths, :command
837
838   def initialize(dataPaths, command)
839     @dataPaths = dataPaths
840     @command = command
841   end
842   
843   def kind
844     :multiFileTimedBenchmark
845   end
846 end
847
848 # Benchmark that consists of one or more data files that should be loaded globally, followed
849 # by a command to run a short tick of the benchmark. The benchmark should be run for as many
850 # ticks as possible, for one quantum (quantum is 1000ms by default).
851 class ThroughputBenchmarkParameters
852   attr_reader :dataPaths, :setUpCommand, :command, :tearDownCommand, :doWarmup, :deterministic, :minimumIterations
853
854   def initialize(dataPaths, setUpCommand, command, tearDownCommand, doWarmup, deterministic, minimumIterations)
855     @dataPaths = dataPaths
856     @setUpCommand = setUpCommand
857     @command = command
858     @tearDownCommand = tearDownCommand
859     @doWarmup = doWarmup
860     @deterministic = deterministic
861     @minimumIterations = minimumIterations
862   end
863   
864   def kind
865     :throughputBenchmark
866   end
867 end
868
869 # Benchmark that can only run in DumpRenderTree or WebKitTestRunner, that has its own callback for reporting
870 # results. Other than that it's just like SingleFileTimedBenchmark.
871 class SingleFileTimedCallbackBenchmarkParameters
872   attr_reader :callbackDecl, :benchPath
873   
874   def initialize(callbackDecl, benchPath)
875     @callbackDecl = callbackDecl
876     @benchPath = benchPath
877   end
878   
879   def kind
880     :singleFileTimedCallbackBenchmark
881   end
882 end
883
884 def emitTimerFunctionCode(file)
885   case $timeMode
886   when :preciseTime
887     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
888     doublePuts($stderr,file,"   return preciseTime()*1000")
889     doublePuts($stderr,file,"}")
890   when :date
891     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
892     doublePuts($stderr,file,"   return Date.now()")
893     doublePuts($stderr,file,"}")
894   else
895     raise
896   end
897 end
898
899 def emitBenchRunCodeFile(name, plan, benchParams)
900   case plan.vm.vmType
901   when :jsc
902     Benchfile.create("bencher") {
903       | file |
904       emitTimerFunctionCode(file)
905       
906       if benchParams.kind == :multiFileTimedBenchmark
907         benchParams.dataPaths.each {
908           | path |
909           doublePuts($stderr,file,"load(#{path.inspect});")
910         }
911         doublePuts($stderr,file,"gc();")
912         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
913         doublePuts($stderr,file,"   var __before = __bencher_curTimeMS();")
914         $rerun.times {
915           doublePuts($stderr,file,"   #{benchParams.command.js}")
916         }
917         doublePuts($stderr,file,"   var __after = __bencher_curTimeMS();")
918         doublePuts($stderr,file,"   if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(__after-__before));");
919         doublePuts($stderr,file,"   gc();") unless plan.vm.shouldMeasureGC
920         doublePuts($stderr,file,"}")
921       elsif benchParams.kind == :throughputBenchmark
922         emitTimerFunctionCode(file)
923         benchParams.dataPaths.each {
924           | path |
925           doublePuts($stderr,file,"load(#{path.inspect});")
926         }
927         doublePuts($stderr,file,"#{benchParams.setUpCommand.js}")
928         if benchParams.doWarmup
929           warmup = $warmup
930         else
931           warmup = 0
932         end
933         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{warmup + $inner}; __bencher_index++) {")
934         doublePuts($stderr,file,"    var __before = __bencher_curTimeMS();")
935         doublePuts($stderr,file,"    var __after = __before;")
936         doublePuts($stderr,file,"    var __runs = 0;")
937         doublePuts($stderr,file,"    var __expected = #{$quantum};")
938         doublePuts($stderr,file,"    while (true) {")
939         $rerun.times {
940           doublePuts($stderr,file,"       #{benchParams.command.js}")
941         }
942         doublePuts($stderr,file,"       __runs++;")
943         doublePuts($stderr,file,"       __after = __bencher_curTimeMS();")
944         if benchParams.deterministic
945           doublePuts($stderr,file,"       if (true) {")
946         else
947           doublePuts($stderr,file,"       if (__after - __before >= __expected) {")
948         end
949         doublePuts($stderr,file,"           if (__runs >= #{benchParams.minimumIterations} || __bencher_index < #{warmup})")
950         doublePuts($stderr,file,"               break;")
951         doublePuts($stderr,file,"           __expected += #{$quantum}")
952         doublePuts($stderr,file,"       }")
953         doublePuts($stderr,file,"    }")
954         doublePuts($stderr,file,"    if (__bencher_index >= #{warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{warmup}) + \": Time: \"+((__after-__before)/__runs));")
955         doublePuts($stderr,file,"}")
956         doublePuts($stderr,file,"#{benchParams.tearDownCommand.js}")
957       else
958         raise unless benchParams.kind == :singleFileTimedBenchmark
959         doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
960         doublePuts($stderr,file,"   var __bencher_before = __bencher_curTimeMS();")
961         $rerun.times {
962           doublePuts($stderr,file,"   run(__bencher_what);")
963         }
964         doublePuts($stderr,file,"   var __bencher_after = __bencher_curTimeMS();")
965         doublePuts($stderr,file,"   return __bencher_after - __bencher_before;")
966         doublePuts($stderr,file,"}")
967         $warmup.times {
968           doublePuts($stderr,file,"__bencher_run(#{benchParams.benchPath.inspect})")
969           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
970         }
971         $inner.times {
972           | innerIndex |
973           doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchParams.benchPath.inspect}));")
974           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
975         }
976       end
977     }
978   when :dumpRenderTree, :webkitTestRunner
979     case $timeMode
980     when :preciseTime
981       curTime = "(testRunner.preciseTime()*1000)"
982     when :date
983       curTime = "(Date.now())"
984     else
985       raise
986     end
987
988     mainCode = Benchfile.create("bencher") {
989       | file |
990       doublePuts($stderr,file,"__bencher_count = 0;")
991       doublePuts($stderr,file,"function __bencher_doNext(result) {")
992       doublePuts($stderr,file,"    if (__bencher_count >= #{$warmup})")
993       doublePuts($stderr,file,"        debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);")
994       doublePuts($stderr,file,"    __bencher_count++;")
995       doublePuts($stderr,file,"    if (__bencher_count < #{$inner+$warmup})")
996       doublePuts($stderr,file,"        __bencher_runImpl(__bencher_doNext);")
997       doublePuts($stderr,file,"    else")
998       doublePuts($stderr,file,"        quit();")
999       doublePuts($stderr,file,"}")
1000       doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);")
1001     }
1002     
1003     cssCode = Benchfile.create("bencher-css") {
1004       | file |
1005       doublePuts($stderr,file,".pass {\n    font-weight: bold;\n    color: green;\n}\n.fail {\n    font-weight: bold;\n    color: red;\n}\n\#console {\n    white-space: pre-wrap;\n    font-family: monospace;\n}")
1006     }
1007     
1008     preCode = Benchfile.create("bencher-pre") {
1009       | file |
1010       doublePuts($stderr,file,"if (window.testRunner) {")
1011       doublePuts($stderr,file,"    testRunner.dumpAsText(window.enablePixelTesting);")
1012       doublePuts($stderr,file,"    testRunner.waitUntilDone();")
1013       doublePuts($stderr,file,"}")
1014       doublePuts($stderr,file,"")
1015       doublePuts($stderr,file,"function debug(msg)")
1016       doublePuts($stderr,file,"{")
1017       doublePuts($stderr,file,"    var span = document.createElement(\"span\");")
1018       doublePuts($stderr,file,"    document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace")
1019       doublePuts($stderr,file,"    span.innerHTML = msg + '<br />';")
1020       doublePuts($stderr,file,"}")
1021       doublePuts($stderr,file,"")
1022       doublePuts($stderr,file,"function quit() {")
1023       doublePuts($stderr,file,"    testRunner.notifyDone();")
1024       doublePuts($stderr,file,"}")
1025       doublePuts($stderr,file,"")
1026       doublePuts($stderr,file,"__bencher_continuation=null;")
1027       doublePuts($stderr,file,"")
1028       doublePuts($stderr,file,"function reportResult(result) {")
1029       doublePuts($stderr,file,"    __bencher_continuation(result);")
1030       doublePuts($stderr,file,"}")
1031       if benchParams.kind == :singleFileTimedCallbackBenchmark
1032         doublePuts($stderr,file,"")
1033         doublePuts($stderr,file,benchParams.callbackDecl)
1034       end
1035       doublePuts($stderr,file,"")
1036       doublePuts($stderr,file,"function __bencher_runImpl(continuation) {")
1037       doublePuts($stderr,file,"    function doit() {")
1038       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"\";")
1039       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";")
1040       doublePuts($stderr,file,"        var testFrame = document.getElementById(\"testframe\");")
1041       doublePuts($stderr,file,"        testFrame.contentDocument.open();")
1042       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");")
1043       if benchParams.kind == :throughputBenchmark or benchParams.kind == :multiFileTimedBenchmark
1044         benchParams.dataPaths.each {
1045           | path |
1046           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{path.inspect.inspect[1..-2]}></script>\");")
1047         }
1048       end
1049       if benchParams.kind == :throughputBenchmark
1050         if benchParams.doWarmup
1051           warmup = $warmup
1052         else
1053           warmup = 0
1054         end
1055         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">\");")
1056         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.setUpCommand.js}\");")
1057         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_before = #{curTime};\");")
1058         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_after = __bencher_before;\");")
1059         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_expected = #{$quantum};\");")
1060         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_runs = 0;\");")
1061         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"while (true) {\");")
1062         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    #{benchParams.command.js}\");")
1063         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_runs++;\");")
1064         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_after = #{curTime};\");")
1065         if benchParams.deterministic
1066           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (true) {\");")
1067         else
1068           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (__bencher_after - __bencher_before >= __bencher_expected) {\");")
1069         end
1070         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        if (__bencher_runs >= #{benchParams.minimumIterations} || window.parent.__bencher_count < #{warmup})\");")
1071         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"            break;\");")
1072         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        __bencher_expected += #{$quantum}\");")
1073         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    }\");")
1074         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"}\");")
1075         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.tearDownCommand.js}\");")
1076         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"window.parent.reportResult((__bencher_after - __bencher_before) / __bencher_runs);\");")
1077         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</script>\");")
1078       else
1079         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">var __bencher_before = #{curTime};</script>\");")
1080         if benchParams.kind == :multiFileTimedBenchmark
1081           doublePuts($stderr,file,"        testFrame.contentDocument.write(#{benchParams.command.html.inspect});")
1082         else
1083           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{benchParams.benchPath.inspect.inspect[1..-2]}></script>\");")
1084         end
1085         unless benchParams.kind == :singleFileTimedCallbackBenchmark
1086           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">window.parent.reportResult(#{curTime} - __bencher_before);</script>\");")
1087         end
1088       end
1089       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</body></html>\");")
1090       doublePuts($stderr,file,"        testFrame.contentDocument.close();")
1091       doublePuts($stderr,file,"    }")
1092       doublePuts($stderr,file,"    __bencher_continuation = continuation;")
1093       doublePuts($stderr,file,"    window.setTimeout(doit, 10);")
1094       doublePuts($stderr,file,"}")
1095     }
1096
1097     Benchfile.create(["bencher-htmldoc",".html"]) {
1098       | file |
1099       doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>")
1100     }
1101   else
1102     raise
1103   end
1104 end
1105
1106 def emitBenchRunCode(name, plan, benchParams)
1107   plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchParams), plan)
1108 end
1109
1110 class FileCreator
1111   def initialize(filename)
1112     @filename = filename
1113     @state = :empty
1114   end
1115   
1116   def puts(text)
1117     $script.print "echo #{Shellwords.shellescape(text)}"
1118     if @state == :empty
1119       $script.print " > "
1120       @state = :nonEmpty
1121     else
1122       $script.print " >> "
1123     end
1124     $script.puts "#{Shellwords.shellescape(@filename)}"
1125   end
1126   
1127   def close
1128     if @state == :empty
1129       $script.puts "rm -f #{Shellwords.shellescape(text)}"
1130       $script.puts "touch #{Shellwords.shellescape(text)}"
1131     end
1132   end
1133   
1134   def self.open(filename)
1135     outp = FileCreator.new(filename)
1136     yield outp
1137     outp.close
1138   end
1139 end
1140
1141 def emitSelfContainedBenchRunCode(name, plan, targetFile, configFile, benchmark)
1142   FileCreator.open(configFile) {
1143     | outp |
1144     outp.puts "__bencher_message = \"#{name}: #{plan.vm}: #{plan.iteration}: \";"
1145     outp.puts "__bencher_warmup = #{$warmup};"
1146     outp.puts "__bencher_inner = #{$inner};"
1147     outp.puts "__bencher_benchmark = #{benchmark.to_json};"
1148     case $timeMode
1149     when :preciseTime
1150       outp.puts "__bencher_curTime = (function(){ return testRunner.preciseTime() * 1000; });"
1151     when :date
1152       outp.puts "__bencher_curTime = (function(){ return Date.now(); });"
1153     else
1154       raise
1155     end
1156   }
1157   
1158   plan.vm.emitRunCode(targetFile, plan)
1159 end
1160
1161 def planForDescription(string, plans, benchFullname, vmName, iteration)
1162   raise "Unexpected benchmark full name: #{benchFullname.inspect}, string: #{string.inspect}" unless benchFullname =~ /\//
1163   suiteName = $~.pre_match
1164   return nil if suiteName == "WARMUP"
1165   benchName = $~.post_match
1166   result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration}
1167   raise "Unexpected result dimensions: #{result.inspect}, string: #{string.inspect}" unless result.size == 1
1168   result[0]
1169 end
1170
1171 class ParsedResult
1172   attr_reader :plan, :innerIndex, :time, :result
1173   
1174   def initialize(plan, innerIndex, time)
1175     @plan = plan
1176     @innerIndex = innerIndex
1177     if time == :crashed
1178       @result = :error
1179     else
1180       @time = time
1181       @result = :success
1182     end
1183     
1184     raise unless @plan.is_a? BenchPlan
1185     raise unless @innerIndex.is_a? Integer
1186     raise unless @time.is_a? Numeric or @result == :error
1187   end
1188   
1189   def benchmark
1190     plan.benchmark
1191   end
1192   
1193   def suite
1194     plan.suite
1195   end
1196   
1197   def vm
1198     plan.vm
1199   end
1200   
1201   def outerIndex
1202     plan.iteration
1203   end
1204   
1205   def self.create(plan, innerIndex, time)
1206     if plan
1207       ParsedResult.new(plan, innerIndex, time)
1208     else
1209       nil
1210     end
1211   end
1212   
1213   def self.parse(plans, string)
1214     if string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): Time: /
1215       benchFullname = $1
1216       vmName = $2
1217       outerIndex = $3.to_i
1218       innerIndex = $4.to_i
1219       time = $~.post_match.to_f
1220       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, time)
1221     elsif string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): CRASHED/
1222       benchFullname = $1
1223       vmName = $2
1224       outerIndex = $3.to_i
1225       innerIndex = $4.to_i
1226       time = $~.post_match.to_f
1227       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, :crashed)
1228     else
1229       nil
1230     end
1231   end
1232 end
1233
1234 class VM
1235   @@extraEnvSet = {}
1236   
1237   def initialize(origPath, name, nameKind, svnRevision)
1238     @origPath = origPath.to_s
1239     @path = origPath.to_s
1240     @name = name
1241     @nameKind = nameKind
1242     @extraEnv = {}
1243     
1244     if $forceVMKind
1245       @vmType = $forceVMKind
1246     else
1247       if @origPath =~ /DumpRenderTree$/
1248         @vmType = :dumpRenderTree
1249       elsif @origPath =~ /WebKitTestRunner$/
1250         @vmType = :webkitTestRunner
1251       else
1252         @vmType = :jsc
1253       end
1254     end
1255     
1256     @svnRevision = svnRevision
1257     
1258     # Try to detect information about the VM.
1259     if path =~ /\/WebKitBuild\/(Release|Debug)+\/([a-zA-Z]+)$/
1260       @checkoutPath = $~.pre_match
1261       # FIXME: Use some variant of this: 
1262       # <bdash>   def retrieve_revision
1263       # <bdash>     `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i
1264       # <bdash>   end
1265       unless @svnRevision
1266         begin
1267           Dir.chdir(@checkoutPath) {
1268             $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
1269             IO.popen("svn info", "r") {
1270               | inp |
1271               inp.each_line {
1272                 | line |
1273                 if line =~ /Revision: ([0-9]+)/
1274                   @svnRevision = $1
1275                 end
1276               }
1277             }
1278           }
1279           unless @svnRevision
1280             $stderr.puts "Warning: running svn info for #{name} silently failed."
1281           end
1282         rescue => e
1283           # Failed to detect svn revision.
1284           $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
1285         end
1286       end
1287     else
1288       $stderr.puts "Warning: could not identify checkout location for #{name}"
1289     end
1290     
1291     if @path =~ /\/Release\/([a-zA-Z]+)$/
1292       @libPath, @relativeBinPath = [$~.pre_match+"/Release"], "./#{$1}"
1293     elsif @path =~ /\/Debug\/([a-zA-Z]+)$/
1294       @libPath, @relativeBinPath = [$~.pre_match+"/Debug"], "./#{$1}"
1295     elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/
1296       @libPath = [$~.pre_match + "/Contents/Resources", $~.pre_match + "/Contents/Frameworks"]
1297     elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/
1298       @libPath, @relativeBinPath = [$~.pre_match], $&[1..-1]
1299     elsif @path =~ /(DumpRenderTree|webkitTestRunner|jsc)$/
1300       @libPath, @relativeBinPath = [$~.pre_match+"/"], "./#{$1}"
1301     end
1302   end
1303   
1304   def canCopyIntoBenchPath
1305     if @libPath and @relativeBinPath
1306       true
1307     else
1308       false
1309     end
1310   end
1311   
1312   def addExtraEnv(key, val)
1313     @extraEnv[key] = val
1314     @@extraEnvSet[key] = true
1315   end
1316   
1317   def copyIntoBenchPath
1318     raise unless canCopyIntoBenchPath
1319     basename, filename = Benchfile.uniqueFilename("vm")
1320     raise unless Dir.mkdir(filename)
1321     @libPath.each {
1322       | libPathPart |
1323       cmd = "cp -a #{Shellwords.shellescape(libPathPart)}/* #{Shellwords.shellescape(filename.to_s)}"
1324       $stderr.puts ">> #{cmd}" if $verbosity>=2
1325       raise unless system(cmd)
1326     }
1327     @path = "#{basename}/#{@relativeBinPath}"
1328     @libPath = [basename]
1329   end
1330   
1331   def to_s
1332     @name
1333   end
1334   
1335   def name
1336     @name
1337   end
1338   
1339   def shouldMeasureGC
1340     $measureGC == true or ($measureGC == name)
1341   end
1342   
1343   def origPath
1344     @origPath
1345   end
1346   
1347   def path
1348     @path
1349   end
1350   
1351   def nameKind
1352     @nameKind
1353   end
1354   
1355   def vmType
1356     @vmType
1357   end
1358   
1359   def checkoutPath
1360     @checkoutPath
1361   end
1362   
1363   def svnRevision
1364     @svnRevision
1365   end
1366   
1367   def extraEnv
1368     @extraEnv
1369   end
1370   
1371   def printFunction
1372     case @vmType
1373     when :jsc
1374       "print"
1375     when :dumpRenderTree, :webkitTestRunner
1376       "debug"
1377     else
1378       raise @vmType
1379     end
1380   end
1381   
1382   def emitRunCode(fileToRun, plan)
1383     myLibPath = @libPath
1384     myLibPath = [] unless myLibPath
1385     @@extraEnvSet.keys.each {
1386       | key |
1387       $script.puts "unset #{Shellwords.shellescape(key)}"
1388     }
1389     $script.puts "export DYLD_LIBRARY_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1390     $script.puts "export DYLD_FRAMEWORK_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1391     @extraEnv.each_pair {
1392       | key, val |
1393       $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1394     }
1395     plan.environment.each_pair {
1396         | key, val |
1397         $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1398     }
1399     $script.puts "#{path} #{fileToRun} 2>&1 || {"
1400     $script.puts "    echo " + Shellwords.shellescape("#{name} failed to run!") + " 1>&2"
1401     $inner.times {
1402       | iteration |
1403       $script.puts "    echo " + Shellwords.shellescape("#{plan.prefix}: #{iteration}: CRASHED")
1404     }
1405     $script.puts "}"
1406     plan.environment.keys.each {
1407         | key |
1408         $script.puts "export #{Shellwords.shellescape(key)}="
1409     }
1410   end
1411 end
1412
1413 class StatsAccumulator
1414   def initialize
1415     @stats = []
1416     ($outer*$inner).times {
1417       @stats << Stats.new
1418     }
1419   end
1420   
1421   def statsForIteration(outerIteration, innerIteration)
1422     @stats[outerIteration*$inner + innerIteration]
1423   end
1424   
1425   def stats
1426     result = Stats.new
1427     @stats.each {
1428       | stat |
1429       if stat.ok?
1430         result.add(yield stat)
1431       else
1432         result.add(nil)
1433       end
1434     }
1435     result
1436   end
1437   
1438   def geometricMeanStats
1439     stats {
1440       | stat |
1441       stat.geometricMean
1442     }
1443   end
1444   
1445   def arithmeticMeanStats
1446     stats {
1447       | stat |
1448       stat.arithmeticMean
1449     }
1450   end
1451 end
1452
1453 module Benchmark
1454   attr_accessor :benchmarkSuite
1455   attr_reader :name
1456   
1457   def fullname
1458     benchmarkSuite.name + "/" + name
1459   end
1460   
1461   def to_s
1462     fullname
1463   end
1464   
1465   def weight
1466     1
1467   end
1468   
1469   def weightString
1470     raise unless weight.is_a? Fixnum
1471     raise unless weight >= 1
1472     if weight == 1
1473       ""
1474     else
1475       "x#{weight} "
1476     end
1477   end
1478 end
1479
1480 class SunSpiderBenchmark
1481   include Benchmark
1482   
1483   def initialize(name)
1484     @name = name
1485   end
1486   
1487   def emitRunCode(plan)
1488     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js")))
1489   end
1490 end
1491
1492 class LongSpiderBenchmark
1493   include Benchmark
1494   
1495   def initialize(name)
1496     @name = name
1497   end
1498   
1499   def emitRunCode(plan)
1500     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("LongSpider-#{@name}", "#{LONGSPIDER_PATH}/#{@name}.js")))
1501   end
1502 end
1503
1504 class V8Benchmark
1505   include Benchmark
1506   
1507   def initialize(name)
1508     @name = name
1509   end
1510   
1511   def emitRunCode(plan)
1512     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js")))
1513   end
1514 end
1515
1516 class V8RealBenchmark
1517   include Benchmark
1518   
1519   attr_reader :v8SuiteName
1520   
1521   def initialize(v8SuiteName, name, weight, minimumIterations)
1522     @v8SuiteName = v8SuiteName
1523     @name = name
1524     @weight = weight
1525     @minimumIterations = minimumIterations
1526   end
1527   
1528   def weight
1529     @weight
1530   end
1531   
1532   def emitRunCode(plan)
1533     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(["base", @v8SuiteName, "jsc-#{@name}"].collect{|v| ensureFile("V8Real-#{v}", "#{V8_REAL_PATH}/#{v}.js")}, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, false, @minimumIterations))
1534   end
1535 end
1536
1537 class OctaneBenchmark
1538   include Benchmark
1539   
1540   def initialize(files, name, weight, doWarmup, deterministic, minimumIterations)
1541     @files = files
1542     @name = name
1543     @weight = weight
1544     @doWarmup = doWarmup
1545     @deterministic = deterministic
1546     @minimumIterations = minimumIterations
1547   end
1548   
1549   def weight
1550     @weight
1551   end
1552   
1553   def emitRunCode(plan)
1554     files = []
1555     files += (["base"] + @files).collect {
1556       | v |
1557       ensureFile("Octane-#{v}", "#{OCTANE_PATH}/#{v}.js")
1558     }
1559     files += ["jsc-#{@name}"].collect {
1560       | v |
1561       ensureFile("Octane-#{v}", "#{OCTANE_WRAPPER_PATH}/#{v}.js")
1562     }
1563     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(files, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1564   end
1565 end
1566
1567 class KrakenBenchmark
1568   include Benchmark
1569   
1570   def initialize(name)
1571     @name = name
1572   end
1573   
1574   def emitRunCode(plan)
1575     emitBenchRunCode(fullname, plan, MultiFileTimedBenchmarkParameters.new([ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js")], loadCommandForFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js")))
1576   end
1577 end
1578
1579 class JSBenchBenchmark
1580   include Benchmark
1581   
1582   attr_reader :jsBenchMode
1583   
1584   def initialize(name, jsBenchMode)
1585     @name = name
1586     @jsBenchMode = jsBenchMode
1587   end
1588   
1589   def emitRunCode(plan)
1590     callbackDecl  = "function JSBNG_handleResult(result) {\n"
1591     callbackDecl += "    if (result.error) {\n"
1592     callbackDecl += "        console.log(\"Did not run benchmark correctly!\");\n"
1593     callbackDecl += "        quit();\n"
1594     callbackDecl += "    }\n"
1595     callbackDecl += "    reportResult(result.time);\n"
1596     callbackDecl += "}\n"
1597     emitBenchRunCode(fullname, plan, SingleFileTimedCallbackBenchmarkParameters.new(callbackDecl, ensureFile("JSBench-#{@name}", "#{JSBENCH_PATH}/#{@name}/#{@jsBenchMode}.js")))
1598   end
1599 end
1600
1601 class JSRegressBenchmark
1602   include Benchmark
1603   
1604   def initialize(name)
1605     @name = name
1606   end
1607   
1608   def emitRunCode(plan)
1609     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("JSRegress-#{@name}", "#{JSREGRESS_PATH}/#{@name}.js")))
1610   end
1611 end
1612
1613 class AsmBenchBenchmark
1614   include Benchmark
1615   
1616   def initialize(name)
1617     @name = name
1618   end
1619   
1620   def emitRunCode(plan)
1621     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("AsmBench-#{@name}", "#{ASMBENCH_PATH}/#{@name}.js")))
1622   end
1623 end
1624
1625
1626 class CompressionBenchBenchmark
1627   include Benchmark
1628   
1629   def initialize(files, name, model)
1630     @files = files
1631     @name = name;
1632     @name = name + "-" + model if !model.empty?
1633     @name = @name.gsub(" ", "-").downcase
1634     @scriptName = name
1635     @weight = 1
1636     @doWarmup = true
1637     @deterministic = true
1638     @minimumIterations = 1
1639     @model = model
1640   end
1641   
1642   def weight
1643     @weight
1644   end
1645   
1646   def emitRunCode(plan)
1647     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new((["base"] + @files + ["jsc-#{@scriptName}"]).collect{|v| ensureFile("Compression-#{v}", "#{COMPRESSIONBENCH_PATH}/#{v}.js")}, simpleCommand("jscSetUp('#{@model}');"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1648   end
1649 end
1650
1651 class DSPJSFiltrrBenchmark
1652   include Benchmark
1653   
1654   def initialize(name, filterKey)
1655     @name = name
1656     @filterKey = filterKey
1657   end
1658   
1659   def emitRunCode(plan)
1660     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr.js")
1661     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr_back.jpg")
1662     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-jquery.min.js")
1663     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-bencher.html")
1664     emitSelfContainedBenchRunCode(fullname, plan, "filtrr-bencher.html", "bencher-config.js", @filterKey)
1665   end
1666 end
1667
1668 class DSPJSVP8Benchmark
1669   include Benchmark
1670   
1671   def initialize
1672     @name = "route9-vp8"
1673   end
1674   
1675   def weight
1676     5
1677   end
1678   
1679   def emitRunCode(plan)
1680     ensureBenchmarkFiles(DSPJS_ROUTE9_PATH)
1681     emitSelfContainedBenchRunCode(fullname, plan, "route9-bencher.html", "bencher-config.js", "")
1682   end
1683 end
1684
1685 class DSPStarfieldBenchmark
1686   include Benchmark
1687
1688   def initialize
1689     @name = "starfield"
1690   end
1691   
1692   def weight
1693     5
1694   end
1695
1696   def emitRunCode(plan)
1697     ensureBenchmarkFiles(DSPJS_STARFIELD_PATH)
1698     emitSelfContainedBenchRunCode(fullname, plan, "starfield-bencher.html", "bencher-config.js", "")
1699   end
1700 end
1701
1702 class DSPJSJSLinuxBenchmark
1703   include Benchmark
1704   def initialize
1705     @name = "bellard-jslinux"
1706   end
1707
1708   def weight
1709     5
1710   end
1711
1712   def emitRunCode(plan)
1713     ensureBenchmarkFiles(DSPJS_JSLINUX_PATH)
1714     emitSelfContainedBenchRunCode(fullname, plan, "jslinux-bencher.html", "bencher-config.js", "")
1715   end
1716 end
1717
1718 class DSPJSQuake3Benchmark
1719   include Benchmark
1720
1721   def initialize
1722     @name = "zynaps-quake3"
1723   end
1724
1725   def weight
1726     5
1727   end
1728
1729   def emitRunCode(plan)
1730     ensureBenchmarkFiles(DSPJS_QUAKE3_PATH)
1731     emitSelfContainedBenchRunCode(fullname, plan, "quake-bencher.html", "bencher-config.js", "")
1732   end
1733 end
1734
1735 class DSPJSMandelbrotBenchmark
1736   include Benchmark
1737
1738   def initialize
1739     @name = "zynaps-mandelbrot"
1740   end
1741
1742   def weight
1743     5
1744   end
1745
1746   def emitRunCode(plan)
1747     ensureBenchmarkFiles(DSPJS_MANDELBROT_PATH)
1748     emitSelfContainedBenchRunCode(fullname, plan, "mandelbrot-bencher.html", "bencher-config.js", "")
1749   end
1750 end
1751
1752 class DSPJSAmmoJSASMBenchmark
1753   include Benchmark
1754
1755   def initialize
1756     @name = "ammojs-asm-js"
1757   end
1758
1759   def weight
1760     5
1761   end
1762
1763   def emitRunCode(plan)
1764     ensureBenchmarkFiles(DSPJS_AMMOJS_ASMJS_PATH)
1765     emitSelfContainedBenchRunCode(fullname, plan, "ammo-asmjs-bencher.html", "bencher-config.js", "")
1766   end
1767 end
1768
1769 class DSPJSAmmoJSRegularBenchmark
1770   include Benchmark
1771
1772   def initialize
1773     @name = "ammojs-regular-js"
1774   end
1775
1776   def weight
1777     5
1778   end
1779
1780   def emitRunCode(plan)
1781     ensureBenchmarkFiles(DSPJS_AMMOJS_REGULAR_PATH)
1782     emitSelfContainedBenchRunCode(fullname, plan, "ammo-regular-bencher.html", "bencher-config.js", "")
1783   end
1784 end
1785
1786 class BrowsermarkJSBenchmark
1787   include Benchmark
1788     
1789   def initialize(name)
1790     @name = name
1791   end
1792   
1793   def emitRunCode(plan)
1794     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new([ensureFile(name, "#{BROWSERMARK_JS_PATH}/#{name}/test.js"), ensureFile("browsermark-bencher", "#{BROWSERMARK_JS_PATH}/browsermark-bencher.js")], simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, 32))
1795   end
1796 end
1797
1798 class BrowsermarkDOMBenchmark
1799   include Benchmark
1800     
1801   def initialize(name)
1802     @name = name
1803   end
1804   
1805   def emitRunCode(plan)
1806     ensureBenchmarkFiles(BROWSERMARK_PATH)
1807     emitSelfContainedBenchRunCode(fullname, plan, "tests/benchmarks/dom/#{name}/index.html", "bencher-config.js", name)
1808   end
1809 end
1810
1811 class BenchmarkSuite
1812   def initialize(name, preferredMean, decimalShift)
1813     @name = name
1814     @preferredMean = preferredMean
1815     @benchmarks = []
1816     @subSuites = []
1817     @decimalShift = decimalShift
1818   end
1819   
1820   def name
1821     @name
1822   end
1823   
1824   def to_s
1825     @name
1826   end
1827   
1828   def decimalShift
1829     @decimalShift
1830   end
1831   
1832   def addIgnoringPattern(benchmark)
1833     benchmark.benchmarkSuite = self
1834     @benchmarks << benchmark
1835   end
1836   
1837   def add(benchmark)
1838     if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern
1839         addIgnoringPattern(benchmark)
1840     end
1841   end
1842   
1843   def addSubSuite(subSuite)
1844     @subSuites << subSuite
1845   end
1846   
1847   def benchmarks
1848     @benchmarks
1849   end
1850   
1851   def benchmarkForName(name)
1852     result = @benchmarks.select{|v| v.name == name}
1853     raise unless result.length == 1
1854     result[0]
1855   end
1856   
1857   def hasBenchmark(benchmark)
1858     array = @benchmarks.select{|v| v == benchmark}
1859     raise unless array.length == 1 or array.length == 0
1860     array.length == 1
1861   end
1862   
1863   def subSuites
1864     @subSuites
1865   end
1866   
1867   def suites
1868     [self] + @subSuites
1869   end
1870   
1871   def suitesWithBenchmark(benchmark)
1872     result = [self]
1873     @subSuites.each {
1874       | subSuite |
1875       if subSuite.hasBenchmark(benchmark)
1876         result << subSuite
1877       end
1878     }
1879     result
1880   end
1881   
1882   def empty?
1883     @benchmarks.empty?
1884   end
1885   
1886   def retain_if
1887     @benchmarks.delete_if {
1888       | benchmark |
1889       not yield benchmark
1890     }
1891   end
1892   
1893   def preferredMean
1894     @preferredMean
1895   end
1896   
1897   def computeMean(stat)
1898     if stat.ok?
1899       (stat.send @preferredMean) * (10 ** decimalShift)
1900     else
1901       nil
1902     end
1903   end
1904 end
1905
1906 class BenchRunPlan
1907   def initialize(benchmark, vm, iteration)
1908     @benchmark = benchmark
1909     @vm = vm
1910     @iteration = iteration
1911     @environment = {}
1912     if $environment.has_key?(vm.name)
1913       if $environment[vm.name].has_key?(benchmark.benchmarkSuite.name)
1914         if $environment[vm.name][benchmark.benchmarkSuite.name].has_key?(benchmark.name)
1915           @environment = $environment[vm.name][benchmark.benchmarkSuite.name][benchmark.name]
1916         end
1917       end
1918     end
1919   end
1920   
1921   def benchmark
1922     @benchmark
1923   end
1924   
1925   def suite
1926     @benchmark.benchmarkSuite
1927   end
1928   
1929   def vm
1930     @vm
1931   end
1932   
1933   def iteration
1934     @iteration
1935   end
1936  
1937   def environment
1938     @environment
1939   end
1940  
1941   def prefix
1942     "#{@benchmark.fullname}: #{vm.name}: #{iteration}"
1943   end
1944   
1945   def emitRunCode
1946     @benchmark.emitRunCode(self)
1947   end
1948   
1949   def to_s
1950     benchmark.to_s + "/" + vm.to_s
1951   end
1952 end
1953
1954 class BenchmarkOnVM
1955   def initialize(benchmark, suiteOnVM, subSuitesOnVM)
1956     @benchmark = benchmark
1957     @suiteOnVM = suiteOnVM
1958     @subSuitesOnVM = subSuitesOnVM
1959     @stats = Stats.new
1960   end
1961   
1962   def to_s
1963     "#{@benchmark} on #{@suiteOnVM.vm}"
1964   end
1965   
1966   def benchmark
1967     @benchmark
1968   end
1969   
1970   def vm
1971     @suiteOnVM.vm
1972   end
1973   
1974   def vmStats
1975     @suiteOnVM.vmStats
1976   end
1977   
1978   def suite
1979     @benchmark.benchmarkSuite
1980   end
1981   
1982   def suiteOnVM
1983     @suiteOnVM
1984   end
1985   
1986   def subSuitesOnVM
1987     @subSuitesOnVM
1988   end
1989   
1990   def stats
1991     @stats
1992   end
1993   
1994   def parseResult(result)
1995     raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm
1996     raise unless result.benchmark == @benchmark
1997     @stats.add(result.time)
1998   end
1999 end
2000
2001 class NamedStatsAccumulator < StatsAccumulator
2002   def initialize(name)
2003     super()
2004     @name = name
2005   end
2006   
2007   def reportingName
2008     @name
2009   end
2010 end
2011
2012 class SuiteOnVM < StatsAccumulator
2013   def initialize(vm, vmStats, suite)
2014     super()
2015     @vm = vm
2016     @vmStats = vmStats
2017     @suite = suite
2018     
2019     raise unless @vm.is_a? VM
2020     raise unless @vmStats.is_a? StatsAccumulator
2021     raise unless @suite.is_a? BenchmarkSuite
2022   end
2023   
2024   def to_s
2025     "#{@suite} on #{@vm}"
2026   end
2027   
2028   def suite
2029     @suite
2030   end
2031   
2032   def vm
2033     @vm
2034   end
2035
2036   def reportingName
2037     @vm.name
2038   end
2039   
2040   def vmStats
2041     raise unless @vmStats
2042     @vmStats
2043   end
2044 end
2045
2046 class SubSuiteOnVM < StatsAccumulator
2047   def initialize(vm, suite)
2048     super()
2049     @vm = vm
2050     @suite = suite
2051     raise unless @vm.is_a? VM
2052     raise unless @suite.is_a? BenchmarkSuite
2053   end
2054   
2055   def to_s
2056     "#{@suite} on #{@vm}"
2057   end
2058   
2059   def suite
2060     @suite
2061   end
2062   
2063   def vm
2064     @vm
2065   end
2066   
2067   def reportingName
2068     @vm.name
2069   end
2070 end
2071
2072 class BenchPlan
2073   def initialize(benchmarkOnVM, iteration)
2074     @benchmarkOnVM = benchmarkOnVM
2075     @iteration = iteration
2076   end
2077   
2078   def to_s
2079     "#{@benchmarkOnVM} \##{@iteration+1}"
2080   end
2081   
2082   def benchmarkOnVM
2083     @benchmarkOnVM
2084   end
2085   
2086   def benchmark
2087     @benchmarkOnVM.benchmark
2088   end
2089   
2090   def suite
2091     @benchmarkOnVM.suite
2092   end
2093   
2094   def vm
2095     @benchmarkOnVM.vm
2096   end
2097   
2098   def iteration
2099     @iteration
2100   end
2101   
2102   def parseResult(result)
2103     raise unless result.plan == self
2104     @benchmarkOnVM.parseResult(result)
2105     benchmark.weight.times {
2106       @benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time)
2107       @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time)
2108       @benchmarkOnVM.subSuitesOnVM.each {
2109         | subSuite |
2110         subSuite.statsForIteration(@iteration, result.innerIndex).add(result.time)
2111       }
2112     }
2113   end
2114 end
2115
2116 def lpad(str,chars)
2117   if str.length>chars
2118     str
2119   else
2120     "%#{chars}s"%(str)
2121   end
2122 end
2123
2124 def rpad(str,chars)
2125   while str.length < chars
2126     str+=" "
2127   end
2128   str
2129 end
2130
2131 def center(str,chars)
2132   while str.length<chars
2133     str+=" "
2134     if str.length<chars
2135       str=" "+str
2136     end
2137   end
2138   str
2139 end
2140
2141 def statsToStr(stats, decimalShift)
2142   if stats.error?
2143     lpad(center("ERROR", 10+10+2), 12+10+2)
2144   elsif $inner*$outer == 1
2145     string = numToStr(stats.mean, decimalShift)
2146     raise unless string =~ /\./
2147     left = $~.pre_match
2148     right = $~.post_match
2149     lpad(left, 13 - decimalShift) + "." + rpad(right, 10 + decimalShift)
2150   else
2151     lpad(numToStr(stats.mean, decimalShift), 12) + "+-" + rpad(numToStr(stats.confInt, decimalShift), 10)
2152   end
2153 end
2154
2155 def plural(num)
2156   if num == 1
2157     ""
2158   else
2159     "s"
2160   end
2161 end
2162
2163 def wrap(str, columns)
2164   array = str.split
2165   result = ""
2166   curLine = array.shift
2167   array.each {
2168     | curStr |
2169     if (curLine + " " + curStr).size > columns
2170       result += curLine + "\n"
2171       curLine = curStr
2172     else
2173       curLine += " " + curStr
2174     end
2175   }
2176   result + curLine + "\n"
2177 end
2178   
2179 def runAndGetResults
2180   results = nil
2181   Dir.chdir(BENCH_DATA_PATH) {
2182     $stderr.puts ">> sh ./runscript" if $verbosity >= 2
2183     raise "Script did not complete correctly: #{$?}" unless system("sh ./runscript > runlog")
2184     results = IO::read("runlog")
2185   }
2186   raise unless results
2187   results
2188 end
2189
2190 def parseAndDisplayResults(results)
2191   vmStatses = []
2192   $vms.each {
2193     | vm |
2194     vmStatses << NamedStatsAccumulator.new(vm.name)
2195   }
2196   
2197   suitesOnVMs = []
2198   suitesOnVMsForSuite = {}
2199   subSuitesOnVMsForSubSuite = {}
2200   $suites.each {
2201     | suite |
2202     suitesOnVMsForSuite[suite] = []
2203     suite.subSuites.each {
2204       | subSuite |
2205       subSuitesOnVMsForSubSuite[subSuite] = []
2206     }
2207   }
2208   suitesOnVMsForVM = {}
2209   $vms.each {
2210     | vm |
2211     suitesOnVMsForVM[vm] = []
2212   }
2213   
2214   benchmarksOnVMs = []
2215   benchmarksOnVMsForBenchmark = {}
2216   $benchmarks.each {
2217     | benchmark |
2218     benchmarksOnVMsForBenchmark[benchmark] = []
2219   }
2220   
2221   $vms.each_with_index {
2222     | vm, vmIndex |
2223     vmStats = vmStatses[vmIndex]
2224     $suites.each {
2225       | suite |
2226       suiteOnVM = SuiteOnVM.new(vm, vmStats, suite)
2227       subSuitesOnVM = suite.subSuites.map {
2228         | subSuite |
2229         result = SubSuiteOnVM.new(vm, subSuite)
2230         subSuitesOnVMsForSubSuite[subSuite] << result
2231         result
2232       }
2233       suitesOnVMs << suiteOnVM
2234       suitesOnVMsForSuite[suite] << suiteOnVM
2235       suitesOnVMsForVM[vm] << suiteOnVM
2236       suite.benchmarks.each {
2237         | benchmark |
2238         subSuitesOnVMForThisBenchmark = []
2239         subSuitesOnVM.each {
2240           | subSuiteOnVM |
2241           if subSuiteOnVM.suite.hasBenchmark(benchmark)
2242             subSuitesOnVMForThisBenchmark << subSuiteOnVM
2243           end
2244         }
2245         benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM, subSuitesOnVMForThisBenchmark)
2246         benchmarksOnVMs << benchmarkOnVM
2247         benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
2248       }
2249     }
2250   }
2251   
2252   plans = []
2253   benchmarksOnVMs.each {
2254     | benchmarkOnVM |
2255     $outer.times {
2256       | iteration |
2257       plans << BenchPlan.new(benchmarkOnVM, iteration)
2258     }
2259   }
2260
2261   hostname = nil
2262   hwmodel = nil
2263   results.each_line {
2264     | line |
2265     line.chomp!
2266     if line =~ /HOSTNAME:([^.]+)/
2267       hostname = $1
2268     elsif line =~ /HARDWARE:hw\.model: /
2269       hwmodel = $~.post_match.chomp
2270     else
2271       result = ParsedResult.parse(plans, line)
2272       if result
2273         result.plan.parseResult(result)
2274       end
2275     end
2276   }
2277   
2278   # Compute the geomean of the preferred means of results on a SuiteOnVM
2279   overallResults = []
2280   $vms.each {
2281     | vm |
2282     result = Stats.new
2283     $outer.times {
2284       | outerIndex |
2285       $inner.times {
2286         | innerIndex |
2287         curResult = Stats.new
2288         suitesOnVMsForVM[vm].each {
2289           | suiteOnVM |
2290           # For a given iteration, suite, and VM, compute the suite's preferred mean
2291           # over the data collected for all benchmarks in that suite. We'll have one
2292           # sample per benchmark. For example on V8 this will be the geomean of 1
2293           # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
2294           # splay.
2295           curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
2296         }
2297         
2298         # curResult now holds 1 sample for each of the means computed in the above
2299         # loop. Compute the geomean over this, and store it.
2300         if curResult.ok?
2301           result.add(curResult.geometricMean)
2302         else
2303           result.add(nil)
2304         end
2305       }
2306     }
2307
2308     # $overallResults will have a Stats for each VM. That Stats object will hold
2309     # $inner*$outer geomeans, allowing us to compute the arithmetic mean and
2310     # confidence interval of the geomeans of preferred means. Convoluted, but
2311     # useful and probably sound.
2312     overallResults << result
2313   }
2314   
2315   if $verbosity >= 2
2316     benchmarksOnVMs.each {
2317       | benchmarkOnVM |
2318       $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
2319     }
2320     
2321     $vms.each_with_index {
2322       | vm, vmIndex |
2323       vmStats = vmStatses[vmIndex]
2324       $stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}"
2325       $stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}"
2326     }
2327   end
2328
2329   if $outputName
2330     reportName = $outputName
2331   else
2332     reportName =
2333       (if ($vms.collect {
2334              | vm |
2335              vm.nameKind
2336            }.index :auto)
2337          ""
2338        else
2339          text = $vms.collect {
2340            | vm |
2341            vm.to_s
2342          }.join("_") + "_"
2343          if text.size >= 40
2344            ""
2345          else
2346            text
2347          end
2348        end) +
2349       ($suites.collect {
2350          | suite |
2351          suite.to_s
2352        }.join("")) + "_" +
2353       (if hostname
2354          hostname + "_"
2355        else
2356          ""
2357        end)+
2358       (begin
2359          time = Time.now
2360          "%04d%02d%02d_%02d%02d" %
2361            [ time.year, time.month, time.day,
2362              time.hour, time.min ]
2363        end)
2364   end
2365
2366   unless $brief
2367     puts "Generating benchmark report at #{Dir.pwd}/#{reportName}_report.txt"
2368     puts "And raw data at #{Dir.pwd}/#{reportName}.json"
2369   end
2370   
2371   outp = $stdout
2372   json = {}
2373   begin
2374     outp = File.open(reportName + "_report.txt","w")
2375   rescue => e
2376     $stderr.puts "Error: could not save report to #{reportName}_report.txt: #{e}"
2377     $stderr.puts
2378   end
2379   
2380   def createVMsString
2381     result = ""
2382     result += "   " if $allSuites.size > 1
2383     result += rpad("", $benchpad + $weightpad)
2384     result += " "
2385     $vms.size.times {
2386       | index |
2387       if index != 0
2388         result += " "+NoChange.new(0).shortForm
2389       end
2390       result += lpad(center($vms[index].name, 10+10+2), 12+10+2)
2391     }
2392     result += "    "
2393     if $vms.size >= 3
2394       result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
2395     elsif $vms.size >= 2
2396       result += " "*26
2397     end
2398     result
2399   end
2400   
2401   def andJoin(list)
2402     if list.size == 1
2403       list[0].to_s
2404     elsif list.size == 2
2405       "#{list[0]} and #{list[1]}"
2406     else
2407       "#{list[0..-2].join(', ')}, and #{list[-1]}"
2408     end
2409   end
2410   
2411   json["vms"] = $vms.collect{|v| v.name}
2412   json["suites"] = {}
2413   json["runlog"] = results
2414   
2415   columns = [createVMsString.size, 78].max
2416   
2417   outp.print "Benchmark report for "
2418   outp.print andJoin($suites)
2419   if hostname
2420     outp.print " on #{hostname}"
2421   end
2422   if hwmodel
2423     outp.print " (#{hwmodel})"
2424   end
2425   outp.puts "."
2426   outp.puts
2427   
2428   outp.puts "VMs tested:"
2429   $vms.each {
2430     | vm |
2431     outp.print "\"#{vm.name}\" at #{vm.origPath}"
2432     if vm.svnRevision
2433       outp.print " (r#{vm.svnRevision})"
2434     end
2435     outp.puts
2436     vm.extraEnv.each_pair {
2437       | key, val |
2438       outp.puts "    export #{key}=#{val}"
2439     }
2440   }
2441   
2442   outp.puts
2443   
2444   outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
2445                  "with #{$outer} VM invocation#{plural($outer)} per benchmark."+
2446                  (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
2447                                       "total time of those iterations, for each sample.")
2448                   else "" end)+
2449                  (if $measureGC == true then (" No manual garbage collection invocations were "+
2450                                               "emitted.")
2451                   elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
2452                                          "all VMs except #{$measureGC}.")
2453                   else (" Emitted a call to gc() between sample measurements.") end)+
2454                  (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
2455                                         "began with the very first iteration.")
2456                   else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
2457                         "invocation for warm-up.") end)+
2458                  (case $timeMode
2459                   when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
2460                                           "microsecond-level timing.")
2461                   when :date then (" Used the portable Date.now() method to get millisecond-"+
2462                                    "level timing.")
2463                   else raise end)+
2464                  " Reporting benchmark execution times with 95% confidence "+
2465                  "intervals in milliseconds.",
2466                  columns)
2467   
2468   outp.puts
2469   
2470   def printVMs(outp)
2471     outp.puts createVMsString
2472   end
2473   
2474   def summaryStats(outp, json, accumulators, name, decimalShift, &proc)
2475     resultingJson = {}
2476     outp.print "   " if $allSuites.size > 1
2477     outp.print rpad(name, $benchpad + $weightpad)
2478     outp.print " "
2479     accumulators.size.times {
2480       | index |
2481       if index != 0
2482         outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
2483       end
2484       outp.print statsToStr(accumulators[index].stats(&proc), decimalShift)
2485       resultingJson[accumulators[index].reportingName] = accumulators[index].stats(&proc).jsonMap
2486     }
2487     if accumulators.size>=2
2488       outp.print("    "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm)
2489     end
2490     outp.puts
2491     json[name] = resultingJson
2492   end
2493
2494   def allSummaryStats(outp, json, accumulators, preferredMean, decimalShift)
2495     meanLabel = '<' + preferredMean.to_s.sub(/Mean$/, '') + '>'
2496     summaryStats(outp, json, accumulators, meanLabel, decimalShift) {
2497       | stat |
2498       stat.send(preferredMean)
2499     }
2500   end
2501   
2502   $suites.each {
2503     | suite |
2504     suiteJson = {}
2505     subSuiteJsons = {}
2506     suite.subSuites.each {
2507       | subSuite |
2508       subSuiteJsons[subSuite] = {}
2509     }
2510     
2511     printVMs(outp)
2512     if $allSuites.size > 1
2513       outp.puts(andJoin(suite.suites.map{|v| v.name}) + ":")
2514     else
2515       outp.puts
2516     end
2517     suite.benchmarks.each {
2518       | benchmark |
2519       benchmarkJson = {}
2520       outp.print "   " if $allSuites.size > 1
2521       outp.print rpad(benchmark.name, $benchpad) + rpad(benchmark.weightString, $weightpad)
2522       if benchmark.name.size > $benchNameClip
2523         outp.puts
2524         outp.print "   " if $allSuites.size > 1
2525         outp.print((" " * $benchpad) + (" " * $weightpad))
2526       end
2527       outp.print " "
2528       myConfigs = benchmarksOnVMsForBenchmark[benchmark]
2529       myConfigs.size.times {
2530         | index |
2531         if index != 0
2532           outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
2533         end
2534         outp.print statsToStr(myConfigs[index].stats, suite.decimalShift)
2535         benchmarkJson[myConfigs[index].vm.name] = myConfigs[index].stats.jsonMap
2536       }
2537       if $vms.size>=2
2538         outp.print("    "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
2539       end
2540       outp.puts
2541       suiteJson[benchmark.name] = benchmarkJson
2542       suite.subSuites.each {
2543         | subSuite |
2544         if subSuite.hasBenchmark(benchmark)
2545           subSuiteJsons[subSuite][benchmark.name] = benchmarkJson
2546         end
2547       }
2548     }
2549     outp.puts
2550     unless suite.subSuites.empty?
2551       suite.subSuites.each {
2552         | subSuite |
2553         outp.puts "#{subSuite.name}:"
2554         allSummaryStats(outp, subSuiteJsons[subSuite], subSuitesOnVMsForSubSuite[subSuite], subSuite.preferredMean, subSuite.decimalShift)
2555         outp.puts
2556       }
2557       outp.puts "#{suite.name} including #{andJoin(suite.subSuites.map{|v| v.name})}:"
2558     end
2559     allSummaryStats(outp, suiteJson, suitesOnVMsForSuite[suite], suite.preferredMean, suite.decimalShift)
2560     outp.puts if $allSuites.size > 1
2561     
2562     json["suites"][suite.name] = suiteJson
2563     suite.subSuites.each {
2564       | subSuite |
2565       json["suites"][subSuite.name] = subSuiteJsons[subSuite]
2566     }
2567   }
2568   
2569   if $suites.size > 1
2570     printVMs(outp)
2571
2572     scaledResultJson = {}
2573     
2574     outp.puts
2575     printVMs(outp)
2576     outp.puts "Geomean of preferred means:"
2577     outp.print "   "
2578     outp.print rpad("<scaled-result>", $benchpad + $weightpad)
2579     outp.print " "
2580     $vms.size.times {
2581       | index |
2582       if index != 0
2583         outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm
2584       end
2585       outp.print statsToStr(overallResults[index], 0)
2586       scaledResultJson[$vms[index].name] = overallResults[index].jsonMap
2587     }
2588     if overallResults.size>=2
2589       outp.print("    "+overallResults[-1].compareTo(overallResults[0]).longForm)
2590     end
2591     outp.puts
2592     
2593     json["<scaled-result>"] = scaledResultJson
2594   end
2595   outp.puts
2596   
2597   if outp != $stdout
2598     outp.close
2599   end
2600   
2601   if outp != $stdout and not $brief
2602     puts
2603     File.open(reportName + "_report.txt") {
2604       | inp |
2605       puts inp.read
2606     }
2607   end
2608   
2609   if $brief
2610     puts(overallResults.collect{|stats| stats.mean}.join("\t"))
2611     puts(overallResults.collect{|stats| stats.confInt}.join("\t"))
2612   end
2613   
2614   File.open(reportName + ".json", "w") {
2615     | outp |
2616     outp.puts json.to_json
2617   }
2618 end
2619
2620 begin
2621   $sawBenchOptions = false
2622   
2623   def resetBenchOptionsIfNecessary
2624     unless $sawBenchOptions
2625       $includeSunSpider = false
2626       $includeLongSpider = false
2627       $includeV8 = false
2628       $includeKraken = false
2629       $includeJSBench = false
2630       $includeJSRegress = false
2631       $includeAsmBench = false
2632       $includeDSPJS = false
2633       $includeBrowsermarkJS = false
2634       $includeBrowsermarkDOM = false
2635       $includeOctane = false
2636       $includeCompressionBench = false
2637       $sawBenchOptions = true
2638     end
2639   end
2640   
2641   GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
2642                  ['--inner', GetoptLong::REQUIRED_ARGUMENT],
2643                  ['--outer', GetoptLong::REQUIRED_ARGUMENT],
2644                  ['--warmup', GetoptLong::REQUIRED_ARGUMENT],
2645                  ['--no-ss-warmup', GetoptLong::NO_ARGUMENT],
2646                  ['--quantum', GetoptLong::REQUIRED_ARGUMENT],
2647                  ['--minimum', GetoptLong::REQUIRED_ARGUMENT],
2648                  ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
2649                  ['--sunspider', GetoptLong::NO_ARGUMENT],
2650                  ['--longspider', GetoptLong::NO_ARGUMENT],
2651                  ['--v8-spider', GetoptLong::NO_ARGUMENT],
2652                  ['--kraken', GetoptLong::NO_ARGUMENT],
2653                  ['--js-bench', GetoptLong::NO_ARGUMENT],
2654                  ['--js-regress', GetoptLong::NO_ARGUMENT],
2655                  ['--asm-bench', GetoptLong::NO_ARGUMENT],
2656                  ['--dsp', GetoptLong::NO_ARGUMENT],
2657                  ['--browsermark-js', GetoptLong::NO_ARGUMENT],
2658                  ['--browsermark-dom', GetoptLong::NO_ARGUMENT],
2659                  ['--octane', GetoptLong::NO_ARGUMENT],
2660                  ['--compression-bench', GetoptLong::NO_ARGUMENT],
2661                  ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
2662                  ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
2663                  ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
2664                  ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
2665                  ['--dont-copy-vms', GetoptLong::NO_ARGUMENT],
2666                  ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
2667                  ['--brief', GetoptLong::NO_ARGUMENT],
2668                  ['--silent', GetoptLong::NO_ARGUMENT],
2669                  ['--remote', GetoptLong::REQUIRED_ARGUMENT],
2670                  ['--local', GetoptLong::NO_ARGUMENT],
2671                  ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
2672                  ['--slave', GetoptLong::NO_ARGUMENT],
2673                  ['--prepare-only', GetoptLong::NO_ARGUMENT],
2674                  ['--analyze', GetoptLong::REQUIRED_ARGUMENT],
2675                  ['--vms', GetoptLong::REQUIRED_ARGUMENT],
2676                  ['--output-name', GetoptLong::REQUIRED_ARGUMENT],
2677                  ['--environment', GetoptLong::REQUIRED_ARGUMENT],
2678                  ['--config', GetoptLong::REQUIRED_ARGUMENT],
2679                  ['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
2680     | opt, arg |
2681     case opt
2682     when '--rerun'
2683       $rerun = intArg(opt,arg,1,nil)
2684     when '--inner'
2685       $inner = intArg(opt,arg,1,nil)
2686     when '--outer'
2687       $outer = intArg(opt,arg,1,nil)
2688     when '--warmup'
2689       $warmup = intArg(opt,arg,0,nil)
2690     when '--no-ss-warmup'
2691       $sunSpiderWarmup = false
2692     when '--quantum'
2693       $quantum = intArg(opt,arg,1,nil)
2694     when '--minimum'
2695       $minimum = intArg(opt,arg,1,nil)
2696     when '--timing-mode'
2697       if arg.upcase == "PRECISETIME"
2698         $timeMode = :preciseTime
2699       elsif arg.upcase == "DATE"
2700         $timeMode = :date
2701       elsif arg.upcase == "AUTO"
2702         $timeMode = :auto
2703       else
2704         quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
2705                   "Invalid argument for command-line option")
2706       end
2707     when '--force-vm-kind'
2708       if arg.upcase == "JSC"
2709         $forceVMKind = :jsc
2710       elsif arg.upcase == "DUMPRENDERTREE"
2711         $forceVMKind = :dumpRenderTree
2712       elsif arg.upcase == "WEBKITTESTRUNNER"
2713         $forceVMKind = :webkitTestRunner
2714       elsif arg.upcase == "AUTO"
2715         $forceVMKind = nil
2716       else
2717         quickFail("Expected 'jsc', 'DumpRenderTree', or 'WebKitTestRunner' for --force-vm-kind, but got '#{arg}'.",
2718                   "Invalid argument for command-line option")
2719       end
2720     when '--force-vm-copy'
2721       $needToCopyVMs = true
2722     when '--dont-copy-vms'
2723       $dontCopyVMs = true
2724     when '--sunspider'
2725       resetBenchOptionsIfNecessary
2726       $includeSunSpider = true
2727     when '--longspider'
2728       resetBenchOptionsIfNecessary
2729       $includeLongSpider = true
2730     when '--v8-spider'
2731       resetBenchOptionsIfNecessary
2732       $includeV8 = true
2733     when '--kraken'
2734       resetBenchOptionsIfNecessary
2735       $includeKraken = true
2736     when '--js-bench'
2737       resetBenchOptionsIfNecessary
2738       $includeJSBench = true
2739     when '--js-regress'
2740       resetBenchOptionsIfNecessary
2741       $includeJSRegress = true
2742     when '--asm-bench'
2743       resetBenchOptionsIfNecessary
2744       $includeAsmBench = true
2745     when '--dsp'
2746       resetBenchOptionsIfNecessary
2747       $includeDSPJS = true
2748     when '--browsermark-js'
2749       resetBenchOptionsIfNecessary
2750       $includeBrowsermarkJS = true
2751     when '--browsermark-dom'
2752       resetBenchOptionsIfNecessary
2753       $includeBrowsermarkDOM = true
2754     when '--octane'
2755       resetBenchOptionsIfNecessary
2756       $includeOctane = true
2757     when '--compression-bench'
2758       resetBenchOptionsIfNecessary
2759       $includeCompressionBench = true
2760     when '--benchmarks'
2761       $benchmarkPattern = Regexp.new(arg)
2762     when '--measure-gc'
2763       if arg == ''
2764         $measureGC = true
2765       else
2766         $measureGC = arg
2767       end
2768     when '--verbose'
2769       $verbosity += 1
2770     when '--brief'
2771       $brief = true
2772     when '--silent'
2773       $silent = true
2774     when '--remote'
2775       $remoteHosts += arg.split(',')
2776       $needToCopyVMs = true
2777     when '--ssh-options'
2778       $sshOptions << arg
2779     when '--local'
2780       $alsoLocal = true
2781     when '--prepare-only'
2782       $run = false
2783     when '--analyze'
2784       $prepare = false
2785       $run = false
2786       $analyze << arg
2787     when '--output-name'
2788       $outputName = arg
2789     when '--vms'
2790       JSON::parse(IO::read(arg)).each {
2791         | vmDescription |
2792         path = Pathname.new(vmDescription["path"]).realpath
2793         if vmDescription["name"]
2794           name = vmDescription["name"]
2795           nameKind = :given
2796         else
2797           name = "Conf\##{$vms.length+1}"
2798           nameKind = :auto
2799         end
2800         vm = VM.new(path, name, nameKind, nil)
2801         if vmDescription["env"]
2802           vmDescription["env"].each_pair {
2803             | key, val |
2804             vm.addExtraEnv(key, val)
2805           }
2806         end
2807         $vms << vm
2808       }
2809     when '--environment'
2810       $environment = JSON::parse(IO::read(arg))
2811     when '--config'
2812       $configPath = Pathname.new(arg)
2813     when '--help'
2814       usage
2815     else
2816       raise "bad option: #{opt}"
2817     end
2818   }
2819   
2820   # Figure out the configuration
2821   if $configPath.file?
2822     config = JSON::parse(IO::read($configPath.to_s))
2823   else
2824     config = {}
2825   end
2826   OCTANE_PATH = config["OctanePath"]
2827   BROWSERMARK_PATH = config["BrowserMarkPath"]
2828   BROWSERMARK_JS_PATH = config["BrowserMarkJSPath"]
2829   BROWSERMARK_DOM_PATH = config["BrowserMarkDOMPath"]
2830   ASMBENCH_PATH = config["AsmBenchPath"]
2831   COMPRESSIONBENCH_PATH = config["CompressionBenchPath"]
2832   DSPJS_FILTRR_PATH = config["DSPJSFiltrrPath"]
2833   DSPJS_ROUTE9_PATH = config["DSPJSRoute9Path"]
2834   DSPJS_STARFIELD_PATH = config["DSPJSStarfieldPath"]
2835   DSPJS_QUAKE3_PATH = config["DSPJSQuake3Path"]
2836   DSPJS_MANDELBROT_PATH = config["DSPJSMandelbrotPath"]
2837   DSPJS_JSLINUX_PATH = config["DSPJSLinuxPath"]
2838   DSPJS_AMMOJS_ASMJS_PATH = config["DSPJSAmmoJSAsmJSPath"]
2839   DSPJS_AMMOJS_REGULAR_PATH = config["DSPJSAmmoJSRegularPath"]
2840   JSBENCH_PATH = config["JSBenchPath"]
2841   KRAKEN_PATH = config["KrakenPath"]
2842   
2843   # If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option.
2844   if $dontCopyVMs
2845     $needToCopyVMs = false
2846   end
2847   
2848   ARGV.each {
2849     | vm |
2850     if vm =~ /([a-zA-Z0-9_ ]+):/
2851       name = $1
2852       nameKind = :given
2853       vm = $~.post_match
2854     else
2855       name = "Conf\##{$vms.length+1}"
2856       nameKind = :auto
2857     end
2858     envs = []
2859     while vm =~ /([a-zA-Z0-9_]+)=([a-zA-Z0-9_:]+):/
2860       envs << [$1, $2]
2861       vm = $~.post_match
2862     end
2863     $stderr.puts "#{name}: #{vm}" if $verbosity >= 1
2864     vm = VM.new(Pathname.new(vm).realpath, name, nameKind, nil)
2865     envs.each {
2866       | pair |
2867       vm.addExtraEnv(pair[0], pair[1])
2868     }
2869     $vms << vm
2870   }
2871   
2872   if $vms.empty?
2873     quickFail("Please specify at least on configuraiton on the command line.",
2874               "Insufficient arguments")
2875   end
2876   
2877   $vms.each {
2878     | vm |
2879     if vm.vmType == :jsc
2880       $allDRT = false
2881     end
2882   }
2883   
2884   SUNSPIDER = BenchmarkSuite.new("SunSpider", :arithmeticMean, 0)
2885   WARMUP = BenchmarkSuite.new("WARMUP", :arithmeticMean, 0)
2886   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
2887    "access-fannkuch", "access-nbody", "access-nsieve",
2888    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
2889    "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
2890    "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
2891    "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
2892    "string-base64", "string-fasta", "string-tagcloud",
2893    "string-unpack-code", "string-validate-input"].each {
2894     | name |
2895     SUNSPIDER.add SunSpiderBenchmark.new(name)
2896     WARMUP.addIgnoringPattern SunSpiderBenchmark.new(name)
2897   }
2898
2899   LONGSPIDER = BenchmarkSuite.new("LongSpider", :geometricMean, 0)
2900   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
2901    "access-fannkuch", "access-nbody", "access-nsieve",
2902    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-nsieve-bits",
2903    "controlflow-recursive", "crypto-aes", "crypto-md5", "crypto-sha1",
2904    "date-format-tofte", "date-format-xparb", "math-cordic",
2905    "math-partial-sums", "math-spectral-norm", "string-base64",
2906    "string-fasta", "string-tagcloud"].each {
2907     | name |
2908     LONGSPIDER.add LongSpiderBenchmark.new(name)
2909   }
2910
2911   V8 = BenchmarkSuite.new("V8Spider", :geometricMean, 0)
2912   ["crypto", "deltablue", "earley-boyer", "raytrace",
2913    "regexp", "richards", "splay"].each {
2914     | name |
2915     V8.add V8Benchmark.new(name)
2916   }
2917
2918   OCTANE = BenchmarkSuite.new("Octane", :geometricMean, 1)
2919   [[["crypto"], "encrypt", 1, true, false, 32],
2920    [["crypto"], "decrypt", 1, true, false, 32],
2921    [["deltablue"], "deltablue", 2, true, false, 32],
2922    [["earley-boyer"], "earley", 1, true, false, 32],
2923    [["earley-boyer"], "boyer", 1, true, false, 32],
2924    [["navier-stokes"], "navier-stokes", 2, true, false, 16],
2925    [["raytrace"], "raytrace", 2, true, false, 32],
2926    [["richards"], "richards", 2, true, false, 32],
2927    [["splay"], "splay", 2, true, false, 32],
2928    [["regexp"], "regexp", 2, false, false, 16],
2929    [["pdfjs"], "pdfjs", 2, false, false, 4],
2930    [["mandreel"], "mandreel", 2, false, false, 4],
2931    [["gbemu-part1", "gbemu-part2"], "gbemu", 2, false, false, 4],
2932    [["code-load"], "closure", 1, false, false, 16],
2933    [["code-load"], "jquery", 1, false, false, 16],
2934    [["box2d"], "box2d", 2, false, false, 8],
2935    [["zlib", "zlib-data"], "zlib", 2, false, true, 3],
2936    [["typescript", "typescript-input", "typescript-compiler"], "typescript", 2, false, true, 1]].each {
2937     | args |
2938     OCTANE.add OctaneBenchmark.new(*args)
2939   }
2940
2941   KRAKEN = BenchmarkSuite.new("Kraken", :arithmeticMean, -1)
2942   ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
2943    "audio-oscillator", "imaging-darkroom", "imaging-desaturate",
2944    "imaging-gaussian-blur", "json-parse-financial",
2945    "json-stringify-tinderbox", "stanford-crypto-aes",
2946    "stanford-crypto-ccm", "stanford-crypto-pbkdf2",
2947    "stanford-crypto-sha256-iterative"].each {
2948     | name |
2949     KRAKEN.add KrakenBenchmark.new(name)
2950   }
2951   
2952   JSBENCH = BenchmarkSuite.new("JSBench", :arithmeticMean, 0)
2953   [["amazon", "urm"], ["facebook", "urem"], ["google", "urem"], ["twitter", "urem"],
2954    ["yahoo", "urem"]].each {
2955     | nameAndMode |
2956     JSBENCH.add JSBenchBenchmark.new(*nameAndMode)
2957   }
2958   
2959   JSREGRESS = BenchmarkSuite.new("JSRegress", :geometricMean, 0)
2960   Dir.foreach(JSREGRESS_PATH) {
2961     | filename |
2962     if filename =~ /\.js$/
2963       name = $~.pre_match
2964       JSREGRESS.add JSRegressBenchmark.new(name)
2965     end
2966   }
2967   
2968   ASMBENCH = BenchmarkSuite.new("AsmBench", :geometricMean, 0)
2969   if JSBENCH_PATH
2970     Dir.foreach(ASMBENCH_PATH) {
2971       | filename |
2972       if filename =~ /\.js$/
2973         name = $~.pre_match
2974         ASMBENCH.add AsmBenchBenchmark.new(name)
2975       end
2976     }
2977   end
2978
2979   COMPRESSIONBENCH = BenchmarkSuite.new("CompressionBench", :geometricMean, 0)
2980   [[["huffman", "compression-data"], "huffman", ""],
2981    [["arithmetic", "compression-data"], "arithmetic", "Simple"],
2982    [["arithmetic", "compression-data"], "arithmetic", "Precise"],
2983    [["arithmetic", "compression-data"], "arithmetic", "Complex Precise"],
2984    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 0"],
2985    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 1"],
2986    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 2"],
2987    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 1"],
2988    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 2"],
2989    [["lz-string", "compression-data"], "lz-string", ""]
2990   ].each {
2991     | args |
2992     COMPRESSIONBENCH.add CompressionBenchBenchmark.new(*args)
2993   }
2994
2995   DSPJS = BenchmarkSuite.new("DSP", :geometricMean, 0)
2996   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-posterize-tint", "e2")
2997   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-contrast-sat-bright", "e5")
2998   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-sat-adj-contr-mult", "e7")
2999   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-blur-overlay-sat-contr", "e8")
3000   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sat-blur-mult-sharpen-contr", "e9")
3001   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sepia-bias", "e10")
3002   DSPJS.add DSPJSVP8Benchmark.new
3003   DSPJS.add DSPStarfieldBenchmark.new
3004   DSPJS.add DSPJSJSLinuxBenchmark.new
3005   DSPJS.add DSPJSQuake3Benchmark.new
3006   DSPJS.add DSPJSMandelbrotBenchmark.new
3007   DSPJS.add DSPJSAmmoJSASMBenchmark.new
3008   DSPJS.add DSPJSAmmoJSRegularBenchmark.new
3009
3010   BROWSERMARK_JS = BenchmarkSuite.new("BrowsermarkJS", :geometricMean, 1)
3011   ["array_blur", "array_weighted", "string_chat", "string_filter", "string_weighted"].each {
3012     | name |
3013       BROWSERMARK_JS.add BrowsermarkJSBenchmark.new(name)
3014   }
3015   
3016   BROWSERMARK_DOM = BenchmarkSuite.new("BrowsermarkDOM", :geometricMean, 1)
3017   ["advanced_search", "create_source", "dynamic_create", "search"].each {
3018     | name |
3019       BROWSERMARK_DOM.add BrowsermarkDOMBenchmark.new(name)
3020   }
3021
3022   $suites = []
3023   
3024   if $includeSunSpider and not SUNSPIDER.empty?
3025     $suites << SUNSPIDER
3026   end
3027   
3028   if $includeLongSpider and not LONGSPIDER.empty?
3029     $suites << LONGSPIDER
3030   end
3031   
3032   if $includeV8 and not V8.empty?
3033     $suites << V8
3034   end
3035   
3036   if $includeOctane and not OCTANE.empty?
3037     if OCTANE_PATH
3038       $suites << OCTANE
3039     else
3040       $stderr.puts "Warning: refusing to run Octane because \"OctanePath\" isn't set in #{$configPath}."
3041     end
3042   end
3043   
3044   if $includeKraken and not KRAKEN.empty?
3045     if KRAKEN_PATH
3046       $suites << KRAKEN
3047     else
3048       $stderr.puts "Warning: refusing to run Kraken because \"KrakenPath\" isn't set in #{$configPath}."
3049     end
3050   end
3051   
3052   if $includeJSBench and not JSBENCH.empty?
3053     if $allDRT
3054       if JSBENCH_PATH
3055         $suites << JSBENCH
3056       else
3057         $stderr.puts "Warning: refusing to run JSBench because \"JSBenchPath\" isn't set in #{$configPath}"
3058       end
3059     else
3060       $stderr.puts "Warning: refusing to run JSBench because not all VMs are DumpRenderTree or WebKitTestRunner."
3061     end
3062   end
3063   
3064   if $includeJSRegress and not JSREGRESS.empty?
3065     $suites << JSREGRESS
3066   end
3067   
3068   if $includeAsmBench and not ASMBENCH.empty?
3069     if ASMBENCH_PATH
3070       $suites << ASMBENCH
3071     else
3072       $stderr.puts "Warning: refusing to run AsmBench because \"AsmBenchPath\" isn't set in #{$configPath}."
3073     end
3074   end
3075   
3076   if $includeDSPJS and not DSPJS.empty?
3077     if $allDRT
3078       if DSPJS_FILTRR_PATH and DSPJS_ROUTE9_PATH and DSPJS_STARFIELD_PATH and DSPJS_QUAKE3_PATH and DSPJS_MANDELBROT_PATH and DSPJS_JSLINUX_PATH and DSPJS_AMMOJS_ASMJS_PATH and DSPJS_AMMOJS_REGULAR_PATH
3079         $suites << DSPJS
3080       else
3081         $stderr.puts "Warning: refusing to run DSPJS because one of the following isn't set in #{$configPath}: \"DSPJSFiltrrPath\", \"DSPJSRoute9Path\", \"DSPJSStarfieldPath\", \"DSPJSQuake3Path\", \"DSPJSMandelbrotPath\", \"DSPJSLinuxPath\", \"DSPJSAmmoJSAsmJSPath\", \"DSPJSAmmoJSRegularPath\"."
3082       end
3083     else
3084       $stderr.puts "Warning: refusing to run DSPJS because not all VMs are DumpRenderTree or WebKitTestRunner."
3085     end
3086   end
3087   
3088   if $includeBrowsermarkJS and not BROWSERMARK_JS.empty?
3089     if BROWSERMARK_PATH and BROWSERMARK_JS_PATH
3090       $suites << BROWSERMARK_JS
3091     else
3092       $stderr.puts "Warning: refusing to run Browsermark-JS because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\" or \"BrowserMarkJSPath\"."
3093     end
3094   end
3095
3096   if $includeBrowsermarkDOM and not BROWSERMARK_DOM.empty?
3097     if $allDRT
3098       if BROWSERMARK_PATH and BROWSERMARK_JS_PATH and BROWSERMARK_DOM_PATH
3099         $suites << BROWSERMARK_DOM
3100       else
3101         $stderr.puts "Warning: refusing to run Browsermark-DOM because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\", \"BrowserMarkJSPath\", or \"BrowserMarkDOMPath\"."
3102       end
3103     else
3104       $stderr.puts "Warning: refusing to run Browsermark-DOM because not all VMs are DumpRenderTree or WebKitTestRunner."
3105     end
3106   end
3107
3108   if $includeCompressionBench and not COMPRESSIONBENCH.empty?
3109     if COMPRESSIONBENCH_PATH
3110       $suites << COMPRESSIONBENCH
3111     else
3112       $stderr.puts "Warning: refusing to run CompressionBench because \"CompressionBenchPath\" isn't set in #{$configPath}"
3113     end
3114   end
3115
3116   $allSuites = $suites.map{|v| v.suites}.flatten(1)
3117   
3118   $benchmarks = []
3119   $suites.each {
3120     | suite |
3121     $benchmarks += suite.benchmarks
3122   }
3123   
3124   if $suites.empty? or $benchmarks.empty?
3125     $stderr.puts "No benchmarks found.  Bailing out."
3126     exit 1
3127   end
3128   
3129   if $outer*$inner == 1
3130     $stderr.puts "Warning: will only collect one sample per benchmark/VM.  Confidence interval calculation will fail."
3131   end
3132   
3133   $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
3134   
3135   $runPlans = []
3136   $vms.each {
3137     | vm |
3138     $benchmarks.each {
3139       | benchmark |
3140       $outer.times {
3141         | iteration |
3142         $runPlans << BenchRunPlan.new(benchmark, vm, iteration)
3143       }
3144     }
3145   }
3146   
3147   $runPlans.shuffle!
3148   
3149   if $sunSpiderWarmup
3150     warmupPlans = []
3151     $vms.each {
3152       | vm |
3153       WARMUP.benchmarks.each {
3154         | benchmark |
3155         warmupPlans << BenchRunPlan.new(benchmark, vm, 0)
3156       }
3157     }
3158     
3159     $runPlans = warmupPlans.shuffle + $runPlans
3160   end
3161   
3162   $suitepad = $suites.collect {
3163     | suite |
3164     suite.to_s.size
3165   }.max + 1
3166   
3167   $planpad = $runPlans.collect {
3168     | plan |
3169     plan.to_s.size
3170   }.max + 1
3171   
3172   maxBenchNameLength =
3173     ($benchmarks + ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
3174     | benchmark |
3175     if benchmark.respond_to? :name
3176       benchmark.name.size
3177     else
3178       benchmark.size
3179     end
3180   }.max
3181   $benchNameClip = 40
3182   $benchpad = [maxBenchNameLength, $benchNameClip].min + 1
3183   
3184   $weightpad = $benchmarks.collect {
3185     | benchmark |
3186     benchmark.weightString.size
3187   }.max
3188
3189   $vmpad = $vms.collect {
3190     | vm |
3191     vm.to_s.size
3192   }.max + 1
3193   
3194   $analyze.each_with_index {
3195     | filename, index |
3196     if index >= 1
3197       puts
3198     end
3199     parseAndDisplayResults(IO::read(filename))
3200   }
3201   
3202   if not $prepare and not $run
3203     exit 0
3204   end
3205   
3206   if FileTest.exist? BENCH_DATA_PATH
3207     cmd = "rm -rf #{BENCH_DATA_PATH}"
3208     $stderr.puts ">> #{cmd}" if $verbosity >= 2
3209     raise unless system cmd
3210   end
3211   
3212   Dir.mkdir BENCH_DATA_PATH
3213   
3214   if $needToCopyVMs
3215     canCopyIntoBenchPath = true
3216     $vms.each {
3217       | vm |
3218       canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath
3219     }
3220     
3221     if canCopyIntoBenchPath
3222       $vms.each {
3223         | vm |
3224         $stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..."
3225         vm.copyIntoBenchPath
3226       }
3227       $stderr.puts "All VMs are in place."
3228     else
3229       $stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it."
3230     end
3231   end
3232   
3233   if $measureGC and $measureGC != true
3234     found = false
3235     $vms.each {
3236       | vm |
3237       if vm.name == $measureGC
3238         found = true
3239       end
3240     }
3241     unless found
3242       $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
3243     end
3244   end
3245   
3246   if $prepare
3247     File.open("#{BENCH_DATA_PATH}/runscript", "w") {
3248       | file |
3249       file.puts "echo \"HOSTNAME:\\c\""
3250       file.puts "hostname"
3251       file.puts "echo"
3252       file.puts "echo \"HARDWARE:\\c\""
3253       file.puts "/usr/sbin/sysctl hw.model"
3254       file.puts "echo"
3255       file.puts "set -e"
3256       $script = file
3257       $runPlans.each_with_index {
3258         | plan, idx |
3259         if $verbosity == 0 and not $silent
3260           text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s
3261           text2 = plan.to_s
3262           file.puts("echo " + Shellwords.shellescape("\r#{text1} #{rpad(text2,$planpad)}") + "\"\\c\" 1>&2")
3263           file.puts("echo " + Shellwords.shellescape("\r#{text1} #{text2}") + "\"\\c\" 1>&2")
3264         end
3265         plan.emitRunCode
3266       }
3267       if $verbosity == 0 and not $silent
3268         file.puts("echo " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}") + "\"\\c\" 1>&2")
3269         file.puts("echo " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size}") + " 1>&2")
3270       end
3271     }
3272   end
3273   
3274   if $run
3275     unless $remoteHosts.empty?
3276       $stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0
3277       Dir.chdir(TEMP_PATH) {
3278         cmd = "tar -czf payload.tar.gz benchdata"
3279         $stderr.puts ">> #{cmd}" if $verbosity>=2
3280         raise unless system(cmd)
3281       }
3282       
3283       def grokHost(host)
3284         if host =~ /:([0-9]+)$/
3285           "-p " + $1 + " " + Shellwords.shellescape($~.pre_match)
3286         else
3287           Shellwords.shellescape(host)
3288         end
3289       end
3290       
3291       def sshRead(host, command)
3292         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3293         $stderr.puts ">> #{cmd}" if $verbosity>=2
3294         result = ""
3295         IO.popen(cmd, "r") {
3296           | inp |
3297           inp.each_line {
3298             | line |
3299             $stderr.puts "#{host}: #{line}" if $verbosity>=2
3300             result += line
3301           }
3302         }
3303         raise "#{$?}" unless $?.success?
3304         result
3305       end
3306       
3307       def sshWrite(host, command, data)
3308         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3309         $stderr.puts ">> #{cmd}" if $verbosity>=2
3310         IO.popen(cmd, "w") {
3311           | outp |
3312           outp.write(data)
3313         }
3314         raise "#{$?}" unless $?.success?
3315       end
3316       
3317       $remoteHosts.each {
3318         | host |
3319         $stderr.puts "Sending benchmark payload to #{host}..." if $verbosity==0
3320         
3321         remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"]
3322         raise unless remoteTempPath
3323         
3324         sshWrite(host, "cd #{Shellwords.shellescape(remoteTempPath)} && rm -rf benchdata && tar -xz", IO::read("#{TEMP_PATH}/payload.tar.gz"))
3325         
3326         $stderr.puts "Running on #{host}..." if $verbosity==0
3327         
3328         parseAndDisplayResults(sshRead(host, "cd #{Shellwords.shellescape(remoteTempPath + '/benchdata')} && sh runscript"))
3329       }
3330     end
3331     
3332     if not $remoteHosts.empty? and $alsoLocal
3333       $stderr.puts "Running locally..."
3334     end
3335     
3336     if $remoteHosts.empty? or $alsoLocal
3337       parseAndDisplayResults(runAndGetResults)
3338     end
3339   end
3340   
3341   if $prepare and not $run and $analyze.empty?
3342     puts wrap("Benchmarking script and data are in #{BENCH_DATA_PATH}. You can run "+
3343               "the benchmarks and get the results by doing:", 78)
3344     puts
3345     puts "cd #{BENCH_DATA_PATH}"
3346     puts "sh runscript > results.txt"
3347     puts
3348     puts wrap("Then you can analyze the results by running bencher with the same arguments "+
3349               "as now, but replacing --prepare-only with --analyze results.txt.", 78)
3350   end
3351 rescue => e
3352   fail(e)
3353 end
3354