[BigInt] Add support to BigInt into ValueAdd
[WebKit-https.git] / Tools / Scripts / run-jsc-benchmarks
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2011-2018 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 # 1. Redistributions of source code must retain the above copyright
9 #    notice, this list of conditions and the following disclaimer.
10 # 2. Redistributions in binary form must reproduce the above copyright
11 #    notice, this list of conditions and the following disclaimer in the
12 #    documentation and/or other materials provided with the distribution.
13 #
14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24 # THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'rubygems'
27
28 require 'getoptlong'
29 require 'pathname'
30 require 'shellwords'
31 require 'socket'
32
33 begin
34   require 'json'
35 rescue LoadError => e
36   $stderr.puts "It does not appear that you have the 'json' package installed.  Try running 'sudo gem install json'."
37   exit 1
38 end
39
40 SCRIPT_PATH = Pathname.new(__FILE__).realpath
41
42 raise unless SCRIPT_PATH.dirname.basename.to_s == "Scripts"
43 raise unless SCRIPT_PATH.dirname.dirname.basename.to_s == "Tools"
44
45 OPENSOURCE_PATH = SCRIPT_PATH.dirname.dirname.dirname
46 PERFORMANCETESTS_PATH = OPENSOURCE_PATH + "PerformanceTests"
47
48 SUNSPIDER_PATH = PERFORMANCETESTS_PATH + "SunSpider" + "tests" + "sunspider-1.0"
49 LONGSPIDER_PATH = PERFORMANCETESTS_PATH + "LongSpider"
50 V8_PATH = PERFORMANCETESTS_PATH + "SunSpider" + "tests" + "v8-v6"
51 TAILBENCH_PATH = PERFORMANCETESTS_PATH + "TailBench9000"
52 BIGINTBENCH_PATH = PERFORMANCETESTS_PATH + "BigIntBench"
53 MICROBENCHMARKS_PATH = OPENSOURCE_PATH + "JSTests" + "microbenchmarks"
54 OPENSOURCE_OCTANE_PATH = PERFORMANCETESTS_PATH + "Octane"
55 OCTANE_WRAPPER_PATH = OPENSOURCE_OCTANE_PATH + "wrappers"
56 JSBENCH_PATH = PERFORMANCETESTS_PATH + "JSBench"
57 SIXSPEED_PATH = PERFORMANCETESTS_PATH + "SixSpeed" + "tests"
58 SIXSPEED_WRAPPER_PATH = PERFORMANCETESTS_PATH + "SixSpeed" + "wrappers"
59
60 TEMP_PATH = OPENSOURCE_PATH + "BenchmarkTemp"
61
62 if TEMP_PATH.exist?
63   raise unless TEMP_PATH.directory?
64 else
65   Dir.mkdir(TEMP_PATH)
66 end
67
68 BENCH_DATA_PATH = TEMP_PATH + "benchdata"
69
70 IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933, 
71             0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684, 
72             0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227, 
73             0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674, 
74             0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652, 
75             0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604, 
76             0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338, 
77             0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538, 
78             0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185, 
79             0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694, 
80             0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244, 
81             0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067, 
82             0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323, 
83             0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129, 
84             0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574, 
85             0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722, 
86             0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624, 
87             0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321, 
88             0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843, 
89             0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217, 
90             0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464, 
91             0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599, 
92             0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638, 
93             0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592, 
94             0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471, 
95             0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283, 
96             0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037, 
97             0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738, 
98             0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481, 
99             0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085, 
100             0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651, 
101             0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183, 
102             0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683, 
103             0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154, 
104             0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599, 
105             0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019, 
106             0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417, 
107             0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795, 
108             0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153, 
109             0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494, 
110             0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818, 
111             0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128, 
112             0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423, 
113             0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704, 
114             0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974, 
115             0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232, 
116             0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479, 
117             0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716, 
118             0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943, 
119             0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192, 
120             0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401, 
121             0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963, 
122             0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822, 
123             0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007, 
124             0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186, 
125             0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358, 
126             0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525, 
127             0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686, 
128             0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841, 
129             0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991, 
130             0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137, 
131             0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278, 
132             0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414, 
133             0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547, 
134             0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675, 
135             0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799, 
136             0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192, 
137             0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037, 
138             0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151, 
139             0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262, 
140             0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369, 
141             0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489, 
142             0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259, 
143             0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689, 
144             0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785, 
145             0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879, 
146             0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297, 
147             0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059, 
148             0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145, 
149             0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242, 
150             0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324, 
151             0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404, 
152             0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483, 
153             0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559, 
154             0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634, 
155             0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707, 
156             0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778, 
157             0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848, 
158             0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916, 
159             0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983, 
160             0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057, 
161             0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121, 
162             0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183, 
163             0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244, 
164             0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304, 
165             0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363, 
166             0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442, 
167             0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476, 
168             0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539, 
169             0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946, 
170             0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653, 
171             0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712, 
172             0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762, 
173             0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812, 
174             0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867, 
175             0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915, 
176             0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962, 
177             0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007, 
178             0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059, 
179             0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103, 
180             0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146, 
181             0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189, 
182             0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231, 
183             0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272, 
184             0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312, 
185             0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352, 
186             0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391, 
187             0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429, 
188             0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467, 
189             0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504, 
190             0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546, 
191             0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582, 
192             0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617, 
193             0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652, 
194             0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686, 
195             0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724, 
196             0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757, 
197             0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579, 
198             0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822, 
199             0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853, 
200             0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885, 
201             0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915, 
202             0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595, 
203             0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979, 
204             0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013, 
205             0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041, 
206             0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074, 
207             0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102, 
208             0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129, 
209             0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156, 
210             0.99616, 0.996164]
211
212 # Run-time configuration parameters (can be set with command-line options)
213
214 $rerun=1
215 $inner=1
216 $warmup=1
217 $outer=4
218 $quantum=1000
219 $includeSunSpider=true
220 $includeSunSpiderCompileTime=true
221 $includeLongSpider=false
222 $includeV8=true
223 $includeV8CompileTime=true
224 $includeKraken=true
225 $includeJSBench=true
226 $includeMicrobenchmarks=true
227 $includeAsmBench=true
228 $includeDSPJS=true
229 $includeBrowsermarkJS=false
230 $includeBrowsermarkDOM=false
231 $includeOctane=true
232 $includeCompressionBench = false
233 $includeSixSpeed = false
234 $includeTailBench = true
235 $includeBigIntBench = false
236 $measureGC=false
237 $benchmarkPattern=nil
238 $verbosity=0
239 $timeMode=:preciseTime
240 $forceVMKind=nil
241 $brief=false
242 $silent=false
243 $remoteHosts=[]
244 $alsoLocal=false
245 $sshOptions=[]
246 $vms = []
247 $environment = {}
248 $dependencies = []
249 $needToCopyVMs = false
250 $dontCopyVMs = false
251 $allDRT = true
252 $outputName = nil
253 $sunSpiderWarmup = true
254 $configPath = Pathname.new(ENV["HOME"]) + ".run-jsc-benchmarks"
255
256 $prepare = true
257 $run = true
258 $analyze = []
259
260 # Helpful functions and classes
261
262 def smallUsage
263   puts "Use the --help option to get basic usage information."
264   exit 1
265 end
266
267 def usage
268   puts "run-jsc-benchmarks [options] <vm1> [<vm2> ...]"
269   puts
270   puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
271   puts "benchmarks, and reports detailed statistics.  What makes run-jsc-benchmarks"
272   puts "special is that each benchmark/VM configuration is run in a single VM invocation,"
273   puts "and the invocations are run in random order.  This minimizes systematics due to"
274   puts "one benchmark polluting the running time of another.  The fine-grained"
275   puts "interleaving of VM invocations further minimizes systematics due to changes in"
276   puts "the performance or behavior of your machine."
277   puts 
278   puts "Run-jsc-benchmarks is highly configurable.  You can compare as many VMs as you"
279   puts "like.  You can change the amount of warm-up iterations, number of iterations"
280   puts "executed per VM invocation, and the number of VM invocations per benchmark."
281   puts
282   puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
283   puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
284   puts "the executable and <name> is the name that you would like to give the"
285   puts "configuration for the purposeof reporting.  If no name is given, a generic name"
286   puts "of the form Conf#<n> will be ascribed to the configuration automatically."
287   puts
288   puts "It's also possible to specify per-VM environment variables. For example, you"
289   puts "might specify a VM like Foo:JSC_useJIT=false:/path/to/jsc, in which case the"
290   puts "harness will set the JSC_useJIT environment variable to false just before running"
291   puts "the given VM. Note that the harness will not unset the environment variable, so"
292   puts "you must ensure that your other VMs will use the opposite setting"
293   puts "(JSC_useJIT=true in this case)."
294   puts
295   puts "Options:"
296   puts "--rerun <n>          Set the number of iterations of the benchmark that"
297   puts "                     contribute to the measured run time.  Default is #{$rerun}."
298   puts "--inner <n>          Set the number of inner (per-runtime-invocation)"
299   puts "                     iterations.  Default is #{$inner}."
300   puts "--outer <n>          Set the number of runtime invocations for each benchmark."
301   puts "                     Default is #{$outer}."
302   puts "--warmup <n>         Set the number of warm-up runs per invocation.  Default"
303   puts "                     is #{$warmup}. This has a different effect on different kinds"
304   puts "                     benchmarks. Some benchmarks have no notion of warm-up."
305   puts "--no-ss-warmup       Disable SunSpider-based warm-up runs."
306   puts "--quantum <n>        Set the duration in milliseconds for which an iteration of"
307   puts "                     a throughput benchmark should be run.  Default is #{$quantum}."
308   puts "--timing-mode        Set the way that time is measured.  Possible values"
309   puts "                     are 'preciseTime' and 'date'.  Default is 'preciseTime'."
310   puts "--force-vm-kind      Turn off auto-detection of VM kind, and assume that it is"
311   puts "                     the one specified.  Valid arguments are 'jsc', "
312   puts "                     'DumpRenderTree', or 'WebKitTestRunner'."
313   puts "--force-vm-copy      Force VM builds to be copied to the working directory."
314   puts "                     This may reduce pathologies resulting from path names."
315   puts "--dont-copy-vms      Don't copy VMs even when doing a remote benchmarking run;"
316   puts "                     instead assume that they are already there."
317   puts "--sunspider          Only run SunSpider."
318   puts "--sunspider-compile-time"
319   puts "                     Only run the SunSpider compile time benchmark."
320   puts "--v8-spider          Only run SunSpider-style V8."
321   puts "--v8-spider-compile-time"
322   puts "                     Only run the SunSpider-style V8 compile time benchmark."
323   puts "--kraken             Only run Kraken."
324   puts "--js-bench           Only run JSBench."
325   puts "--microbenchmarks    Only run microbenchmarks."
326   puts "--dsp                Only run DSP."
327   puts "--asm-bench          Only run AsmBench."
328   puts "--browsermark-js     Only run browsermark-js."
329   puts "--browsermark-dom    Only run browsermark-dom."
330   puts "--octane             Only run Octane."
331   puts "--tail-bench         Only run TailBench"
332   puts "--compression-bench  Only run compression bench"
333   puts "                     The default is to run all benchmarks. The above options can"
334   puts "                     be combined to run any subset (so --sunspider --dsp will run"
335   puts "                     both SunSpider and DSP)."
336   puts "--six-speed          Only run SixSpeed."
337   puts "--benchmarks         Only run benchmarks matching the given regular expression."
338   puts "--measure-gc         Turn off manual calls to gc(), so that GC time is measured."
339   puts "                     Works best with large values of --inner.  You can also say"
340   puts "                     --measure-gc <conf>, which turns this on for one"
341   puts "                     configuration only."
342   puts "--verbose or -v      Print more stuff."
343   puts "--brief              Print only the final result for each VM."
344   puts "--silent             Don't print progress. This might slightly reduce some"
345   puts "                     performance perturbation."
346   puts "--remote <sshhosts>  Perform performance measurements remotely, on the given"
347   puts "                     SSH host(s). Easiest way to use this is to specify the SSH"
348   puts "                     user@host string. However, you can also supply a comma-"
349   puts "                     separated list of SSH hosts. Alternatively, you can use this"
350   puts "                     option multiple times to specify multiple hosts. This"
351   puts "                     automatically copies the WebKit release builds of the VMs"
352   puts "                     you specified to all of the hosts."
353   puts "--ssh-options        Pass additional options to SSH."
354   puts "--local              Also do a local benchmark run even when doing --remote."
355   puts "--vms                Use a JSON file to specify which VMs to run, as opposed to"
356   puts "                     specifying them on the command line."
357   puts "--prepare-only       Only prepare the runscript (a shell script that"
358   puts "                     invokes the VMs to run benchmarks) but don't run it."
359   puts "--analyze            Only read the output of the runscript but don't do anything"
360   puts "                     else. This requires passing the same arguments that you"
361   puts "                     passed when running --prepare-only."
362   puts "--output-name        Base of the filenames to put results into. Will write a file"
363   puts "                     called <base>_report.txt and <base>.json.  By default this"
364   puts "                     name is automatically synthesized from the machine name,"
365   puts "                     date, set of benchmarks run, and set of configurations."
366   puts "--environment        JSON file that specifies the environment variables that should"
367   puts "                     be used for particular VMs and benchmarks."
368   puts "--config <path>      Specify the path of the configuration file. Defaults to"
369   puts "                     ~/.run-jsc-benchmarks"
370   puts "--dependencies       Additional dependent library paths."
371   puts "--help or -h         Display this message."
372   puts
373   puts "Example:"
374   puts "run-jsc-benchmarks TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
375   exit 1
376 end
377
378 def fail(reason)
379   if reason.respond_to? :backtrace
380     puts "FAILED: #{reason.inspect}"
381     puts "Stack trace:"
382     puts reason.backtrace.join("\n")
383   else
384     puts "FAILED: #{reason.inspect}"
385   end
386   smallUsage
387 end
388
389 def quickFail(r1,r2)
390   $stderr.puts "#{$0}: #{r1}"
391   puts
392   fail(r2)
393 end
394
395 def intArg(argName,arg,min,max)
396   result=arg.to_i
397   unless result.to_s == arg
398     quickFail("Expected an integer value for #{argName}, but got #{arg}.",
399               "Invalid argument for command-line option")
400   end
401   if min and result<min
402     quickFail("Argument for #{argName} cannot be smaller than #{min}.",
403               "Invalid argument for command-line option")
404   end
405   if max and result>max
406     quickFail("Argument for #{argName} cannot be greater than #{max}.",
407               "Invalid argument for command-line option")
408   end
409   result
410 end
411
412 def computeMean(array)
413   sum=0.0
414   array.each {
415     | value |
416     sum += value
417   }
418   sum/array.length
419 end
420
421 def computeGeometricMean(array)
422   sum = 0.0
423   array.each {
424     | value |
425     sum += Math.log(value)
426   }
427   Math.exp(sum * (1.0/array.length))
428 end
429
430 def computeHarmonicMean(array)
431   1.0 / computeMean(array.collect{ | value | 1.0 / value })
432 end
433
434 def computeStdDev(array)
435   case array.length
436   when 0
437     0.0/0.0
438   when 1
439     0.0
440   else
441     begin
442       mean=computeMean(array)
443       sum=0.0
444       array.each {
445         | value |
446         sum += (value-mean)**2
447       }
448       Math.sqrt(sum/(array.length-1))
449     rescue
450       0.0/0.0
451     end
452   end
453 end
454
455 class Array
456   def shuffle!
457     size.downto(1) { |n| push delete_at(rand(n)) }
458     self
459   end
460 end
461
462 def inverseBetaRegularized(n)
463   IBR_LOOKUP[n-1]
464 end
465
466 def numToStr(num, decimalShift)
467   ("%." + (4 + decimalShift).to_s + "f") % (num.to_f)
468 end
469
470 class CantSay
471   def initialize
472   end
473   
474   def shortForm
475     " "
476   end
477   
478   def longForm
479     ""
480   end
481   
482   def to_s
483     ""
484   end
485 end
486   
487 class NoChange
488   attr_reader :amountFaster
489   
490   def initialize(amountFaster)
491     @amountFaster = amountFaster
492   end
493   
494   def shortForm
495     " "
496   end
497   
498   def longForm
499     "  might be #{numToStr(@amountFaster, 0)}x faster"
500   end
501   
502   def to_s
503     if @amountFaster < 1.01
504       ""
505     else
506       longForm
507     end
508   end
509 end
510
511 class Faster
512   attr_reader :amountFaster
513   
514   def initialize(amountFaster)
515     @amountFaster = amountFaster
516   end
517   
518   def shortForm
519     "^"
520   end
521   
522   def longForm
523     "^ definitely #{numToStr(@amountFaster, 0)}x faster"
524   end
525   
526   def to_s
527     longForm
528   end
529 end
530
531 class Slower
532   attr_reader :amountSlower
533   
534   def initialize(amountSlower)
535     @amountSlower = amountSlower
536   end
537   
538   def shortForm
539     "!"
540   end
541   
542   def longForm
543     "! definitely #{numToStr(@amountSlower, 0)}x slower"
544   end
545   
546   def to_s
547     longForm
548   end
549 end
550
551 class MayBeSlower
552   attr_reader :amountSlower
553   
554   def initialize(amountSlower)
555     @amountSlower = amountSlower
556   end
557   
558   def shortForm
559     "?"
560   end
561   
562   def longForm
563     "? might be #{numToStr(@amountSlower, 0)}x slower"
564   end
565   
566   def to_s
567     if @amountSlower < 1.01
568       "?"
569     else
570       longForm
571     end
572   end
573 end
574
575 def jsonSanitize(value)
576   if value.is_a? Fixnum
577     value
578   elsif value.is_a? Float
579     if value.nan? or value.infinite?
580       value.to_s
581     else
582       value
583     end
584   elsif value.is_a? Array
585     value.map{|v| jsonSanitize(v)}
586   elsif value.nil?
587     value
588   else
589     raise "Unrecognized value #{value.inspect}"
590   end
591 end
592
593 class Stats
594   def initialize
595     @array = []
596   end
597   
598   def add(value)
599     if not value or not @array
600       @array = nil
601     elsif value.is_a? Float
602       if value.nan? or value.infinite?
603         @array = nil
604       else
605         @array << value
606       end
607     elsif value.is_a? Stats
608       add(value.array)
609     elsif value.respond_to? :each
610       value.each {
611         | v |
612         add(v)
613       }
614     else
615       @array << value.to_f
616     end
617   end
618   
619   def status
620     if @array
621       :ok
622     else
623       :error
624     end
625   end
626   
627   def error?
628     # TODO: We're probably still not handling this case correctly. 
629     not @array or @array.empty?
630   end
631   
632   def ok?
633     not not @array
634   end
635     
636   def array
637     @array
638   end
639   
640   def sum
641     result=0
642     @array.each {
643       | value |
644       result += value
645     }
646     result
647   end
648   
649   def min
650     @array.min
651   end
652   
653   def max
654     @array.max
655   end
656   
657   def size
658     @array.length
659   end
660   
661   def mean
662     computeMean(array)
663   end
664   
665   def arithmeticMean
666     mean
667   end
668   
669   def stdDev
670     computeStdDev(array)
671   end
672
673   def stdErr
674     stdDev/Math.sqrt(size)
675   end
676   
677   # Computes a 95% Student's t distribution confidence interval
678   def confInt
679     if size < 2
680       0.0/0.0
681     else
682       raise if size > 1000
683       Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
684     end
685   end
686   
687   def lower
688     mean-confInt
689   end
690   
691   def upper
692     mean+confInt
693   end
694   
695   def geometricMean
696     computeGeometricMean(array)
697   end
698   
699   def harmonicMean
700     computeHarmonicMean(array)
701   end
702   
703   def compareTo(other)
704     return CantSay.new unless ok? and other.ok?
705     
706     if upper < other.lower
707       Faster.new(other.mean/mean)
708     elsif lower > other.upper
709       Slower.new(mean/other.mean)
710     elsif mean > other.mean
711       MayBeSlower.new(mean/other.mean)
712     else
713       NoChange.new(other.mean/mean)
714     end
715   end
716   
717   def to_s
718     "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
719   end
720   
721   def jsonMap
722     if ok?
723       {"data"=>jsonSanitize(@array), "mean"=>jsonSanitize(mean), "confInt"=>jsonSanitize(confInt)}
724     else
725       "ERROR"
726     end
727   end
728 end
729
730 def doublePuts(out1,out2,msg)
731   out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
732   out2.puts msg
733 end
734
735 class Benchfile < File
736   @@counter = 0
737   
738   attr_reader :filename, :basename
739   
740   def initialize(name)
741     @basename, @filename = Benchfile.uniqueFilename(name)
742     super(@filename, "w")
743   end
744   
745   def self.uniqueFilename(name)
746     if name.is_a? Array
747       basename = name[0] + @@counter.to_s + name[1]
748     else
749       basename = name + @@counter.to_s
750     end
751     filename = BENCH_DATA_PATH + basename
752     @@counter += 1
753     raise "Benchfile #{filename} already exists" if FileTest.exist?(filename)
754     [basename, filename]
755   end
756   
757   def self.create(name)
758     file = Benchfile.new(name)
759     yield file
760     file.close
761     file.basename
762   end
763 end
764
765 $dataFiles={}
766 def ensureFile(key, filename)
767   unless $dataFiles[key]
768     $dataFiles[key] = Benchfile.create(key) {
769       | outp |
770       doublePuts($stderr,outp,IO::read(filename))
771     }
772   end
773   $dataFiles[key]
774 end
775
776 # Helper for files that cannot be renamed.
777 $absoluteFiles={}
778 def ensureAbsoluteFile(filename, basedir=nil)
779   return if $absoluteFiles[filename]
780   filename = Pathname.new(filename)
781
782   directory = Pathname.new('')
783   if basedir and filename.dirname != basedir
784     remainingPath = filename.dirname
785     while remainingPath != basedir
786       directory = remainingPath.basename + directory
787       remainingPath = remainingPath.dirname
788     end
789     if not $absoluteFiles[directory]
790       cmd = "mkdir -p #{Shellwords.shellescape((BENCH_DATA_PATH + directory).to_s)}"
791       $stderr.puts ">> #{cmd}" if $verbosity >= 2
792       raise unless system(cmd)
793       intermediateDirectory = Pathname.new(directory)
794       while intermediateDirectory.basename.to_s != "."
795         $absoluteFiles[intermediateDirectory] = true
796         intermediateDirectory = intermediateDirectory.dirname
797       end
798     end
799   end
800   
801   cmd = "cp #{Shellwords.shellescape(filename.to_s)} #{Shellwords.shellescape((BENCH_DATA_PATH + directory + filename.basename).to_s)}"
802   $stderr.puts ">> #{cmd}" if $verbosity >= 2
803   raise unless system(cmd)
804   $absoluteFiles[filename] = true
805 end
806
807 # Helper for large benchmarks with lots of files and directories.
808 def ensureBenchmarkFiles(rootdir)
809     toProcess = [rootdir]
810     while not toProcess.empty?
811       currdir = toProcess.pop
812       Dir.foreach(currdir.to_s) {
813         | filename |
814         path = currdir + filename
815         next if filename.match(/^\./)
816         toProcess.push(path) if File.directory?(path.to_s)
817         ensureAbsoluteFile(path, rootdir) if File.file?(path.to_s)
818       }
819     end
820 end
821
822 class JSCommand
823   attr_reader :js, :html
824   def initialize(js, html)
825     @js = js
826     @html = html
827   end
828 end
829
830 def loadCommandForFile(key, filename)
831   file = ensureFile(key, filename)
832   JSCommand.new("load(#{file.inspect});", "<script src=#{file.inspect}></script>")
833 end
834
835 def simpleCommand(command)
836   JSCommand.new(command, "<script type=\"text/javascript\">#{command}</script>")
837 end
838
839 # Benchmark that consists of a single file and must be loaded in its own global object each
840 # time (i.e. run()).
841 class SingleFileTimedBenchmarkParameters
842   attr_reader :benchPath
843   
844   def initialize(benchPath)
845     @benchPath = benchPath
846   end
847   
848   def kind
849     :singleFileTimedBenchmark
850   end
851 end
852
853 # Benchmark that consists of a single file and must be loaded in its own global object each
854 # time (i.e. run()), but returns its result in a custom way.
855 class SingleFileCustomTimedBenchmarkParameters
856   attr_reader :benchPath
857   
858   def initialize(benchPath)
859     @benchPath = benchPath
860   end
861   
862   def kind
863     :singleFileCustomTimedBenchmark
864   end
865 end
866
867 # Benchmark that consists of a single file and must be loaded in its own global object each
868 # time (i.e. run()), and returns the time spent compiling
869 class SingleFileCompileTimeBenchmarkParameters
870   attr_reader :benchPath
871   
872   def initialize(benchPath)
873     @benchPath = benchPath
874   end
875   
876   def kind
877     :singleFileCompileTimeBenchmark
878   end
879 end
880
881 # Benchmark that consists of one or more data files that should be loaded globally, followed
882 # by a command to run the benchmark.
883 class MultiFileTimedBenchmarkParameters
884   attr_reader :dataPaths, :command
885
886   def initialize(dataPaths, command)
887     @dataPaths = dataPaths
888     @command = command
889   end
890   
891   def kind
892     :multiFileTimedBenchmark
893   end
894 end
895
896 # Benchmark that consists of one or more data files that should be loaded globally, followed
897 # by a command to run a short tick of the benchmark. The benchmark should be run for as many
898 # ticks as possible, for one quantum (quantum is 1000ms by default).
899 class ThroughputBenchmarkParameters
900   attr_reader :dataPaths, :setUpCommand, :command, :tearDownCommand, :doWarmup, :deterministic, :minimumIterations
901
902   def initialize(dataPaths, setUpCommand, command, tearDownCommand, doWarmup, deterministic, minimumIterations)
903     @dataPaths = dataPaths
904     @setUpCommand = setUpCommand
905     @command = command
906     @tearDownCommand = tearDownCommand
907     @doWarmup = doWarmup
908     @deterministic = deterministic
909     @minimumIterations = minimumIterations
910   end
911   
912   def kind
913     :throughputBenchmark
914   end
915 end
916
917 # Benchmark that can only run in DumpRenderTree or WebKitTestRunner, that has its own callback for reporting
918 # results. Other than that it's just like SingleFileTimedBenchmark.
919 class SingleFileTimedCallbackBenchmarkParameters
920   attr_reader :callbackDecl, :benchPath
921   
922   def initialize(callbackDecl, benchPath)
923     @callbackDecl = callbackDecl
924     @benchPath = benchPath
925   end
926   
927   def kind
928     :singleFileTimedCallbackBenchmark
929   end
930 end
931
932 def emitTimerFunctionCode(file)
933   case $timeMode
934   when :preciseTime
935     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
936     doublePuts($stderr,file,"   return preciseTime()*1000")
937     doublePuts($stderr,file,"}")
938   when :date
939     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
940     doublePuts($stderr,file,"   return Date.now()")
941     doublePuts($stderr,file,"}")
942   else
943     raise
944   end
945 end
946
947 def emitBenchRunCodeFile(name, plan, benchParams)
948   case plan.vm.vmType
949   when :jsc
950     Benchfile.create("bencher") {
951       | file |
952       emitTimerFunctionCode(file)
953       
954       if benchParams.kind == :multiFileTimedBenchmark
955         benchParams.dataPaths.each {
956           | path |
957           doublePuts($stderr,file,"load(#{path.inspect});")
958         }
959         doublePuts($stderr,file,"gc();")
960         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
961         doublePuts($stderr,file,"   var __before = __bencher_curTimeMS();")
962         $rerun.times {
963           doublePuts($stderr,file,"   #{benchParams.command.js}")
964         }
965         doublePuts($stderr,file,"   var __after = __bencher_curTimeMS();")
966         doublePuts($stderr,file,"   if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(__after-__before));");
967         doublePuts($stderr,file,"   gc();") unless plan.vm.shouldMeasureGC
968         doublePuts($stderr,file,"}")
969       elsif benchParams.kind == :throughputBenchmark
970         emitTimerFunctionCode(file)
971         benchParams.dataPaths.each {
972           | path |
973           doublePuts($stderr,file,"load(#{path.inspect});")
974         }
975         doublePuts($stderr,file,"#{benchParams.setUpCommand.js}")
976         if benchParams.doWarmup
977           warmup = $warmup
978         else
979           warmup = 0
980         end
981         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{warmup + $inner}; __bencher_index++) {")
982         doublePuts($stderr,file,"    var __before = __bencher_curTimeMS();")
983         doublePuts($stderr,file,"    var __after = __before;")
984         doublePuts($stderr,file,"    var __runs = 0;")
985         doublePuts($stderr,file,"    var __expected = #{$quantum};")
986         doublePuts($stderr,file,"    while (true) {")
987         $rerun.times {
988           doublePuts($stderr,file,"       #{benchParams.command.js}")
989         }
990         doublePuts($stderr,file,"       __runs++;")
991         doublePuts($stderr,file,"       __after = __bencher_curTimeMS();")
992         if benchParams.deterministic
993           doublePuts($stderr,file,"       if (true) {")
994         else
995           doublePuts($stderr,file,"       if (__after - __before >= __expected) {")
996         end
997         doublePuts($stderr,file,"           if (__runs >= #{benchParams.minimumIterations} || __bencher_index < #{warmup})")
998         doublePuts($stderr,file,"               break;")
999         doublePuts($stderr,file,"           __expected += #{$quantum}")
1000         doublePuts($stderr,file,"       }")
1001         doublePuts($stderr,file,"    }")
1002         doublePuts($stderr,file,"    if (__bencher_index >= #{warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{warmup}) + \": Time: \"+((__after-__before)/__runs));")
1003         doublePuts($stderr,file,"}")
1004         doublePuts($stderr,file,"#{benchParams.tearDownCommand.js}")
1005       elsif benchParams.kind == :singleFileCustomTimedBenchmark
1006         doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
1007         doublePuts($stderr,file,"   var __bencher_result = {value: null};")
1008         $rerun.times {
1009           doublePuts($stderr,file,"   run(__bencher_what, __bencher_result);")
1010         }
1011         doublePuts($stderr,file,"   if (__bencher_result.value == null)")
1012         doublePuts($stderr,file,"      throw new Error(\"Null result\");")
1013         doublePuts($stderr,file,"   return __bencher_result.value;")
1014         doublePuts($stderr,file,"}")
1015         $warmup.times {
1016           doublePuts($stderr,file,"__bencher_run(#{benchParams.benchPath.inspect})")
1017           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
1018         }
1019         $inner.times {
1020           | innerIndex |
1021           doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchParams.benchPath.inspect}));")
1022           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
1023         }
1024       elsif benchParams.kind == :singleFileCompileTimeBenchmark
1025         doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
1026         doublePuts($stderr,file,"   var __compileTimeBefore = totalCompileTime();")
1027         $rerun.times {
1028           doublePuts($stderr,file,"   run(__bencher_what);")
1029         }
1030         doublePuts($stderr,file,"   return totalCompileTime() - __compileTimeBefore;")
1031         doublePuts($stderr,file,"}")
1032         $warmup.times {
1033           doublePuts($stderr,file,"__bencher_run(#{benchParams.benchPath.inspect})")
1034           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
1035         }
1036         $inner.times {
1037           | innerIndex |
1038           doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchParams.benchPath.inspect}));")
1039           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
1040         }
1041       else
1042         raise unless benchParams.kind == :singleFileTimedBenchmark
1043         doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
1044         doublePuts($stderr,file,"   var __bencher_before = __bencher_curTimeMS();")
1045         $rerun.times {
1046           doublePuts($stderr,file,"   run(__bencher_what);")
1047         }
1048         doublePuts($stderr,file,"   var __bencher_after = __bencher_curTimeMS();")
1049         doublePuts($stderr,file,"   return __bencher_after - __bencher_before;")
1050         doublePuts($stderr,file,"}")
1051         $warmup.times {
1052           doublePuts($stderr,file,"__bencher_run(#{benchParams.benchPath.inspect})")
1053           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
1054         }
1055         $inner.times {
1056           | innerIndex |
1057           doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchParams.benchPath.inspect}));")
1058           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
1059         }
1060       end
1061     }
1062   when :dumpRenderTree, :webkitTestRunner
1063     case $timeMode
1064     when :preciseTime
1065       curTime = "(testRunner.preciseTime()*1000)"
1066     when :date
1067       curTime = "(Date.now())"
1068     else
1069       raise
1070     end
1071
1072     mainCode = Benchfile.create("bencher") {
1073       | file |
1074       doublePuts($stderr,file,"__bencher_count = 0;")
1075       doublePuts($stderr,file,"function __bencher_doNext(result) {")
1076       doublePuts($stderr,file,"    if (__bencher_count >= #{$warmup})")
1077       doublePuts($stderr,file,"        debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);")
1078       doublePuts($stderr,file,"    __bencher_count++;")
1079       doublePuts($stderr,file,"    if (__bencher_count < #{$inner+$warmup})")
1080       doublePuts($stderr,file,"        __bencher_runImpl(__bencher_doNext);")
1081       doublePuts($stderr,file,"    else")
1082       doublePuts($stderr,file,"        quit();")
1083       doublePuts($stderr,file,"}")
1084       doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);")
1085     }
1086     
1087     cssCode = Benchfile.create("bencher-css") {
1088       | file |
1089       doublePuts($stderr,file,".pass {\n    font-weight: bold;\n    color: green;\n}\n.fail {\n    font-weight: bold;\n    color: red;\n}\n\#console {\n    white-space: pre-wrap;\n    font-family: monospace;\n}")
1090     }
1091     
1092     preCode = Benchfile.create("bencher-pre") {
1093       | file |
1094       doublePuts($stderr,file,"if (window.testRunner) {")
1095       doublePuts($stderr,file,"    testRunner.dumpAsText(window.enablePixelTesting);")
1096       doublePuts($stderr,file,"    testRunner.waitUntilDone();")
1097       doublePuts($stderr,file,"}")
1098       doublePuts($stderr,file,"")
1099       doublePuts($stderr,file,"function debug(msg)")
1100       doublePuts($stderr,file,"{")
1101       doublePuts($stderr,file,"    var span = document.createElement(\"span\");")
1102       doublePuts($stderr,file,"    document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace")
1103       doublePuts($stderr,file,"    span.innerHTML = msg + '<br />';")
1104       doublePuts($stderr,file,"}")
1105       doublePuts($stderr,file,"")
1106       doublePuts($stderr,file,"function quit() {")
1107       doublePuts($stderr,file,"    testRunner.notifyDone();")
1108       doublePuts($stderr,file,"}")
1109       doublePuts($stderr,file,"")
1110       doublePuts($stderr,file,"__bencher_continuation=null;")
1111       doublePuts($stderr,file,"")
1112       doublePuts($stderr,file,"function reportResult(result) {")
1113       doublePuts($stderr,file,"    __bencher_continuation(result);")
1114       doublePuts($stderr,file,"}")
1115       doublePuts($stderr,file,"")
1116       doublePuts($stderr,file,"function currentTimeInMS(msg)")
1117       doublePuts($stderr,file,"{")
1118       doublePuts($stderr,file,"    return #{curTime};")
1119       doublePuts($stderr,file,"}")
1120       if benchParams.kind == :singleFileTimedCallbackBenchmark
1121         doublePuts($stderr,file,"")
1122         doublePuts($stderr,file,benchParams.callbackDecl)
1123       end
1124       doublePuts($stderr,file,"")
1125       doublePuts($stderr,file,"function __bencher_runImpl(continuation) {")
1126       doublePuts($stderr,file,"    function doit() {")
1127       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"\";")
1128       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";")
1129       doublePuts($stderr,file,"        var testFrame = document.getElementById(\"testframe\");")
1130       doublePuts($stderr,file,"        testFrame.contentDocument.open();")
1131       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");")
1132       if benchParams.kind == :throughputBenchmark or benchParams.kind == :multiFileTimedBenchmark
1133         benchParams.dataPaths.each {
1134           | path |
1135           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{path.inspect.inspect[1..-2]}></script>\");")
1136         }
1137       end
1138       if benchParams.kind == :throughputBenchmark
1139         if benchParams.doWarmup
1140           warmup = $warmup
1141         else
1142           warmup = 0
1143         end
1144         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">\");")
1145         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.setUpCommand.js}\");")
1146         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_before = #{curTime};\");")
1147         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_after = __bencher_before;\");")
1148         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_expected = #{$quantum};\");")
1149         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_runs = 0;\");")
1150         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"while (true) {\");")
1151         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    #{benchParams.command.js}\");")
1152         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_runs++;\");")
1153         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_after = #{curTime};\");")
1154         if benchParams.deterministic
1155           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (true) {\");")
1156         else
1157           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (__bencher_after - __bencher_before >= __bencher_expected) {\");")
1158         end
1159         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        if (__bencher_runs >= #{benchParams.minimumIterations} || window.parent.__bencher_count < #{warmup})\");")
1160         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"            break;\");")
1161         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        __bencher_expected += #{$quantum}\");")
1162         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    }\");")
1163         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"}\");")
1164         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.tearDownCommand.js}\");")
1165         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"window.parent.reportResult((__bencher_after - __bencher_before) / __bencher_runs);\");")
1166         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</script>\");")
1167       else
1168         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">var __bencher_before = #{curTime};</script>\");")
1169         if benchParams.kind == :multiFileTimedBenchmark
1170           doublePuts($stderr,file,"        testFrame.contentDocument.write(#{benchParams.command.html.inspect});")
1171         else
1172           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{benchParams.benchPath.inspect.inspect[1..-2]}></script>\");")
1173         end
1174         unless benchParams.kind == :singleFileTimedCallbackBenchmark
1175           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">window.parent.reportResult(#{curTime} - __bencher_before);</script>\");")
1176         end
1177       end
1178       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</body></html>\");")
1179       doublePuts($stderr,file,"        testFrame.contentDocument.close();")
1180       doublePuts($stderr,file,"    }")
1181       doublePuts($stderr,file,"    __bencher_continuation = continuation;")
1182       doublePuts($stderr,file,"    window.setTimeout(doit, 10);")
1183       doublePuts($stderr,file,"}")
1184     }
1185
1186     Benchfile.create(["bencher-htmldoc",".html"]) {
1187       | file |
1188       doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>")
1189     }
1190   else
1191     raise
1192   end
1193 end
1194
1195 def emitBenchRunCode(name, plan, benchParams)
1196   plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchParams), plan)
1197 end
1198
1199 class FileCreator
1200   def initialize(filename)
1201     @filename = filename
1202     @state = :empty
1203   end
1204   
1205   def puts(text)
1206     $script.print "echo #{Shellwords.shellescape(text)}"
1207     if @state == :empty
1208       $script.print " > "
1209       @state = :nonEmpty
1210     else
1211       $script.print " >> "
1212     end
1213     $script.puts "#{Shellwords.shellescape(@filename)}"
1214   end
1215   
1216   def close
1217     if @state == :empty
1218       $script.puts "rm -f #{Shellwords.shellescape(text)}"
1219       $script.puts "touch #{Shellwords.shellescape(text)}"
1220     end
1221   end
1222   
1223   def self.open(filename)
1224     outp = FileCreator.new(filename)
1225     yield outp
1226     outp.close
1227   end
1228 end
1229
1230 def emitSelfContainedBenchRunCode(name, plan, targetFile, configFile, benchmark)
1231   FileCreator.open(configFile) {
1232     | outp |
1233     outp.puts "__bencher_message = \"#{name}: #{plan.vm}: #{plan.iteration}: \";"
1234     outp.puts "__bencher_warmup = #{$warmup};"
1235     outp.puts "__bencher_inner = #{$inner};"
1236     outp.puts "__bencher_benchmark = #{benchmark.to_json};"
1237     case $timeMode
1238     when :preciseTime
1239       outp.puts "__bencher_curTime = (function(){ return testRunner.preciseTime() * 1000; });"
1240     when :date
1241       outp.puts "__bencher_curTime = (function(){ return Date.now(); });"
1242     else
1243       raise
1244     end
1245   }
1246   
1247   plan.vm.emitRunCode(targetFile, plan)
1248 end
1249
1250 def planForDescription(string, plans, benchFullname, vmName, iteration)
1251   raise "Unexpected benchmark full name: #{benchFullname.inspect}, string: #{string.inspect}" unless benchFullname =~ /\//
1252   suiteName = $~.pre_match
1253   return nil if suiteName == "WARMUP"
1254   benchName = $~.post_match
1255   result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration}
1256   raise "Unexpected result dimensions: #{result.inspect}, string: #{string.inspect}" unless result.size == 1
1257   result[0]
1258 end
1259
1260 class ParsedResult
1261   attr_reader :plan, :innerIndex, :time, :result
1262   
1263   def initialize(plan, innerIndex, time)
1264     @plan = plan
1265     @innerIndex = innerIndex
1266     if time == :crashed
1267       @result = :error
1268     else
1269       @time = time
1270       @result = :success
1271     end
1272     
1273     raise unless @plan.is_a? BenchPlan
1274     raise unless @innerIndex.is_a? Integer
1275     raise unless @time.is_a? Numeric or @result == :error
1276   end
1277   
1278   def benchmark
1279     plan.benchmark
1280   end
1281   
1282   def suite
1283     plan.suite
1284   end
1285   
1286   def vm
1287     plan.vm
1288   end
1289   
1290   def outerIndex
1291     plan.iteration
1292   end
1293   
1294   def self.create(plan, innerIndex, time)
1295     if plan
1296       ParsedResult.new(plan, innerIndex, time)
1297     else
1298       nil
1299     end
1300   end
1301   
1302   def self.parse(plans, string)
1303     if string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): Time: /
1304       benchFullname = $1
1305       vmName = $2
1306       outerIndex = $3.to_i
1307       innerIndex = $4.to_i
1308       time = $~.post_match.to_f
1309       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, time)
1310     elsif string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): CRASHED/
1311       benchFullname = $1
1312       vmName = $2
1313       outerIndex = $3.to_i
1314       innerIndex = $4.to_i
1315       time = $~.post_match.to_f
1316       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, :crashed)
1317     else
1318       nil
1319     end
1320   end
1321 end
1322
1323 class VM
1324   @@extraEnvSet = {}
1325   
1326   def initialize(origPath, name, nameKind, svnRevision)
1327     @origPath = origPath.to_s
1328     @path = origPath.to_s
1329     @name = name
1330     @nameKind = nameKind
1331     @extraEnv = {}
1332     
1333     if $forceVMKind
1334       @vmType = $forceVMKind
1335     else
1336       if @origPath =~ /DumpRenderTree$/
1337         @vmType = :dumpRenderTree
1338       elsif @origPath =~ /WebKitTestRunner$/
1339         @vmType = :webkitTestRunner
1340       else
1341         @vmType = :jsc
1342       end
1343     end
1344     
1345     @svnRevision = svnRevision
1346     
1347     # Try to detect information about the VM.
1348     if path =~ /\/WebKitBuild\/(Release|Debug)+\/([a-zA-Z]+)$/
1349       @checkoutPath = $~.pre_match
1350       # FIXME: Use some variant of this: 
1351       # <bdash>   def retrieve_revision
1352       # <bdash>     `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i
1353       # <bdash>   end
1354       unless @svnRevision
1355         begin
1356           Dir.chdir(@checkoutPath) {
1357             $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
1358             IO.popen("svn info", "r") {
1359               | inp |
1360               inp.each_line {
1361                 | line |
1362                 if line =~ /Revision: ([0-9]+)/
1363                   @svnRevision = $1
1364                 end
1365               }
1366             }
1367           }
1368           unless @svnRevision
1369             $stderr.puts "Warning: running svn info for #{name} silently failed."
1370           end
1371         rescue => e
1372           # Failed to detect svn revision.
1373           $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
1374         end
1375       end
1376     else
1377       $stderr.puts "Warning: could not identify checkout location for #{name}"
1378     end
1379     
1380     if @path =~ /\/Release\/([a-zA-Z]+)$/
1381       @libPath, @relativeBinPath = [$~.pre_match+"/Release"], "./#{$1}"
1382     elsif @path =~ /\/Debug\/([a-zA-Z]+)$/
1383       @libPath, @relativeBinPath = [$~.pre_match+"/Debug"], "./#{$1}"
1384     elsif @path =~ /\/Release\/bin(64|32|)\/([\.a-zA-Z]+)$/
1385       @libPath, @relativeBinPath = [$~.pre_match+"/Release/lib#{$1}"], "./#{$2}"
1386     elsif @path =~ /\/Debug\/bin(64|32|)\/([\.a-zA-Z]+)$/
1387       @libPath, @relativeBinPath = [$~.pre_match+"/Debug/lib#{$1}"], "./#{$2}"
1388     elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/
1389       @libPath = [$~.pre_match + "/Contents/Resources", $~.pre_match + "/Contents/Frameworks"]
1390     elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/
1391       @libPath, @relativeBinPath = [$~.pre_match], $&[1..-1]
1392     elsif @path =~ /(DumpRenderTree|webkitTestRunner|jsc)$/
1393       @libPath, @relativeBinPath = [$~.pre_match+"/"], "./#{$1}"
1394     end
1395     @libPath += $dependencies
1396   end
1397   
1398   def canCopyIntoBenchPath
1399     if @libPath and @relativeBinPath
1400       true
1401     else
1402       false
1403     end
1404   end
1405   
1406   def addExtraEnv(key, val)
1407     @extraEnv[key] = val
1408     @@extraEnvSet[key] = true
1409   end
1410   
1411   def copyIntoBenchPath
1412     raise unless canCopyIntoBenchPath
1413     basename, filename = Benchfile.uniqueFilename("vm")
1414     raise unless Dir.mkdir(filename)
1415     @libPath.each {
1416       | libPathPart |
1417       cmd = "cp -a #{Shellwords.shellescape(libPathPart)}/* #{Shellwords.shellescape(filename.to_s)}"
1418       $stderr.puts ">> #{cmd}" if $verbosity>=2
1419       raise unless system(cmd)
1420     }
1421     @path = "#{basename}/#{@relativeBinPath}"
1422     @libPath = [basename]
1423   end
1424   
1425   def to_s
1426     @name
1427   end
1428   
1429   def name
1430     @name
1431   end
1432   
1433   def shouldMeasureGC
1434     $measureGC == true or ($measureGC == name)
1435   end
1436   
1437   def origPath
1438     @origPath
1439   end
1440   
1441   def path
1442     @path
1443   end
1444   
1445   def nameKind
1446     @nameKind
1447   end
1448   
1449   def vmType
1450     @vmType
1451   end
1452   
1453   def checkoutPath
1454     @checkoutPath
1455   end
1456   
1457   def svnRevision
1458     @svnRevision
1459   end
1460   
1461   def extraEnv
1462     @extraEnv
1463   end
1464   
1465   def printFunction
1466     case @vmType
1467     when :jsc
1468       "print"
1469     when :dumpRenderTree, :webkitTestRunner
1470       "debug"
1471     else
1472       raise @vmType
1473     end
1474   end
1475   
1476   def emitRunCode(fileToRun, plan)
1477     myLibPath = @libPath
1478     myLibPath = [] unless myLibPath
1479     @@extraEnvSet.keys.each {
1480       | key |
1481       $script.puts "unset #{Shellwords.shellescape(key)}"
1482     }
1483     $script.puts "export JSC_useDollarVM=true" if @vmType == :jsc
1484     $script.puts "export DYLD_LIBRARY_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1485     $script.puts "export DYLD_FRAMEWORK_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1486     $script.puts "export LD_LIBRARY_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1487
1488     unless myLibPath.empty?
1489         primaryLibPath = myLibPath.first
1490         $script.puts "export TEST_RUNNER_TEST_PLUGIN_PATH=#{Shellwords.shellescape(Pathname.new(primaryLibPath).join('plugins').to_s)}"
1491         $script.puts "export TEST_RUNNER_INJECTED_BUNDLE_FILENAME=#{Shellwords.shellescape(Pathname.new(primaryLibPath).join('libTestRunnerInjectedBundle.so').to_s)}"
1492     end
1493
1494     @extraEnv.each_pair {
1495       | key, val |
1496       $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1497     }
1498     plan.environment.each_pair {
1499         | key, val |
1500         $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1501     }
1502     $script.puts "#{path} #{fileToRun} 2>&1 || {"
1503     $script.puts "    echo " + Shellwords.shellescape("#{name} failed to run!") + " 1>&2"
1504     $inner.times {
1505       | iteration |
1506       $script.puts "    echo " + Shellwords.shellescape("#{plan.prefix}: #{iteration}: CRASHED")
1507     }
1508     $script.puts "}"
1509     plan.environment.keys.each {
1510         | key |
1511         $script.puts "unset #{Shellwords.shellescape(key)}"
1512     }
1513   end
1514 end
1515
1516 class StatsAccumulator
1517   def initialize
1518     @stats = []
1519     ($outer*$inner).times {
1520       @stats << Stats.new
1521     }
1522   end
1523   
1524   def statsForIteration(outerIteration, innerIteration)
1525     @stats[outerIteration*$inner + innerIteration]
1526   end
1527   
1528   def stats
1529     result = Stats.new
1530     @stats.each {
1531       | stat |
1532       if stat.ok?
1533         result.add(yield stat)
1534       else
1535         result.add(nil)
1536       end
1537     }
1538     result
1539   end
1540   
1541   def geometricMeanStats
1542     stats {
1543       | stat |
1544       stat.geometricMean
1545     }
1546   end
1547   
1548   def arithmeticMeanStats
1549     stats {
1550       | stat |
1551       stat.arithmeticMean
1552     }
1553   end
1554 end
1555
1556 module Benchmark
1557   attr_accessor :benchmarkSuite
1558   attr_reader :name
1559   
1560   def fullname
1561     benchmarkSuite.name + "/" + name
1562   end
1563   
1564   def to_s
1565     fullname
1566   end
1567   
1568   def weight
1569     1
1570   end
1571   
1572   def weightString
1573     raise unless weight.is_a? Fixnum
1574     raise unless weight >= 1
1575     if weight == 1
1576       ""
1577     else
1578       "x#{weight} "
1579     end
1580   end
1581   
1582   def environment
1583     {}
1584   end
1585 end
1586
1587 class SunSpiderBenchmark
1588   include Benchmark
1589   
1590   def initialize(name)
1591     @name = name
1592   end
1593   
1594   def emitRunCode(plan)
1595     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js")))
1596   end
1597 end
1598
1599 class LongSpiderBenchmark
1600   include Benchmark
1601   
1602   def initialize(name)
1603     @name = name
1604   end
1605   
1606   def emitRunCode(plan)
1607     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("LongSpider-#{@name}", "#{LONGSPIDER_PATH}/#{@name}.js")))
1608   end
1609 end
1610
1611 class V8Benchmark
1612   include Benchmark
1613   
1614   def initialize(name)
1615     @name = name
1616   end
1617   
1618   def emitRunCode(plan)
1619     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js")))
1620   end
1621 end
1622
1623 class V8RealBenchmark
1624   include Benchmark
1625   
1626   attr_reader :v8SuiteName
1627   
1628   def initialize(v8SuiteName, name, weight, minimumIterations)
1629     @v8SuiteName = v8SuiteName
1630     @name = name
1631     @weight = weight
1632     @minimumIterations = minimumIterations
1633   end
1634   
1635   def weight
1636     @weight
1637   end
1638   
1639   def emitRunCode(plan)
1640     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(["base", @v8SuiteName, "jsc-#{@name}"].collect{|v| ensureFile("V8Real-#{v}", "#{V8_REAL_PATH}/#{v}.js")}, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, false, @minimumIterations))
1641   end
1642 end
1643
1644 class OctaneBenchmark
1645   include Benchmark
1646   
1647   def initialize(files, name, weight, doWarmup, deterministic, minimumIterations)
1648     @files = files
1649     @name = name
1650     @weight = weight
1651     @doWarmup = doWarmup
1652     @deterministic = deterministic
1653     @minimumIterations = minimumIterations
1654   end
1655   
1656   def weight
1657     @weight
1658   end
1659   
1660   def emitRunCode(plan)
1661     files = []
1662     files += (["base"] + @files).collect {
1663       | v |
1664       ensureFile("Octane-#{v}", "#{OCTANE_PATH}/#{v}.js")
1665     }
1666     files += ["jsc-#{@name}"].collect {
1667       | v |
1668       ensureFile("Octane-#{v}", "#{OCTANE_WRAPPER_PATH}/#{v}.js")
1669     }
1670     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(files, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1671   end
1672 end
1673
1674 class CustomTimedBenchmark
1675   include Benchmark
1676   
1677   def initialize(name, fullPath)
1678     @name = name
1679     @fullPath = fullPath
1680   end
1681   
1682   def emitRunCode(plan)
1683     emitBenchRunCode(fullname, plan, SingleFileCustomTimedBenchmarkParameters.new(ensureFile("CustomTimed-#{@name}", @fullPath)))
1684   end
1685 end
1686
1687 class CompileTimeBenchmark
1688   include Benchmark
1689   
1690   def initialize(name, fullPath)
1691     @name = name
1692     @fullPath = fullPath
1693   end
1694   
1695   def emitRunCode(plan)
1696     emitBenchRunCode(fullname, plan, SingleFileCompileTimeBenchmarkParameters.new(ensureFile("CompileTime-#{@name}", @fullPath)))
1697   end
1698   
1699   def environment
1700     {"JSC_useConcurrentJIT" => "false", "JSC_reportTotalCompileTimes" => "true"}
1701   end
1702 end
1703
1704 class KrakenBenchmark
1705   include Benchmark
1706   
1707   def initialize(name)
1708     @name = name
1709   end
1710   
1711   def emitRunCode(plan)
1712     emitBenchRunCode(fullname, plan, MultiFileTimedBenchmarkParameters.new([ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js")], loadCommandForFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js")))
1713   end
1714 end
1715
1716 class JSBenchBenchmark
1717   include Benchmark
1718   
1719   attr_reader :jsBenchMode
1720   
1721   def initialize(name, jsBenchMode)
1722     @name = name
1723     @jsBenchMode = jsBenchMode
1724   end
1725   
1726   def emitRunCode(plan)
1727     callbackDecl  = "function JSBNG_handleResult(result) {\n"
1728     callbackDecl += "    if (result.error) {\n"
1729     callbackDecl += "        console.log(\"Did not run benchmark correctly!\");\n"
1730     callbackDecl += "        quit();\n"
1731     callbackDecl += "    }\n"
1732     callbackDecl += "    reportResult(result.time);\n"
1733     callbackDecl += "}\n"
1734     emitBenchRunCode(fullname, plan, SingleFileTimedCallbackBenchmarkParameters.new(callbackDecl, ensureFile("JSBench-#{@name}", "#{JSBENCH_PATH}/#{@name}/#{@jsBenchMode}.js")))
1735   end
1736 end
1737
1738 class TailBenchBenchmark
1739   include Benchmark
1740
1741   def initialize(name)
1742     @name = name
1743   end
1744
1745   def emitRunCode(plan)
1746     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("TailBench-#{@name}", "#{TAILBENCH_PATH}/#{@name}.js")))
1747   end
1748 end
1749
1750 class BigIntBenchBenchmark
1751   include Benchmark
1752
1753   def initialize(name)
1754     @name = name
1755   end
1756
1757   def emitRunCode(plan)
1758     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("BigIntBench-#{@name}", "#{BIGINTBENCH_PATH}/#{@name}.js")))
1759   end
1760
1761   def environment
1762     {"JSC_useBigInt" => "true"}
1763   end
1764 end
1765
1766
1767 class MicrobenchmarksBenchmark
1768   include Benchmark
1769   
1770   def initialize(name)
1771     @name = name
1772   end
1773   
1774   def emitRunCode(plan)
1775     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("Microbenchmarks-#{@name}", "#{MICROBENCHMARKS_PATH}/#{@name}.js")))
1776   end
1777 end
1778
1779 class AsmBenchBenchmark
1780   include Benchmark
1781   
1782   def initialize(name)
1783     @name = name
1784   end
1785   
1786   def emitRunCode(plan)
1787     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("AsmBench-#{@name}", "#{ASMBENCH_PATH}/#{@name}.js")))
1788   end
1789 end
1790
1791 class SixSpeedBenchmark
1792   include Benchmark
1793
1794   def initialize(name, path, iterations)
1795     @name = name
1796     @path = path
1797     @iterations = iterations
1798   end
1799
1800   def emitRunCode(plan)
1801     emitBenchRunCode(fullname, plan, MultiFileTimedBenchmarkParameters.new([ensureFile("SixSpeed-#{@name}-wrapper", "#{SIXSPEED_WRAPPER_PATH}/wrapper.js"), ensureFile("SixSpeed-#{@name}", "#{SIXSPEED_PATH}/#{@path}")], simpleCommand("jscRun(#{@iterations});")))
1802   end
1803 end
1804
1805
1806 class CompressionBenchBenchmark
1807   include Benchmark
1808   
1809   def initialize(files, name, model)
1810     @files = files
1811     @name = name;
1812     @name = name + "-" + model if !model.empty?
1813     @name = @name.gsub(" ", "-").downcase
1814     @scriptName = name
1815     @weight = 1
1816     @doWarmup = true
1817     @deterministic = true
1818     @minimumIterations = 1
1819     @model = model
1820   end
1821   
1822   def weight
1823     @weight
1824   end
1825   
1826   def emitRunCode(plan)
1827     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new((["base"] + @files + ["jsc-#{@scriptName}"]).collect{|v| ensureFile("Compression-#{v}", "#{COMPRESSIONBENCH_PATH}/#{v}.js")}, simpleCommand("jscSetUp('#{@model}');"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1828   end
1829 end
1830
1831 class DSPJSFiltrrBenchmark
1832   include Benchmark
1833   
1834   def initialize(name, filterKey)
1835     @name = name
1836     @filterKey = filterKey
1837   end
1838   
1839   def emitRunCode(plan)
1840     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr.js")
1841     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr_back.jpg")
1842     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-jquery.min.js")
1843     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-bencher.html")
1844     emitSelfContainedBenchRunCode(fullname, plan, "filtrr-bencher.html", "bencher-config.js", @filterKey)
1845   end
1846 end
1847
1848 class DSPJSVP8Benchmark
1849   include Benchmark
1850   
1851   def initialize
1852     @name = "route9-vp8"
1853   end
1854   
1855   def weight
1856     5
1857   end
1858   
1859   def emitRunCode(plan)
1860     ensureBenchmarkFiles(DSPJS_ROUTE9_PATH)
1861     emitSelfContainedBenchRunCode(fullname, plan, "route9-bencher.html", "bencher-config.js", "")
1862   end
1863 end
1864
1865 class DSPStarfieldBenchmark
1866   include Benchmark
1867
1868   def initialize
1869     @name = "starfield"
1870   end
1871   
1872   def weight
1873     5
1874   end
1875
1876   def emitRunCode(plan)
1877     ensureBenchmarkFiles(DSPJS_STARFIELD_PATH)
1878     emitSelfContainedBenchRunCode(fullname, plan, "starfield-bencher.html", "bencher-config.js", "")
1879   end
1880 end
1881
1882 class DSPJSJSLinuxBenchmark
1883   include Benchmark
1884   def initialize
1885     @name = "bellard-jslinux"
1886   end
1887
1888   def weight
1889     5
1890   end
1891
1892   def emitRunCode(plan)
1893     ensureBenchmarkFiles(DSPJS_JSLINUX_PATH)
1894     emitSelfContainedBenchRunCode(fullname, plan, "jslinux-bencher.html", "bencher-config.js", "")
1895   end
1896 end
1897
1898 class DSPJSQuake3Benchmark
1899   include Benchmark
1900
1901   def initialize
1902     @name = "zynaps-quake3"
1903   end
1904
1905   def weight
1906     5
1907   end
1908
1909   def emitRunCode(plan)
1910     ensureBenchmarkFiles(DSPJS_QUAKE3_PATH)
1911     emitSelfContainedBenchRunCode(fullname, plan, "quake-bencher.html", "bencher-config.js", "")
1912   end
1913 end
1914
1915 class DSPJSMandelbrotBenchmark
1916   include Benchmark
1917
1918   def initialize
1919     @name = "zynaps-mandelbrot"
1920   end
1921
1922   def weight
1923     5
1924   end
1925
1926   def emitRunCode(plan)
1927     ensureBenchmarkFiles(DSPJS_MANDELBROT_PATH)
1928     emitSelfContainedBenchRunCode(fullname, plan, "mandelbrot-bencher.html", "bencher-config.js", "")
1929   end
1930 end
1931
1932 class DSPJSAmmoJSASMBenchmark
1933   include Benchmark
1934
1935   def initialize
1936     @name = "ammojs-asm-js"
1937   end
1938
1939   def weight
1940     5
1941   end
1942
1943   def emitRunCode(plan)
1944     ensureBenchmarkFiles(DSPJS_AMMOJS_ASMJS_PATH)
1945     emitSelfContainedBenchRunCode(fullname, plan, "ammo-asmjs-bencher.html", "bencher-config.js", "")
1946   end
1947 end
1948
1949 class DSPJSAmmoJSRegularBenchmark
1950   include Benchmark
1951
1952   def initialize
1953     @name = "ammojs-regular-js"
1954   end
1955
1956   def weight
1957     5
1958   end
1959
1960   def emitRunCode(plan)
1961     ensureBenchmarkFiles(DSPJS_AMMOJS_REGULAR_PATH)
1962     emitSelfContainedBenchRunCode(fullname, plan, "ammo-regular-bencher.html", "bencher-config.js", "")
1963   end
1964 end
1965
1966 class BrowsermarkJSBenchmark
1967   include Benchmark
1968     
1969   def initialize(name)
1970     @name = name
1971   end
1972   
1973   def emitRunCode(plan)
1974     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new([ensureFile(name, "#{BROWSERMARK_JS_PATH}/#{name}/test.js"), ensureFile("browsermark-bencher", "#{BROWSERMARK_JS_PATH}/browsermark-bencher.js")], simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, 32))
1975   end
1976 end
1977
1978 class BrowsermarkDOMBenchmark
1979   include Benchmark
1980     
1981   def initialize(name)
1982     @name = name
1983   end
1984   
1985   def emitRunCode(plan)
1986     ensureBenchmarkFiles(BROWSERMARK_PATH)
1987     emitSelfContainedBenchRunCode(fullname, plan, "tests/benchmarks/dom/#{name}/index.html", "bencher-config.js", name)
1988   end
1989 end
1990
1991 class BenchmarkSuite
1992   def initialize(name, preferredMean, decimalShift)
1993     @name = name
1994     @preferredMean = preferredMean
1995     @benchmarks = []
1996     @subSuites = []
1997     @decimalShift = decimalShift
1998   end
1999   
2000   def name
2001     @name
2002   end
2003   
2004   def to_s
2005     @name
2006   end
2007   
2008   def decimalShift
2009     @decimalShift
2010   end
2011   
2012   def addIgnoringPattern(benchmark)
2013     benchmark.benchmarkSuite = self
2014     @benchmarks << benchmark
2015   end
2016   
2017   def add(benchmark)
2018     if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern
2019         addIgnoringPattern(benchmark)
2020     end
2021   end
2022   
2023   def addSubSuite(subSuite)
2024     @subSuites << subSuite
2025   end
2026   
2027   def benchmarks
2028     @benchmarks
2029   end
2030   
2031   def benchmarkForName(name)
2032     result = @benchmarks.select{|v| v.name == name}
2033     raise unless result.length == 1
2034     result[0]
2035   end
2036   
2037   def hasBenchmark(benchmark)
2038     array = @benchmarks.select{|v| v == benchmark}
2039     raise unless array.length == 1 or array.length == 0
2040     array.length == 1
2041   end
2042   
2043   def subSuites
2044     @subSuites
2045   end
2046   
2047   def suites
2048     [self] + @subSuites
2049   end
2050   
2051   def suitesWithBenchmark(benchmark)
2052     result = [self]
2053     @subSuites.each {
2054       | subSuite |
2055       if subSuite.hasBenchmark(benchmark)
2056         result << subSuite
2057       end
2058     }
2059     result
2060   end
2061   
2062   def empty?
2063     @benchmarks.empty?
2064   end
2065   
2066   def retain_if
2067     @benchmarks.delete_if {
2068       | benchmark |
2069       not yield benchmark
2070     }
2071   end
2072   
2073   def preferredMean
2074     @preferredMean
2075   end
2076   
2077   def computeMean(stat)
2078     if stat.ok?
2079       (stat.send @preferredMean) * (10 ** decimalShift)
2080     else
2081       nil
2082     end
2083   end
2084 end
2085
2086 class BenchRunPlan
2087   def initialize(benchmark, vm, iteration)
2088     @benchmark = benchmark
2089     @vm = vm
2090     @iteration = iteration
2091     @environment = {}
2092     if $environment.has_key?(vm.name)
2093       if $environment[vm.name].has_key?(benchmark.benchmarkSuite.name)
2094         if $environment[vm.name][benchmark.benchmarkSuite.name].has_key?(benchmark.name)
2095           @environment = $environment[vm.name][benchmark.benchmarkSuite.name][benchmark.name]
2096         end
2097       end
2098     end
2099   end
2100   
2101   def benchmark
2102     @benchmark
2103   end
2104   
2105   def suite
2106     @benchmark.benchmarkSuite
2107   end
2108   
2109   def vm
2110     @vm
2111   end
2112   
2113   def iteration
2114     @iteration
2115   end
2116  
2117   def environment
2118     result = @environment.clone
2119     benchmark.environment.each_pair {
2120       | key, value |
2121       result[key] = value
2122     }
2123     result
2124   end
2125  
2126   def prefix
2127     "#{@benchmark.fullname}: #{vm.name}: #{iteration}"
2128   end
2129   
2130   def emitRunCode
2131     @benchmark.emitRunCode(self)
2132   end
2133   
2134   def to_s
2135     benchmark.to_s + "/" + vm.to_s
2136   end
2137 end
2138
2139 class BenchmarkOnVM
2140   def initialize(benchmark, suiteOnVM, subSuitesOnVM)
2141     @benchmark = benchmark
2142     @suiteOnVM = suiteOnVM
2143     @subSuitesOnVM = subSuitesOnVM
2144     @stats = Stats.new
2145   end
2146   
2147   def to_s
2148     "#{@benchmark} on #{@suiteOnVM.vm}"
2149   end
2150   
2151   def benchmark
2152     @benchmark
2153   end
2154   
2155   def vm
2156     @suiteOnVM.vm
2157   end
2158   
2159   def vmStats
2160     @suiteOnVM.vmStats
2161   end
2162   
2163   def suite
2164     @benchmark.benchmarkSuite
2165   end
2166   
2167   def suiteOnVM
2168     @suiteOnVM
2169   end
2170   
2171   def subSuitesOnVM
2172     @subSuitesOnVM
2173   end
2174   
2175   def stats
2176     @stats
2177   end
2178   
2179   def parseResult(result)
2180     raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm
2181     raise unless result.benchmark == @benchmark
2182     @stats.add(result.time)
2183   end
2184 end
2185
2186 class NamedStatsAccumulator < StatsAccumulator
2187   def initialize(name)
2188     super()
2189     @name = name
2190   end
2191   
2192   def reportingName
2193     @name
2194   end
2195 end
2196
2197 class SuiteOnVM < StatsAccumulator
2198   def initialize(vm, vmStats, suite)
2199     super()
2200     @vm = vm
2201     @vmStats = vmStats
2202     @suite = suite
2203     
2204     raise unless @vm.is_a? VM
2205     raise unless @vmStats.is_a? StatsAccumulator
2206     raise unless @suite.is_a? BenchmarkSuite
2207   end
2208   
2209   def to_s
2210     "#{@suite} on #{@vm}"
2211   end
2212   
2213   def suite
2214     @suite
2215   end
2216   
2217   def vm
2218     @vm
2219   end
2220
2221   def reportingName
2222     @vm.name
2223   end
2224   
2225   def vmStats
2226     raise unless @vmStats
2227     @vmStats
2228   end
2229 end
2230
2231 class SubSuiteOnVM < StatsAccumulator
2232   def initialize(vm, suite)
2233     super()
2234     @vm = vm
2235     @suite = suite
2236     raise unless @vm.is_a? VM
2237     raise unless @suite.is_a? BenchmarkSuite
2238   end
2239   
2240   def to_s
2241     "#{@suite} on #{@vm}"
2242   end
2243   
2244   def suite
2245     @suite
2246   end
2247   
2248   def vm
2249     @vm
2250   end
2251   
2252   def reportingName
2253     @vm.name
2254   end
2255 end
2256
2257 class BenchPlan
2258   def initialize(benchmarkOnVM, iteration)
2259     @benchmarkOnVM = benchmarkOnVM
2260     @iteration = iteration
2261   end
2262   
2263   def to_s
2264     "#{@benchmarkOnVM} \##{@iteration+1}"
2265   end
2266   
2267   def benchmarkOnVM
2268     @benchmarkOnVM
2269   end
2270   
2271   def benchmark
2272     @benchmarkOnVM.benchmark
2273   end
2274   
2275   def suite
2276     @benchmarkOnVM.suite
2277   end
2278   
2279   def vm
2280     @benchmarkOnVM.vm
2281   end
2282   
2283   def iteration
2284     @iteration
2285   end
2286   
2287   def parseResult(result)
2288     raise unless result.plan == self
2289     @benchmarkOnVM.parseResult(result)
2290     benchmark.weight.times {
2291       @benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time)
2292       @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time)
2293       @benchmarkOnVM.subSuitesOnVM.each {
2294         | subSuite |
2295         subSuite.statsForIteration(@iteration, result.innerIndex).add(result.time)
2296       }
2297     }
2298   end
2299 end
2300
2301 def lpad(str,chars)
2302   if str.length>chars
2303     str
2304   else
2305     "%#{chars}s"%(str)
2306   end
2307 end
2308
2309 def rpad(str,chars)
2310   while str.length < chars
2311     str+=" "
2312   end
2313   str
2314 end
2315
2316 def center(str,chars)
2317   while str.length<chars
2318     str+=" "
2319     if str.length<chars
2320       str=" "+str
2321     end
2322   end
2323   str
2324 end
2325
2326 def statsToStr(stats, decimalShift)
2327   if stats.error?
2328     lpad(center("ERROR", 10+10+2), 12+10+2)
2329   elsif $inner*$outer == 1
2330     string = numToStr(stats.mean, decimalShift)
2331     raise unless string =~ /\./
2332     left = $~.pre_match
2333     right = $~.post_match
2334     lpad(left, 13 - decimalShift) + "." + rpad(right, 10 + decimalShift)
2335   else
2336     lpad(numToStr(stats.mean, decimalShift), 12) + "+-" + rpad(numToStr(stats.confInt, decimalShift), 10)
2337   end
2338 end
2339
2340 def plural(num)
2341   if num == 1
2342     ""
2343   else
2344     "s"
2345   end
2346 end
2347
2348 def wrap(str, columns)
2349   array = str.split
2350   result = ""
2351   curLine = array.shift
2352   array.each {
2353     | curStr |
2354     if (curLine + " " + curStr).size > columns
2355       result += curLine + "\n"
2356       curLine = curStr
2357     else
2358       curLine += " " + curStr
2359     end
2360   }
2361   result + curLine + "\n"
2362 end
2363   
2364 def runAndGetResults
2365   results = nil
2366   Dir.chdir(BENCH_DATA_PATH) {
2367     $stderr.puts ">> sh ./runscript" if $verbosity >= 2
2368     raise "Script did not complete correctly: #{$?}" unless system("sh ./runscript > runlog")
2369     results = IO::read("runlog")
2370   }
2371   raise unless results
2372   results
2373 end
2374
2375 def parseAndDisplayResults(results)
2376   vmStatses = []
2377   $vms.each {
2378     | vm |
2379     vmStatses << NamedStatsAccumulator.new(vm.name)
2380   }
2381   
2382   suitesOnVMs = []
2383   suitesOnVMsForSuite = {}
2384   subSuitesOnVMsForSubSuite = {}
2385   $suites.each {
2386     | suite |
2387     suitesOnVMsForSuite[suite] = []
2388     suite.subSuites.each {
2389       | subSuite |
2390       subSuitesOnVMsForSubSuite[subSuite] = []
2391     }
2392   }
2393   suitesOnVMsForVM = {}
2394   $vms.each {
2395     | vm |
2396     suitesOnVMsForVM[vm] = []
2397   }
2398   
2399   benchmarksOnVMs = []
2400   benchmarksOnVMsForBenchmark = {}
2401   $benchmarks.each {
2402     | benchmark |
2403     benchmarksOnVMsForBenchmark[benchmark] = []
2404   }
2405   
2406   $vms.each_with_index {
2407     | vm, vmIndex |
2408     vmStats = vmStatses[vmIndex]
2409     $suites.each {
2410       | suite |
2411       suiteOnVM = SuiteOnVM.new(vm, vmStats, suite)
2412       subSuitesOnVM = suite.subSuites.map {
2413         | subSuite |
2414         result = SubSuiteOnVM.new(vm, subSuite)
2415         subSuitesOnVMsForSubSuite[subSuite] << result
2416         result
2417       }
2418       suitesOnVMs << suiteOnVM
2419       suitesOnVMsForSuite[suite] << suiteOnVM
2420       suitesOnVMsForVM[vm] << suiteOnVM
2421       suite.benchmarks.each {
2422         | benchmark |
2423         subSuitesOnVMForThisBenchmark = []
2424         subSuitesOnVM.each {
2425           | subSuiteOnVM |
2426           if subSuiteOnVM.suite.hasBenchmark(benchmark)
2427             subSuitesOnVMForThisBenchmark << subSuiteOnVM
2428           end
2429         }
2430         benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM, subSuitesOnVMForThisBenchmark)
2431         benchmarksOnVMs << benchmarkOnVM
2432         benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
2433       }
2434     }
2435   }
2436   
2437   plans = []
2438   benchmarksOnVMs.each {
2439     | benchmarkOnVM |
2440     $outer.times {
2441       | iteration |
2442       plans << BenchPlan.new(benchmarkOnVM, iteration)
2443     }
2444   }
2445
2446   hostname = nil
2447   hwmodel = nil
2448   results.each_line {
2449     | line |
2450     line.chomp!
2451     if line =~ /HOSTNAME:([^.]+)/
2452       hostname = $1
2453     elsif line =~ /HARDWARE:hw\.model: /
2454       hwmodel = $~.post_match.chomp
2455     else
2456       result = ParsedResult.parse(plans, line)
2457       if result
2458         result.plan.parseResult(result)
2459       end
2460     end
2461   }
2462   
2463   # Compute the geomean of the preferred means of results on a SuiteOnVM
2464   overallResults = []
2465   $vms.each {
2466     | vm |
2467     result = Stats.new
2468     $outer.times {
2469       | outerIndex |
2470       $inner.times {
2471         | innerIndex |
2472         curResult = Stats.new
2473         suitesOnVMsForVM[vm].each {
2474           | suiteOnVM |
2475           # For a given iteration, suite, and VM, compute the suite's preferred mean
2476           # over the data collected for all benchmarks in that suite. We'll have one
2477           # sample per benchmark. For example on V8 this will be the geomean of 1
2478           # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
2479           # splay.
2480           curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
2481         }
2482         
2483         # curResult now holds 1 sample for each of the means computed in the above
2484         # loop. Compute the geomean over this, and store it.
2485         if curResult.ok?
2486           result.add(curResult.geometricMean)
2487         else
2488           result.add(nil)
2489         end
2490       }
2491     }
2492
2493     # $overallResults will have a Stats for each VM. That Stats object will hold
2494     # $inner*$outer geomeans, allowing us to compute the arithmetic mean and
2495     # confidence interval of the geomeans of preferred means. Convoluted, but
2496     # useful and probably sound.
2497     overallResults << result
2498   }
2499   
2500   if $verbosity >= 2
2501     benchmarksOnVMs.each {
2502       | benchmarkOnVM |
2503       $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
2504     }
2505     
2506     $vms.each_with_index {
2507       | vm, vmIndex |
2508       vmStats = vmStatses[vmIndex]
2509       $stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}"
2510       $stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}"
2511     }
2512   end
2513
2514   if $outputName
2515     reportName = $outputName
2516   else
2517     reportName =
2518       (if ($vms.collect {
2519              | vm |
2520              vm.nameKind
2521            }.index :auto)
2522          ""
2523        else
2524          text = $vms.collect {
2525            | vm |
2526            vm.to_s
2527          }.join("_") + "_"
2528          if text.size >= 40
2529            ""
2530          else
2531            text
2532          end
2533        end) +
2534       ($suites.collect {
2535          | suite |
2536          suite.to_s
2537        }.join("")) + "_" +
2538       (if hostname
2539          hostname + "_"
2540        else
2541          ""
2542        end)+
2543       (begin
2544          time = Time.now
2545          "%04d%02d%02d_%02d%02d" %
2546            [ time.year, time.month, time.day,
2547              time.hour, time.min ]
2548        end)
2549   end
2550
2551   unless $brief
2552     puts "Generating benchmark report at #{Dir.pwd}/#{reportName}_report.txt"
2553     puts "And raw data at #{Dir.pwd}/#{reportName}.json"
2554   end
2555   
2556   outp = $stdout
2557   json = {}
2558   begin
2559     outp = File.open(reportName + "_report.txt","w")
2560   rescue => e
2561     $stderr.puts "Error: could not save report to #{reportName}_report.txt: #{e}"
2562     $stderr.puts
2563   end
2564   
2565   def createVMsString
2566     result = ""
2567     result += "   " if $allSuites.size > 1
2568     result += rpad("", $benchpad + $weightpad)
2569     result += " "
2570     $vms.size.times {
2571       | index |
2572       if index != 0
2573         result += " "+NoChange.new(0).shortForm
2574       end
2575       result += lpad(center($vms[index].name, 10+10+2), 12+10+2)
2576     }
2577     result += "    "
2578     if $vms.size >= 3
2579       result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
2580     elsif $vms.size >= 2
2581       result += " "*26
2582     end
2583     result
2584   end
2585   
2586   def andJoin(list)
2587     if list.size == 1
2588       list[0].to_s
2589     elsif list.size == 2
2590       "#{list[0]} and #{list[1]}"
2591     else
2592       "#{list[0..-2].join(', ')}, and #{list[-1]}"
2593     end
2594   end
2595   
2596   json["vms"] = $vms.collect{|v| v.name}
2597   json["suites"] = {}
2598   json["runlog"] = results
2599   
2600   columns = [createVMsString.size, 78].max
2601   
2602   outp.print "Benchmark report for "
2603   outp.print andJoin($suites)
2604   if hostname
2605     outp.print " on #{hostname}"
2606   end
2607   if hwmodel
2608     outp.print " (#{hwmodel})"
2609   end
2610   outp.puts "."
2611   outp.puts
2612   
2613   outp.puts "VMs tested:"
2614   $vms.each {
2615     | vm |
2616     outp.print "\"#{vm.name}\" at #{vm.origPath}"
2617     if vm.svnRevision
2618       outp.print " (r#{vm.svnRevision})"
2619     end
2620     outp.puts
2621     vm.extraEnv.each_pair {
2622       | key, val |
2623       outp.puts "    export #{key}=#{val}"
2624     }
2625   }
2626   
2627   outp.puts
2628   
2629   outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
2630                  "with #{$outer} VM invocation#{plural($outer)} per benchmark."+
2631                  (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
2632                                       "total time of those iterations, for each sample.")
2633                   else "" end)+
2634                  (if $measureGC == true then (" No manual garbage collection invocations were "+
2635                                               "emitted.")
2636                   elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
2637                                          "all VMs except #{$measureGC}.")
2638                   else (" Emitted a call to gc() between sample measurements.") end)+
2639                  (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
2640                                         "began with the very first iteration.")
2641                   else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
2642                         "invocation for warm-up.") end)+
2643                  (case $timeMode
2644                   when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
2645                                           "microsecond-level timing.")
2646                   when :date then (" Used the portable Date.now() method to get millisecond-"+
2647                                    "level timing.")
2648                   else raise end)+
2649                  " Reporting benchmark execution times with 95% confidence "+
2650                  "intervals in milliseconds.",
2651                  columns)
2652   
2653   outp.puts
2654   
2655   def printVMs(outp)
2656     outp.puts createVMsString
2657   end
2658   
2659   def summaryStats(outp, json, accumulators, name, decimalShift, &proc)
2660     resultingJson = {}
2661     outp.print "   " if $allSuites.size > 1
2662     outp.print rpad(name, $benchpad + $weightpad)
2663     outp.print " "
2664     accumulators.size.times {
2665       | index |
2666       if index != 0
2667         outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
2668       end
2669       outp.print statsToStr(accumulators[index].stats(&proc), decimalShift)
2670       resultingJson[accumulators[index].reportingName] = accumulators[index].stats(&proc).jsonMap
2671     }
2672     if accumulators.size>=2
2673       outp.print("    "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm)
2674     end
2675     outp.puts
2676     json[name] = resultingJson
2677   end
2678
2679   def allSummaryStats(outp, json, accumulators, preferredMean, decimalShift)
2680     meanLabel = '<' + preferredMean.to_s.sub(/Mean$/, '') + '>'
2681     summaryStats(outp, json, accumulators, meanLabel, decimalShift) {
2682       | stat |
2683       stat.send(preferredMean)
2684     }
2685   end
2686   
2687   $suites.each {
2688     | suite |
2689     suiteJson = {}
2690     subSuiteJsons = {}
2691     suite.subSuites.each {
2692       | subSuite |
2693       subSuiteJsons[subSuite] = {}
2694     }
2695     
2696     printVMs(outp)
2697     if $allSuites.size > 1
2698       outp.puts(andJoin(suite.suites.map{|v| v.name}) + ":")
2699     else
2700       outp.puts
2701     end
2702     suite.benchmarks.each {
2703       | benchmark |
2704       benchmarkJson = {}
2705       outp.print "   " if $allSuites.size > 1
2706       outp.print rpad(benchmark.name, $benchpad) + rpad(benchmark.weightString, $weightpad)
2707       if benchmark.name.size > $benchNameClip
2708         outp.puts
2709         outp.print "   " if $allSuites.size > 1
2710         outp.print((" " * $benchpad) + (" " * $weightpad))
2711       end
2712       outp.print " "
2713       myConfigs = benchmarksOnVMsForBenchmark[benchmark]
2714       myConfigs.size.times {
2715         | index |
2716         if index != 0
2717           outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
2718         end
2719         outp.print statsToStr(myConfigs[index].stats, suite.decimalShift)
2720         benchmarkJson[myConfigs[index].vm.name] = myConfigs[index].stats.jsonMap
2721       }
2722       if $vms.size>=2
2723         outp.print("    "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
2724       end
2725       outp.puts
2726       suiteJson[benchmark.name] = benchmarkJson
2727       suite.subSuites.each {
2728         | subSuite |
2729         if subSuite.hasBenchmark(benchmark)
2730           subSuiteJsons[subSuite][benchmark.name] = benchmarkJson
2731         end
2732       }
2733     }
2734     outp.puts
2735     unless suite.subSuites.empty?
2736       suite.subSuites.each {
2737         | subSuite |
2738         outp.puts "#{subSuite.name}:"
2739         allSummaryStats(outp, subSuiteJsons[subSuite], subSuitesOnVMsForSubSuite[subSuite], subSuite.preferredMean, subSuite.decimalShift)
2740         outp.puts
2741       }
2742       outp.puts "#{suite.name} including #{andJoin(suite.subSuites.map{|v| v.name})}:"
2743     end
2744     allSummaryStats(outp, suiteJson, suitesOnVMsForSuite[suite], suite.preferredMean, suite.decimalShift)
2745     outp.puts if $allSuites.size > 1
2746     
2747     json["suites"][suite.name] = suiteJson
2748     suite.subSuites.each {
2749       | subSuite |
2750       json["suites"][subSuite.name] = subSuiteJsons[subSuite]
2751     }
2752   }
2753   
2754   if $suites.size > 1
2755     scaledResultJson = {}
2756     printVMs(outp)
2757     outp.puts "Geomean of preferred means:"
2758     outp.print "   "
2759     outp.print rpad("<scaled-result>", $benchpad + $weightpad)
2760     outp.print " "
2761     $vms.size.times {
2762       | index |
2763       if index != 0
2764         outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm
2765       end
2766       outp.print statsToStr(overallResults[index], 0)
2767       scaledResultJson[$vms[index].name] = overallResults[index].jsonMap
2768     }
2769     if overallResults.size>=2
2770       outp.print("    "+overallResults[-1].compareTo(overallResults[0]).longForm)
2771     end
2772     outp.puts
2773     
2774     json["<scaled-result>"] = scaledResultJson
2775   end
2776   outp.puts
2777   
2778   if outp != $stdout
2779     outp.close
2780   end
2781   
2782   if outp != $stdout and not $brief
2783     puts
2784     File.open(reportName + "_report.txt") {
2785       | inp |
2786       puts inp.read
2787     }
2788   end
2789   
2790   if $brief
2791     puts(overallResults.collect{|stats| stats.mean}.join("\t"))
2792     puts(overallResults.collect{|stats| stats.confInt}.join("\t"))
2793   end
2794   
2795   File.open(reportName + ".json", "w") {
2796     | outp |
2797     outp.puts json.to_json
2798   }
2799 end
2800
2801 begin
2802   $sawBenchOptions = false
2803   
2804   def resetBenchOptionsIfNecessary
2805     unless $sawBenchOptions
2806       $includeSunSpider = false
2807       $includeSunSpiderCompileTime = false
2808       $includeLongSpider = false
2809       $includeV8 = false
2810       $includeV8CompileTime = false
2811       $includeKraken = false
2812       $includeJSBench = false
2813       $includeMicrobenchmarks = false
2814       $includeAsmBench = false
2815       $includeDSPJS = false
2816       $includeBrowsermarkJS = false
2817       $includeBrowsermarkDOM = false
2818       $includeOctane = false
2819       $includeCompressionBench = false
2820       $includeSixSpeed = false
2821       $includeTailBench = false;
2822       $sawBenchOptions = true
2823     end
2824   end
2825   
2826   GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
2827                  ['--inner', GetoptLong::REQUIRED_ARGUMENT],
2828                  ['--outer', GetoptLong::REQUIRED_ARGUMENT],
2829                  ['--warmup', GetoptLong::REQUIRED_ARGUMENT],
2830                  ['--no-ss-warmup', GetoptLong::NO_ARGUMENT],
2831                  ['--quantum', GetoptLong::REQUIRED_ARGUMENT],
2832                  ['--minimum', GetoptLong::REQUIRED_ARGUMENT],
2833                  ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
2834                  ['--sunspider', GetoptLong::NO_ARGUMENT],
2835                  ['--sunspider-compile-time', GetoptLong::NO_ARGUMENT],
2836                  ['--longspider', GetoptLong::NO_ARGUMENT],
2837                  ['--v8-spider', GetoptLong::NO_ARGUMENT],
2838                  ['--v8-spider-compile-time', GetoptLong::NO_ARGUMENT],
2839                  ['--kraken', GetoptLong::NO_ARGUMENT],
2840                  ['--js-bench', GetoptLong::NO_ARGUMENT],
2841                  ['--microbenchmarks', GetoptLong::NO_ARGUMENT],
2842                  ['--asm-bench', GetoptLong::NO_ARGUMENT],
2843                  ['--dsp', GetoptLong::NO_ARGUMENT],
2844                  ['--browsermark-js', GetoptLong::NO_ARGUMENT],
2845                  ['--browsermark-dom', GetoptLong::NO_ARGUMENT],
2846                  ['--octane', GetoptLong::NO_ARGUMENT],
2847                  ['--compression-bench', GetoptLong::NO_ARGUMENT],
2848                  ['--six-speed', GetoptLong::NO_ARGUMENT],
2849                  ['--tail-bench', GetoptLong::NO_ARGUMENT],
2850                  ['--big-int-bench', GetoptLong::NO_ARGUMENT],
2851                  ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
2852                  ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
2853                  ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
2854                  ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
2855                  ['--dont-copy-vms', GetoptLong::NO_ARGUMENT],
2856                  ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
2857                  ['--brief', GetoptLong::NO_ARGUMENT],
2858                  ['--silent', GetoptLong::NO_ARGUMENT],
2859                  ['--remote', GetoptLong::REQUIRED_ARGUMENT],
2860                  ['--local', GetoptLong::NO_ARGUMENT],
2861                  ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
2862                  ['--slave', GetoptLong::NO_ARGUMENT],
2863                  ['--prepare-only', GetoptLong::NO_ARGUMENT],
2864                  ['--analyze', GetoptLong::REQUIRED_ARGUMENT],
2865                  ['--vms', GetoptLong::REQUIRED_ARGUMENT],
2866                  ['--output-name', GetoptLong::REQUIRED_ARGUMENT],
2867                  ['--environment', GetoptLong::REQUIRED_ARGUMENT],
2868                  ['--dependencies', GetoptLong::REQUIRED_ARGUMENT],
2869                  ['--config', GetoptLong::REQUIRED_ARGUMENT],
2870                  ['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
2871     | opt, arg |
2872     case opt
2873     when '--rerun'
2874       $rerun = intArg(opt,arg,1,nil)
2875     when '--inner'
2876       $inner = intArg(opt,arg,1,nil)
2877     when '--outer'
2878       $outer = intArg(opt,arg,1,nil)
2879     when '--warmup'
2880       $warmup = intArg(opt,arg,0,nil)
2881     when '--no-ss-warmup'
2882       $sunSpiderWarmup = false
2883     when '--quantum'
2884       $quantum = intArg(opt,arg,1,nil)
2885     when '--minimum'
2886       $minimum = intArg(opt,arg,1,nil)
2887     when '--timing-mode'
2888       if arg.upcase == "PRECISETIME"
2889         $timeMode = :preciseTime
2890       elsif arg.upcase == "DATE"
2891         $timeMode = :date
2892       elsif arg.upcase == "AUTO"
2893         $timeMode = :auto
2894       else
2895         quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
2896                   "Invalid argument for command-line option")
2897       end
2898     when '--force-vm-kind'
2899       if arg.upcase == "JSC"
2900         $forceVMKind = :jsc
2901       elsif arg.upcase == "DUMPRENDERTREE"
2902         $forceVMKind = :dumpRenderTree
2903       elsif arg.upcase == "WEBKITTESTRUNNER"
2904         $forceVMKind = :webkitTestRunner
2905       elsif arg.upcase == "AUTO"
2906         $forceVMKind = nil
2907       else
2908         quickFail("Expected 'jsc', 'DumpRenderTree', or 'WebKitTestRunner' for --force-vm-kind, but got '#{arg}'.",
2909                   "Invalid argument for command-line option")
2910       end
2911     when '--force-vm-copy'
2912       $needToCopyVMs = true
2913     when '--dont-copy-vms'
2914       $dontCopyVMs = true
2915     when '--sunspider'
2916       resetBenchOptionsIfNecessary
2917       $includeSunSpider = true
2918     when '--sunspider-compile-time'
2919       resetBenchOptionsIfNecessary
2920       $includeSunSpiderCompileTime = true
2921     when '--longspider'
2922       resetBenchOptionsIfNecessary
2923       $includeLongSpider = true
2924     when '--v8-spider'
2925       resetBenchOptionsIfNecessary
2926       $includeV8 = true
2927     when '--v8-spider-compile-time'
2928       resetBenchOptionsIfNecessary
2929       $includeV8CompileTime = true
2930     when '--kraken'
2931       resetBenchOptionsIfNecessary
2932       $includeKraken = true
2933     when '--js-bench'
2934       resetBenchOptionsIfNecessary
2935       $includeJSBench = true
2936     when '--tail-bench'
2937       resetBenchOptionsIfNecessary
2938       $includeTailBench = true
2939     when '--microbenchmarks'
2940       resetBenchOptionsIfNecessary
2941       $includeMicrobenchmarks = true
2942     when '--asm-bench'
2943       resetBenchOptionsIfNecessary
2944       $includeAsmBench = true
2945     when '--dsp'
2946       resetBenchOptionsIfNecessary
2947       $includeDSPJS = true
2948     when '--browsermark-js'
2949       resetBenchOptionsIfNecessary
2950       $includeBrowsermarkJS = true
2951     when '--browsermark-dom'
2952       resetBenchOptionsIfNecessary
2953       $includeBrowsermarkDOM = true
2954     when '--octane'
2955       resetBenchOptionsIfNecessary
2956       $includeOctane = true
2957     when '--compression-bench'
2958       resetBenchOptionsIfNecessary
2959       $includeCompressionBench = true
2960     when '--six-speed'
2961       resetBenchOptionsIfNecessary
2962       $includeSixSpeed = true
2963     when '--big-int-bench'
2964       resetBenchOptionsIfNecessary
2965       $includeBigIntBench = true
2966     when '--benchmarks'
2967       $benchmarkPattern = Regexp.new(arg)
2968     when '--measure-gc'
2969       if arg == ''
2970         $measureGC = true
2971       else
2972         $measureGC = arg
2973       end
2974     when '--verbose'
2975       $verbosity += 1
2976     when '--brief'
2977       $brief = true
2978     when '--silent'
2979       $silent = true
2980     when '--remote'
2981       $remoteHosts += arg.split(',')
2982       $needToCopyVMs = true
2983     when '--ssh-options'
2984       $sshOptions << arg
2985     when '--local'
2986       $alsoLocal = true
2987     when '--prepare-only'
2988       $run = false
2989     when '--analyze'
2990       $prepare = false
2991       $run = false
2992       $analyze << arg
2993     when '--output-name'
2994       $outputName = arg
2995     when '--vms'
2996       JSON::parse(IO::read(arg)).each {
2997         | vmDescription |
2998         path = Pathname.new(vmDescription["path"]).realpath
2999         if vmDescription["name"]
3000           name = vmDescription["name"]
3001           nameKind = :given
3002         else
3003           name = "Conf\##{$vms.length+1}"
3004           nameKind = :auto
3005         end
3006         vm = VM.new(path, name, nameKind, nil)
3007         if vmDescription["env"]
3008           vmDescription["env"].each_pair {
3009             | key, val |
3010             vm.addExtraEnv(key, val)
3011           }
3012         end
3013         $vms << vm
3014       }
3015     when '--environment'
3016       $environment = JSON::parse(IO::read(arg))
3017     when '--dependencies'
3018       $dependencies.push(Pathname.new(arg).realpath)
3019     when '--config'
3020       $configPath = Pathname.new(arg)
3021     when '--help'
3022       usage
3023     else
3024       raise "bad option: #{opt}"
3025     end
3026   }
3027   
3028   # Figure out the configuration
3029   if $configPath.file?
3030     config = JSON::parse(IO::read($configPath.to_s))
3031   else
3032     config = {}
3033   end
3034
3035   def pathname_if_exist config
3036     if config
3037       Pathname.new config
3038     else
3039       config
3040     end
3041   end
3042
3043   OCTANE_PATH = pathname_if_exist(config["OctanePath"])
3044   BROWSERMARK_PATH = pathname_if_exist(config["BrowserMarkPath"])
3045   BROWSERMARK_JS_PATH = pathname_if_exist(config["BrowserMarkJSPath"])
3046   BROWSERMARK_DOM_PATH = pathname_if_exist(config["BrowserMarkDOMPath"])
3047   ASMBENCH_PATH = pathname_if_exist(config["AsmBenchPath"])
3048   COMPRESSIONBENCH_PATH = pathname_if_exist(config["CompressionBenchPath"])
3049   DSPJS_FILTRR_PATH = pathname_if_exist(config["DSPJSFiltrrPath"])
3050   DSPJS_ROUTE9_PATH = pathname_if_exist(config["DSPJSRoute9Path"])
3051   DSPJS_STARFIELD_PATH = pathname_if_exist(config["DSPJSStarfieldPath"])
3052   DSPJS_QUAKE3_PATH = pathname_if_exist(config["DSPJSQuake3Path"])
3053   DSPJS_MANDELBROT_PATH = pathname_if_exist(config["DSPJSMandelbrotPath"])
3054   DSPJS_JSLINUX_PATH = pathname_if_exist(config["DSPJSLinuxPath"])
3055   DSPJS_AMMOJS_ASMJS_PATH = pathname_if_exist(config["DSPJSAmmoJSAsmJSPath"])
3056   DSPJS_AMMOJS_REGULAR_PATH = pathname_if_exist(config["DSPJSAmmoJSRegularPath"])
3057   KRAKEN_PATH = pathname_if_exist(config["KrakenPath"])
3058   
3059   # If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option.
3060   if $dontCopyVMs
3061     $needToCopyVMs = false
3062   end
3063   
3064   ARGV.each {
3065     | vm |
3066     if vm =~ /([a-zA-Z0-9_ .]+):/
3067       name = $1
3068       nameKind = :given
3069       vm = $~.post_match
3070     else
3071       name = "Conf\##{$vms.length+1}"
3072       nameKind = :auto
3073     end
3074     envs = []
3075     while vm =~ /([a-zA-Z0-9_]+)=([a-zA-Z0-9_:.\/-]+):/
3076       envs << [$1, $2]
3077       vm = $~.post_match
3078     end
3079     $stderr.puts "#{name}: #{vm}" if $verbosity >= 1
3080     vm = VM.new(Pathname.new(vm).realpath, name, nameKind, nil)
3081     envs.each {
3082       | pair |
3083       vm.addExtraEnv(pair[0], pair[1])
3084     }
3085     $vms << vm
3086   }
3087   
3088   if $vms.empty?
3089     quickFail("Please specify at least on configuraiton on the command line.",
3090               "Insufficient arguments")
3091   end
3092   
3093   $vms.each {
3094     | vm |
3095     if vm.vmType == :jsc
3096       $allDRT = false
3097     end
3098   }
3099   
3100   SUNSPIDER = BenchmarkSuite.new("SunSpider", :arithmeticMean, 0)
3101   SUNSPIDER_COMPILE_TIME = BenchmarkSuite.new("SunSpider-CompileTime", :arithmeticMean, 0)
3102   WARMUP = BenchmarkSuite.new("WARMUP", :arithmeticMean, 0)
3103   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
3104    "access-fannkuch", "access-nbody", "access-nsieve",
3105    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
3106    "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
3107    "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
3108    "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
3109    "string-base64", "string-fasta", "string-tagcloud",
3110    "string-unpack-code", "string-validate-input"].each {
3111     | name |
3112     SUNSPIDER.add SunSpiderBenchmark.new(name)
3113     SUNSPIDER_COMPILE_TIME.add CompileTimeBenchmark.new(name, "#{SUNSPIDER_PATH}/#{name}.js")
3114     WARMUP.addIgnoringPattern SunSpiderBenchmark.new(name)
3115   }
3116
3117   LONGSPIDER = BenchmarkSuite.new("LongSpider", :geometricMean, 0)
3118   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
3119    "access-fannkuch", "access-nbody", "access-nsieve",
3120    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-nsieve-bits",
3121    "controlflow-recursive", "crypto-aes", "crypto-md5", "crypto-sha1",
3122    "date-format-tofte", "date-format-xparb", "hash-map", "math-cordic",
3123    "math-partial-sums", "math-spectral-norm", "string-base64",
3124    "string-fasta", "string-tagcloud"].each {
3125     | name |
3126     LONGSPIDER.add LongSpiderBenchmark.new(name)
3127   }
3128
3129   V8 = BenchmarkSuite.new("V8Spider", :geometricMean, 0)
3130   V8_COMPILE_TIME = BenchmarkSuite.new("V8Spider-CompileTime", :geometricMean, 0)
3131   ["crypto", "deltablue", "earley-boyer", "raytrace",
3132    "regexp", "richards", "splay"].each {
3133     | name |
3134     V8.add V8Benchmark.new(name)
3135     V8_COMPILE_TIME.add CompileTimeBenchmark.new(name, "#{V8_PATH}/v8-#{name}.js")
3136   }
3137
3138   OCTANE = BenchmarkSuite.new("Octane", :geometricMean, 1)
3139   [[["crypto"], "encrypt", 1, true, false, 32],
3140    [["crypto"], "decrypt", 1, true, false, 32],
3141    [["deltablue"], "deltablue", 2, true, false, 32],
3142    [["earley-boyer"], "earley", 1, true, false, 32],
3143    [["earley-boyer"], "boyer", 1, true, false, 32],
3144    [["navier-stokes"], "navier-stokes", 2, true, false, 16],
3145    [["raytrace"], "raytrace", 2, true, false, 32],
3146    [["richards"], "richards", 2, true, false, 32],
3147    [["splay"], "splay", 2, true, false, 32],
3148    [["regexp"], "regexp", 2, true, false, 16],
3149    [["pdfjs"], "pdfjs", 2, false, false, 4],
3150    [["mandreel"], "mandreel", 2, false, false, 4],
3151    [["gbemu-part1", "gbemu-part2"], "gbemu", 2, false, false, 4],
3152    [["code-load"], "closure", 1, false, false, 16],
3153    [["code-load"], "jquery", 1, false, false, 16],
3154    [["box2d"], "box2d", 2, false, false, 8],
3155    [["zlib", "zlib-data"], "zlib", 2, false, true, 3],
3156    [["typescript", "typescript-input", "typescript-compiler"], "typescript", 2, false, true, 1]].each {
3157     | args |
3158     OCTANE.add OctaneBenchmark.new(*args)
3159   }
3160   OCTANE.add CustomTimedBenchmark.new("splay-latency", (OPENSOURCE_OCTANE_PATH + "splay.js").to_s)
3161
3162   KRAKEN = BenchmarkSuite.new("Kraken", :arithmeticMean, -1)
3163   ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
3164    "audio-oscillator", "imaging-darkroom", "imaging-desaturate",
3165    "imaging-gaussian-blur", "json-parse-financial",
3166    "json-stringify-tinderbox", "stanford-crypto-aes",
3167    "stanford-crypto-ccm", "stanford-crypto-pbkdf2",
3168    "stanford-crypto-sha256-iterative"].each {
3169     | name |
3170     KRAKEN.add KrakenBenchmark.new(name)
3171   }
3172   
3173   JSBENCH = BenchmarkSuite.new("JSBench", :geometricMean, 0)
3174   [["amazon-chrome", "urem"], ["amazon-chrome-win", "urem"], ["amazon-firefox", "urm"],
3175    ["amazon-firefox-win", "urm"], ["amazon-safari", "urem"], ["facebook-chrome", "urem"],
3176    ["facebook-chrome-win", "urem"], ["facebook-firefox", "urem"],
3177    ["facebook-firefox-win", "urem"], ["facebook-safari", "urem"],
3178    ["google-chrome", "urem"], ["google-chrome-win", "urem"], ["google-firefox", "uem"],
3179    ["google-firefox-win", "urem"], ["google-safari", "urem"], ["twitter-chrome", "urem"],
3180    ["twitter-chrome-win", "rem"], ["twitter-firefox", "urem"],
3181    ["twitter-firefox-win", "urem"], ["twitter-safari", "urem"], ["yahoo-chrome", "urem"],
3182    ["yahoo-chrome-win", "urem"], ["yahoo-firefox", "urem"], ["yahoo-firefox-win", "urem"],
3183    ["yahoo-safari", "urem"]].each {
3184     | nameAndMode |
3185     JSBENCH.add JSBenchBenchmark.new(*nameAndMode)
3186   }
3187
3188   TAILBENCH = BenchmarkSuite.new("TailBench", :geometricMean, 0)
3189   ["n-body", "merge-sort", "merge-sort-cps", "bf-interpreter"].each {
3190     | name |
3191     TAILBENCH.add TailBenchBenchmark.new(name);
3192   }
3193
3194   BIGINTBENCH = BenchmarkSuite.new("BigIntBench", :geometricMean, 0)
3195   Dir.foreach(BIGINTBENCH_PATH) {
3196     | filename |
3197     if filename =~ /\.js$/
3198       name = $~.pre_match
3199       BIGINTBENCH.add BigIntBenchBenchmark.new(name)
3200     end
3201   }
3202
3203   MICROBENCHMARKS = BenchmarkSuite.new("Microbenchmarks", :geometricMean, 0)
3204   Dir.foreach(MICROBENCHMARKS_PATH) {
3205     | filename |
3206     if filename =~ /\.js$/
3207       name = $~.pre_match
3208       MICROBENCHMARKS.add MicrobenchmarksBenchmark.new(name)
3209     end
3210   }
3211   
3212   ASMBENCH = BenchmarkSuite.new("AsmBench", :geometricMean, 0)
3213   if ASMBENCH_PATH
3214     Dir.foreach(ASMBENCH_PATH) {
3215       | filename |
3216       if filename =~ /\.js$/
3217         name = $~.pre_match
3218         ASMBENCH.add AsmBenchBenchmark.new(name)
3219       end
3220     }
3221   end
3222
3223   COMPRESSIONBENCH = BenchmarkSuite.new("CompressionBench", :geometricMean, 0)
3224   [[["huffman", "compression-data"], "huffman", ""],
3225    [["arithmetic", "compression-data"], "arithmetic", "Simple"],
3226    [["arithmetic", "compression-data"], "arithmetic", "Precise"],
3227    [["arithmetic", "compression-data"], "arithmetic", "Complex Precise"],
3228    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 0"],
3229    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 1"],
3230    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 2"],
3231    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 1"],
3232    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 2"],
3233    [["lz-string", "compression-data"], "lz-string", ""]
3234   ].each {
3235     | args |
3236     COMPRESSIONBENCH.add CompressionBenchBenchmark.new(*args)
3237   }
3238
3239   SIXSPEED = BenchmarkSuite.new("SixSpeed", :geometricMean, 0)
3240   [[ "template_string", 200000000 ],
3241    [ "defaults", 100000000 ],
3242    [ "map-set-lookup", 200000 ],
3243    [ "spread", 1000000 ],
3244    [ "object-assign", 600000 ],
3245    [ "spread-literal", 1000000 ],
3246    [ "map-set", 10000 ],
3247    [ "destructuring-simple", 20000000 ],
3248    [ "super", 3000000 ],
3249    [ "for-of-object", 1000000 ],
3250    [ "rest", 500000 ],
3251    [ "regex-u", 1000000 ],
3252    [ "arrow", 20000000 ],
3253    [ "bindings-compound", 20000000 ],
3254    [ "classes", 10000000 ],
3255    [ "template_string_tag", 2000000 ],
3256    [ "map-string", 30000000 ],
3257    [ "arrow-declare", 30000000 ],
3258    [ "spread-generator", 1000000 ],
3259    [ "object-literal-ext", 1000000 ],
3260    [ "generator", 3000000 ],
3261    [ "arrow-args", 20000000 ],
3262    [ "for-of-array", 5000000 ],
3263    [ "bindings", 20000000 ],
3264    [ "destructuring", 20000000 ],
3265    [ "map-set-object", 5000 ],
3266   ].each {
3267     | name, iterations |
3268     SIXSPEED.add SixSpeedBenchmark.new("#{name}.es5", "#{name}/#{name}.es5", iterations)
3269     SIXSPEED.add SixSpeedBenchmark.new("#{name}.es6", "#{name}/#{name}.es6", iterations)
3270   }
3271
3272   DSPJS = BenchmarkSuite.new("DSP", :geometricMean, 0)
3273   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-posterize-tint", "e2")
3274   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-contrast-sat-bright", "e5")
3275   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-sat-adj-contr-mult", "e7")
3276   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-blur-overlay-sat-contr", "e8")
3277   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sat-blur-mult-sharpen-contr", "e9")
3278   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sepia-bias", "e10")
3279   DSPJS.add DSPJSVP8Benchmark.new
3280   DSPJS.add DSPStarfieldBenchmark.new
3281   DSPJS.add DSPJSJSLinuxBenchmark.new
3282   DSPJS.add DSPJSQuake3Benchmark.new
3283   DSPJS.add DSPJSMandelbrotBenchmark.new
3284   DSPJS.add DSPJSAmmoJSASMBenchmark.new
3285   DSPJS.add DSPJSAmmoJSRegularBenchmark.new
3286
3287   BROWSERMARK_JS = BenchmarkSuite.new("BrowsermarkJS", :geometricMean, 1)
3288   ["array_blur", "array_weighted", "string_chat", "string_filter", "string_weighted"].each {
3289     | name |
3290       BROWSERMARK_JS.add BrowsermarkJSBenchmark.new(name)
3291   }
3292   
3293   BROWSERMARK_DOM = BenchmarkSuite.new("BrowsermarkDOM", :geometricMean, 1)
3294   ["advanced_search", "create_source", "dynamic_create", "search"].each {
3295     | name |
3296       BROWSERMARK_DOM.add BrowsermarkDOMBenchmark.new(name)
3297   }
3298
3299   $suites = []
3300   
3301   if $includeSunSpider and not SUNSPIDER.empty?
3302     $suites << SUNSPIDER
3303   end
3304   
3305   if $includeSunSpiderCompileTime and not SUNSPIDER_COMPILE_TIME.empty?
3306     $suites << SUNSPIDER_COMPILE_TIME
3307   end
3308   
3309   if $includeLongSpider and not LONGSPIDER.empty?
3310     $suites << LONGSPIDER
3311   end
3312   
3313   if $includeV8 and not V8.empty?
3314     $suites << V8
3315   end
3316   
3317   if $includeV8CompileTime and not V8_COMPILE_TIME.empty?
3318     $suites << V8_COMPILE_TIME
3319   end
3320   
3321   if $includeOctane and not OCTANE.empty?
3322     if OCTANE_PATH
3323       $suites << OCTANE
3324     else
3325       $stderr.puts "Warning: refusing to run Octane because \"OctanePath\" isn't set in #{$configPath}."
3326     end
3327   end
3328   
3329   if $includeKraken and not KRAKEN.empty?
3330     if KRAKEN_PATH
3331       $suites << KRAKEN
3332     else
3333       $stderr.puts "Warning: refusing to run Kraken because \"KrakenPath\" isn't set in #{$configPath}."
3334     end
3335   end
3336   
3337   if $includeJSBench and not JSBENCH.empty?
3338     if $allDRT
3339       if JSBENCH_PATH
3340         $suites << JSBENCH
3341       else
3342         $stderr.puts "Warning: refusing to run JSBench because \"JSBenchPath\" isn't set in #{$configPath}"
3343       end
3344     else
3345       $stderr.puts "Warning: refusing to run JSBench because not all VMs are DumpRenderTree or WebKitTestRunner."
3346     end
3347   end
3348   
3349   if $includeTailBench and not TAILBENCH.empty?
3350     $suites << TAILBENCH
3351   end
3352
3353   if $includeMicrobenchmarks and not MICROBENCHMARKS.empty?
3354     $suites << MICROBENCHMARKS
3355   end
3356
3357   if $includeBigIntBench and not BIGINTBENCH.empty?
3358     $suites << BIGINTBENCH
3359   end
3360
3361   if $includeAsmBench and not ASMBENCH.empty?
3362     if ASMBENCH_PATH
3363       $suites << ASMBENCH
3364     else
3365       $stderr.puts "Warning: refusing to run AsmBench because \"AsmBenchPath\" isn't set in #{$configPath}."
3366     end
3367   end
3368   
3369   if $includeDSPJS and not DSPJS.empty?
3370     if $allDRT
3371       if DSPJS_FILTRR_PATH and DSPJS_ROUTE9_PATH and DSPJS_STARFIELD_PATH and DSPJS_QUAKE3_PATH and DSPJS_MANDELBROT_PATH and DSPJS_JSLINUX_PATH and DSPJS_AMMOJS_ASMJS_PATH and DSPJS_AMMOJS_REGULAR_PATH
3372         $suites << DSPJS
3373       else
3374         $stderr.puts "Warning: refusing to run DSPJS because one of the following isn't set in #{$configPath}: \"DSPJSFiltrrPath\", \"DSPJSRoute9Path\", \"DSPJSStarfieldPath\", \"DSPJSQuake3Path\", \"DSPJSMandelbrotPath\", \"DSPJSLinuxPath\", \"DSPJSAmmoJSAsmJSPath\", \"DSPJSAmmoJSRegularPath\"."
3375       end
3376     else
3377       $stderr.puts "Warning: refusing to run DSPJS because not all VMs are DumpRenderTree or WebKitTestRunner."
3378     end
3379   end
3380   
3381   if $includeBrowsermarkJS and not BROWSERMARK_JS.empty?
3382     if BROWSERMARK_PATH and BROWSERMARK_JS_PATH
3383       $suites << BROWSERMARK_JS
3384     else
3385       $stderr.puts "Warning: refusing to run Browsermark-JS because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\" or \"BrowserMarkJSPath\"."
3386     end
3387   end
3388
3389   if $includeBrowsermarkDOM and not BROWSERMARK_DOM.empty?
3390     if $allDRT
3391       if BROWSERMARK_PATH and BROWSERMARK_JS_PATH and BROWSERMARK_DOM_PATH
3392         $suites << BROWSERMARK_DOM
3393       else
3394         $stderr.puts "Warning: refusing to run Browsermark-DOM because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\", \"BrowserMarkJSPath\", or \"BrowserMarkDOMPath\"."
3395       end
3396     else
3397       $stderr.puts "Warning: refusing to run Browsermark-DOM because not all VMs are DumpRenderTree or WebKitTestRunner."
3398     end
3399   end
3400
3401   if $includeCompressionBench and not COMPRESSIONBENCH.empty?
3402     if COMPRESSIONBENCH_PATH
3403       $suites << COMPRESSIONBENCH
3404     else
3405       $stderr.puts "Warning: refusing to run CompressionBench because \"CompressionBenchPath\" isn't set in #{$configPath}"
3406     end
3407   end
3408
3409   if $includeSixSpeed and not SIXSPEED.empty?
3410     if SIXSPEED_PATH
3411       $suites << SIXSPEED
3412     else
3413       $stderr.puts "Warning: refusing to run SixSpeed because \"SixSpeedPath\" isn't set in #{$configPath}."
3414     end
3415   end
3416
3417
3418   $allSuites = $suites.map{|v| v.suites}.flatten(1)
3419   
3420   $benchmarks = []
3421   $suites.each {
3422     | suite |
3423     $benchmarks += suite.benchmarks
3424   }
3425   
3426   if $suites.empty? or $benchmarks.empty?
3427     $stderr.puts "No benchmarks found.  Bailing out."
3428     exit 1
3429   end
3430   
3431   if $outer*$inner == 1
3432     $stderr.puts "Warning: will only collect one sample per benchmark/VM.  Confidence interval calculation will fail."
3433   end
3434   
3435   $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
3436   
3437   $runPlans = []
3438   $vms.each {
3439     | vm |
3440     $benchmarks.each {
3441       | benchmark |
3442       $outer.times {
3443         | iteration |
3444         $runPlans << BenchRunPlan.new(benchmark, vm, iteration)
3445       }
3446     }
3447   }
3448   
3449   $runPlans.shuffle!
3450   
3451   if $sunSpiderWarmup
3452     warmupPlans = []
3453     $vms.each {
3454       | vm |
3455       WARMUP.benchmarks.each {
3456         | benchmark |
3457         warmupPlans << BenchRunPlan.new(benchmark, vm, 0)
3458       }
3459     }
3460     
3461     $runPlans = warmupPlans.shuffle + $runPlans
3462   end
3463   
3464   $suitepad = $suites.collect {
3465     | suite |
3466     suite.to_s.size
3467   }.max + 1
3468   
3469   $planpad = $runPlans.collect {
3470     | plan |
3471     plan.to_s.size
3472   }.max + 1
3473   
3474   maxBenchNameLength =
3475     ($benchmarks + ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
3476     | benchmark |
3477     if benchmark.respond_to? :name
3478       benchmark.name.size
3479     else
3480       benchmark.size
3481     end
3482   }.max
3483   $benchNameClip = 40
3484   $benchpad = [maxBenchNameLength, $benchNameClip].min + 1
3485   
3486   $weightpad = $benchmarks.collect {
3487     | benchmark |
3488     benchmark.weightString.size
3489   }.max
3490
3491   $vmpad = $vms.collect {
3492     | vm |
3493     vm.to_s.size
3494   }.max + 1
3495   
3496   $analyze.each_with_index {
3497     | filename, index |
3498     if index >= 1
3499       puts
3500     end
3501     parseAndDisplayResults(IO::read(filename))
3502   }
3503   
3504   if not $prepare and not $run
3505     exit 0
3506   end
3507   
3508   if FileTest.exist? BENCH_DATA_PATH
3509     cmd = "rm -rf #{BENCH_DATA_PATH}"
3510     $stderr.puts ">> #{cmd}" if $verbosity >= 2
3511     raise unless system cmd
3512   end
3513   
3514   Dir.mkdir BENCH_DATA_PATH
3515   
3516   if $needToCopyVMs
3517     canCopyIntoBenchPath = true
3518     $vms.each {
3519       | vm |
3520       canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath
3521     }
3522     
3523     if canCopyIntoBenchPath
3524       $vms.each {
3525         | vm |
3526         $stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..."
3527         vm.copyIntoBenchPath
3528       }
3529       $stderr.puts "All VMs are in place."
3530     else
3531       $stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it."
3532     end
3533   end
3534   
3535   if $measureGC and $measureGC != true
3536     found = false
3537     $vms.each {
3538       | vm |
3539       if vm.name == $measureGC
3540         found = true
3541       end
3542     }
3543     unless found
3544       $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
3545     end
3546   end
3547   
3548   if $prepare
3549     File.open("#{BENCH_DATA_PATH}/runscript", "w") {
3550       | file |
3551       file.puts "echo -e \"HOSTNAME:\\c\""
3552       file.puts "hostname"
3553       file.puts "echo"
3554       file.puts "echo -e \"HARDWARE:\\c\""
3555       file.puts "/usr/sbin/sysctl hw.model"
3556       file.puts "echo"
3557       file.puts "set -e"
3558       $script = file
3559       $runPlans.each_with_index {
3560         | plan, idx |
3561         if $verbosity == 0 and not $silent
3562           text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s
3563           text2 = plan.to_s
3564           file.puts("echo -e " + Shellwords.shellescape("\r#{text1} #{rpad(text2,$planpad)}") + "\"\\c\" 1>&2")
3565           file.puts("echo -e " + Shellwords.shellescape("\r#{text1} #{text2}") + "\"\\c\" 1>&2")
3566         end
3567         plan.emitRunCode
3568       }
3569       if $verbosity == 0 and not $silent
3570         file.puts("echo -e " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}") + "\"\\c\" 1>&2")
3571         file.puts("echo -e " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size}") + " 1>&2")
3572       end
3573     }
3574   end
3575   
3576   if $run
3577     unless $remoteHosts.empty?
3578       $stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0
3579       Dir.chdir(TEMP_PATH) {
3580         cmd = "tar -czf payload.tar.gz benchdata"
3581         $stderr.puts ">> #{cmd}" if $verbosity>=2