[JSC][GTK][EFL] Allow run-jsc-benchmark to use WebKitTestRunner in EFL / GTK ports
[WebKit-https.git] / Tools / Scripts / run-jsc-benchmarks
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2011-2015 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 # 1. Redistributions of source code must retain the above copyright
9 #    notice, this list of conditions and the following disclaimer.
10 # 2. Redistributions in binary form must reproduce the above copyright
11 #    notice, this list of conditions and the following disclaimer in the
12 #    documentation and/or other materials provided with the distribution.
13 #
14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24 # THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'rubygems'
27
28 require 'getoptlong'
29 require 'pathname'
30 require 'shellwords'
31 require 'socket'
32
33 begin
34   require 'json'
35 rescue LoadError => e
36   $stderr.puts "It does not appear that you have the 'json' package installed.  Try running 'sudo gem install json'."
37   exit 1
38 end
39
40 SCRIPT_PATH = Pathname.new(__FILE__).realpath
41
42 raise unless SCRIPT_PATH.dirname.basename.to_s == "Scripts"
43 raise unless SCRIPT_PATH.dirname.dirname.basename.to_s == "Tools"
44
45 OPENSOURCE_PATH = SCRIPT_PATH.dirname.dirname.dirname
46
47 SUNSPIDER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "SunSpider" + "tests" + "sunspider-1.0"
48 LONGSPIDER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "LongSpider"
49 V8_PATH = OPENSOURCE_PATH + "PerformanceTests" + "SunSpider" + "tests" + "v8-v6"
50 JSREGRESS_PATH = OPENSOURCE_PATH + "LayoutTests" + "js" + "regress" + "script-tests"
51 OCTANE_WRAPPER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "Octane" + "wrappers"
52
53 TEMP_PATH = OPENSOURCE_PATH + "BenchmarkTemp"
54
55 if TEMP_PATH.exist?
56   raise unless TEMP_PATH.directory?
57 else
58   Dir.mkdir(TEMP_PATH)
59 end
60
61 BENCH_DATA_PATH = TEMP_PATH + "benchdata"
62
63 IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933, 
64             0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684, 
65             0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227, 
66             0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674, 
67             0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652, 
68             0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604, 
69             0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338, 
70             0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538, 
71             0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185, 
72             0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694, 
73             0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244, 
74             0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067, 
75             0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323, 
76             0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129, 
77             0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574, 
78             0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722, 
79             0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624, 
80             0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321, 
81             0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843, 
82             0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217, 
83             0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464, 
84             0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599, 
85             0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638, 
86             0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592, 
87             0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471, 
88             0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283, 
89             0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037, 
90             0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738, 
91             0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481, 
92             0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085, 
93             0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651, 
94             0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183, 
95             0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683, 
96             0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154, 
97             0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599, 
98             0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019, 
99             0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417, 
100             0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795, 
101             0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153, 
102             0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494, 
103             0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818, 
104             0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128, 
105             0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423, 
106             0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704, 
107             0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974, 
108             0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232, 
109             0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479, 
110             0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716, 
111             0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943, 
112             0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192, 
113             0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401, 
114             0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963, 
115             0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822, 
116             0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007, 
117             0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186, 
118             0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358, 
119             0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525, 
120             0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686, 
121             0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841, 
122             0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991, 
123             0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137, 
124             0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278, 
125             0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414, 
126             0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547, 
127             0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675, 
128             0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799, 
129             0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192, 
130             0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037, 
131             0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151, 
132             0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262, 
133             0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369, 
134             0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489, 
135             0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259, 
136             0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689, 
137             0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785, 
138             0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879, 
139             0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297, 
140             0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059, 
141             0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145, 
142             0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242, 
143             0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324, 
144             0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404, 
145             0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483, 
146             0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559, 
147             0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634, 
148             0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707, 
149             0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778, 
150             0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848, 
151             0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916, 
152             0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983, 
153             0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057, 
154             0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121, 
155             0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183, 
156             0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244, 
157             0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304, 
158             0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363, 
159             0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442, 
160             0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476, 
161             0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539, 
162             0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946, 
163             0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653, 
164             0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712, 
165             0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762, 
166             0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812, 
167             0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867, 
168             0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915, 
169             0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962, 
170             0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007, 
171             0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059, 
172             0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103, 
173             0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146, 
174             0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189, 
175             0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231, 
176             0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272, 
177             0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312, 
178             0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352, 
179             0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391, 
180             0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429, 
181             0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467, 
182             0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504, 
183             0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546, 
184             0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582, 
185             0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617, 
186             0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652, 
187             0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686, 
188             0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724, 
189             0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757, 
190             0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579, 
191             0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822, 
192             0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853, 
193             0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885, 
194             0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915, 
195             0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595, 
196             0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979, 
197             0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013, 
198             0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041, 
199             0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074, 
200             0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102, 
201             0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129, 
202             0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156, 
203             0.99616, 0.996164]
204
205 # Run-time configuration parameters (can be set with command-line options)
206
207 $rerun=1
208 $inner=1
209 $warmup=1
210 $outer=4
211 $quantum=1000
212 $includeSunSpider=true
213 $includeLongSpider=true
214 $includeV8=true
215 $includeKraken=true
216 $includeJSBench=true
217 $includeJSRegress=true
218 $includeAsmBench=true
219 $includeDSPJS=true
220 $includeBrowsermarkJS=false
221 $includeBrowsermarkDOM=false
222 $includeOctane=true
223 $includeCompressionBench = true
224 $measureGC=false
225 $benchmarkPattern=nil
226 $verbosity=0
227 $timeMode=:preciseTime
228 $forceVMKind=nil
229 $brief=false
230 $silent=false
231 $remoteHosts=[]
232 $alsoLocal=false
233 $sshOptions=[]
234 $vms = []
235 $environment = {}
236 $dependencies = []
237 $needToCopyVMs = false
238 $dontCopyVMs = false
239 $allDRT = true
240 $outputName = nil
241 $sunSpiderWarmup = true
242 $configPath = Pathname.new(ENV["HOME"]) + ".run-jsc-benchmarks"
243
244 $prepare = true
245 $run = true
246 $analyze = []
247
248 # Helpful functions and classes
249
250 def smallUsage
251   puts "Use the --help option to get basic usage information."
252   exit 1
253 end
254
255 def usage
256   puts "run-jsc-benchmarks [options] <vm1> [<vm2> ...]"
257   puts
258   puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
259   puts "benchmarks, and reports detailed statistics.  What makes run-jsc-benchmarks"
260   puts "special is that each benchmark/VM configuration is run in a single VM invocation,"
261   puts "and the invocations are run in random order.  This minimizes systematics due to"
262   puts "one benchmark polluting the running time of another.  The fine-grained"
263   puts "interleaving of VM invocations further minimizes systematics due to changes in"
264   puts "the performance or behavior of your machine."
265   puts 
266   puts "Run-jsc-benchmarks is highly configurable.  You can compare as many VMs as you"
267   puts "like.  You can change the amount of warm-up iterations, number of iterations"
268   puts "executed per VM invocation, and the number of VM invocations per benchmark."
269   puts
270   puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
271   puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
272   puts "the executable and <name> is the name that you would like to give the"
273   puts "configuration for the purposeof reporting.  If no name is given, a generic name"
274   puts "of the form Conf#<n> will be ascribed to the configuration automatically."
275   puts
276   puts "It's also possible to specify per-VM environment variables. For example, you"
277   puts "might specify a VM like Foo:JSC_useJIT=false:/path/to/jsc, in which case the"
278   puts "harness will set the JSC_useJIT environment variable to false just before running"
279   puts "the given VM. Note that the harness will not unset the environment variable, so"
280   puts "you must ensure that your other VMs will use the opposite setting"
281   puts "(JSC_useJIT=true in this case)."
282   puts
283   puts "Options:"
284   puts "--rerun <n>          Set the number of iterations of the benchmark that"
285   puts "                     contribute to the measured run time.  Default is #{$rerun}."
286   puts "--inner <n>          Set the number of inner (per-runtime-invocation)"
287   puts "                     iterations.  Default is #{$inner}."
288   puts "--outer <n>          Set the number of runtime invocations for each benchmark."
289   puts "                     Default is #{$outer}."
290   puts "--warmup <n>         Set the number of warm-up runs per invocation.  Default"
291   puts "                     is #{$warmup}. This has a different effect on different kinds"
292   puts "                     benchmarks. Some benchmarks have no notion of warm-up."
293   puts "--no-ss-warmup       Disable SunSpider-based warm-up runs."
294   puts "--quantum <n>        Set the duration in milliseconds for which an iteration of"
295   puts "                     a throughput benchmark should be run.  Default is #{$quantum}."
296   puts "--timing-mode        Set the way that time is measured.  Possible values"
297   puts "                     are 'preciseTime' and 'date'.  Default is 'preciseTime'."
298   puts "--force-vm-kind      Turn off auto-detection of VM kind, and assume that it is"
299   puts "                     the one specified.  Valid arguments are 'jsc', "
300   puts "                     'DumpRenderTree', or 'WebKitTestRunner'."
301   puts "--force-vm-copy      Force VM builds to be copied to the working directory."
302   puts "                     This may reduce pathologies resulting from path names."
303   puts "--dont-copy-vms      Don't copy VMs even when doing a remote benchmarking run;"
304   puts "                     instead assume that they are already there."
305   puts "--sunspider          Only run SunSpider."
306   puts "--v8-spider          Only run V8."
307   puts "--kraken             Only run Kraken."
308   puts "--js-bench           Only run JSBench."
309   puts "--js-regress         Only run JSRegress."
310   puts "--dsp                Only run DSP."
311   puts "--asm-bench          Only run AsmBench."
312   puts "--browsermark-js     Only run browsermark-js."
313   puts "--browsermark-dom    Only run browsermark-dom."
314   puts "--octane             Only run Octane."
315   puts "--compression-bench  Only run compression bench"
316   puts "                     The default is to run all benchmarks. The above options can"
317   puts "                     be combined to run any subset (so --sunspider --dsp will run"
318   puts "                     both SunSpider and DSP)."
319   puts "--benchmarks         Only run benchmarks matching the given regular expression."
320   puts "--measure-gc         Turn off manual calls to gc(), so that GC time is measured."
321   puts "                     Works best with large values of --inner.  You can also say"
322   puts "                     --measure-gc <conf>, which turns this on for one"
323   puts "                     configuration only."
324   puts "--verbose or -v      Print more stuff."
325   puts "--brief              Print only the final result for each VM."
326   puts "--silent             Don't print progress. This might slightly reduce some"
327   puts "                     performance perturbation."
328   puts "--remote <sshhosts>  Perform performance measurements remotely, on the given"
329   puts "                     SSH host(s). Easiest way to use this is to specify the SSH"
330   puts "                     user@host string. However, you can also supply a comma-"
331   puts "                     separated list of SSH hosts. Alternatively, you can use this"
332   puts "                     option multiple times to specify multiple hosts. This"
333   puts "                     automatically copies the WebKit release builds of the VMs"
334   puts "                     you specified to all of the hosts."
335   puts "--ssh-options        Pass additional options to SSH."
336   puts "--local              Also do a local benchmark run even when doing --remote."
337   puts "--vms                Use a JSON file to specify which VMs to run, as opposed to"
338   puts "                     specifying them on the command line."
339   puts "--prepare-only       Only prepare the runscript (a shell script that"
340   puts "                     invokes the VMs to run benchmarks) but don't run it."
341   puts "--analyze            Only read the output of the runscript but don't do anything"
342   puts "                     else. This requires passing the same arguments that you"
343   puts "                     passed when running --prepare-only."
344   puts "--output-name        Base of the filenames to put results into. Will write a file"
345   puts "                     called <base>_report.txt and <base>.json.  By default this"
346   puts "                     name is automatically synthesized from the machine name,"
347   puts "                     date, set of benchmarks run, and set of configurations."
348   puts "--environment        JSON file that specifies the environment variables that should"
349   puts "                     be used for particular VMs and benchmarks."
350   puts "--config <path>      Specify the path of the configuration file. Defaults to"
351   puts "                     ~/.run-jsc-benchmarks"
352   puts "--dependencies       Additional dependent library paths."
353   puts "--help or -h         Display this message."
354   puts
355   puts "Example:"
356   puts "run-jsc-benchmarks TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
357   exit 1
358 end
359
360 def fail(reason)
361   if reason.respond_to? :backtrace
362     puts "FAILED: #{reason.inspect}"
363     puts "Stack trace:"
364     puts reason.backtrace.join("\n")
365   else
366     puts "FAILED: #{reason.inspect}"
367   end
368   smallUsage
369 end
370
371 def quickFail(r1,r2)
372   $stderr.puts "#{$0}: #{r1}"
373   puts
374   fail(r2)
375 end
376
377 def intArg(argName,arg,min,max)
378   result=arg.to_i
379   unless result.to_s == arg
380     quickFail("Expected an integer value for #{argName}, but got #{arg}.",
381               "Invalid argument for command-line option")
382   end
383   if min and result<min
384     quickFail("Argument for #{argName} cannot be smaller than #{min}.",
385               "Invalid argument for command-line option")
386   end
387   if max and result>max
388     quickFail("Argument for #{argName} cannot be greater than #{max}.",
389               "Invalid argument for command-line option")
390   end
391   result
392 end
393
394 def computeMean(array)
395   sum=0.0
396   array.each {
397     | value |
398     sum += value
399   }
400   sum/array.length
401 end
402
403 def computeGeometricMean(array)
404   sum = 0.0
405   array.each {
406     | value |
407     sum += Math.log(value)
408   }
409   Math.exp(sum * (1.0/array.length))
410 end
411
412 def computeHarmonicMean(array)
413   1.0 / computeMean(array.collect{ | value | 1.0 / value })
414 end
415
416 def computeStdDev(array)
417   case array.length
418   when 0
419     0.0/0.0
420   when 1
421     0.0
422   else
423     begin
424       mean=computeMean(array)
425       sum=0.0
426       array.each {
427         | value |
428         sum += (value-mean)**2
429       }
430       Math.sqrt(sum/(array.length-1))
431     rescue
432       0.0/0.0
433     end
434   end
435 end
436
437 class Array
438   def shuffle!
439     size.downto(1) { |n| push delete_at(rand(n)) }
440     self
441   end
442 end
443
444 def inverseBetaRegularized(n)
445   IBR_LOOKUP[n-1]
446 end
447
448 def numToStr(num, decimalShift)
449   ("%." + (4 + decimalShift).to_s + "f") % (num.to_f)
450 end
451
452 class CantSay
453   def initialize
454   end
455   
456   def shortForm
457     " "
458   end
459   
460   def longForm
461     ""
462   end
463   
464   def to_s
465     ""
466   end
467 end
468   
469 class NoChange
470   attr_reader :amountFaster
471   
472   def initialize(amountFaster)
473     @amountFaster = amountFaster
474   end
475   
476   def shortForm
477     " "
478   end
479   
480   def longForm
481     "  might be #{numToStr(@amountFaster, 0)}x faster"
482   end
483   
484   def to_s
485     if @amountFaster < 1.01
486       ""
487     else
488       longForm
489     end
490   end
491 end
492
493 class Faster
494   attr_reader :amountFaster
495   
496   def initialize(amountFaster)
497     @amountFaster = amountFaster
498   end
499   
500   def shortForm
501     "^"
502   end
503   
504   def longForm
505     "^ definitely #{numToStr(@amountFaster, 0)}x faster"
506   end
507   
508   def to_s
509     longForm
510   end
511 end
512
513 class Slower
514   attr_reader :amountSlower
515   
516   def initialize(amountSlower)
517     @amountSlower = amountSlower
518   end
519   
520   def shortForm
521     "!"
522   end
523   
524   def longForm
525     "! definitely #{numToStr(@amountSlower, 0)}x slower"
526   end
527   
528   def to_s
529     longForm
530   end
531 end
532
533 class MayBeSlower
534   attr_reader :amountSlower
535   
536   def initialize(amountSlower)
537     @amountSlower = amountSlower
538   end
539   
540   def shortForm
541     "?"
542   end
543   
544   def longForm
545     "? might be #{numToStr(@amountSlower, 0)}x slower"
546   end
547   
548   def to_s
549     if @amountSlower < 1.01
550       "?"
551     else
552       longForm
553     end
554   end
555 end
556
557 def jsonSanitize(value)
558   if value.is_a? Fixnum
559     value
560   elsif value.is_a? Float
561     if value.nan? or value.infinite?
562       value.to_s
563     else
564       value
565     end
566   elsif value.is_a? Array
567     value.map{|v| jsonSanitize(v)}
568   elsif value.nil?
569     value
570   else
571     raise "Unrecognized value #{value.inspect}"
572   end
573 end
574
575 class Stats
576   def initialize
577     @array = []
578   end
579   
580   def add(value)
581     if not value or not @array
582       @array = nil
583     elsif value.is_a? Float
584       if value.nan? or value.infinite?
585         @array = nil
586       else
587         @array << value
588       end
589     elsif value.is_a? Stats
590       add(value.array)
591     elsif value.respond_to? :each
592       value.each {
593         | v |
594         add(v)
595       }
596     else
597       @array << value.to_f
598     end
599   end
600   
601   def status
602     if @array
603       :ok
604     else
605       :error
606     end
607   end
608   
609   def error?
610     # TODO: We're probably still not handling this case correctly. 
611     not @array or @array.empty?
612   end
613   
614   def ok?
615     not not @array
616   end
617     
618   def array
619     @array
620   end
621   
622   def sum
623     result=0
624     @array.each {
625       | value |
626       result += value
627     }
628     result
629   end
630   
631   def min
632     @array.min
633   end
634   
635   def max
636     @array.max
637   end
638   
639   def size
640     @array.length
641   end
642   
643   def mean
644     computeMean(array)
645   end
646   
647   def arithmeticMean
648     mean
649   end
650   
651   def stdDev
652     computeStdDev(array)
653   end
654
655   def stdErr
656     stdDev/Math.sqrt(size)
657   end
658   
659   # Computes a 95% Student's t distribution confidence interval
660   def confInt
661     if size < 2
662       0.0/0.0
663     else
664       raise if size > 1000
665       Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
666     end
667   end
668   
669   def lower
670     mean-confInt
671   end
672   
673   def upper
674     mean+confInt
675   end
676   
677   def geometricMean
678     computeGeometricMean(array)
679   end
680   
681   def harmonicMean
682     computeHarmonicMean(array)
683   end
684   
685   def compareTo(other)
686     return CantSay.new unless ok? and other.ok?
687     
688     if upper < other.lower
689       Faster.new(other.mean/mean)
690     elsif lower > other.upper
691       Slower.new(mean/other.mean)
692     elsif mean > other.mean
693       MayBeSlower.new(mean/other.mean)
694     else
695       NoChange.new(other.mean/mean)
696     end
697   end
698   
699   def to_s
700     "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
701   end
702   
703   def jsonMap
704     if ok?
705       {"data"=>jsonSanitize(@array), "mean"=>jsonSanitize(mean), "confInt"=>jsonSanitize(confInt)}
706     else
707       "ERROR"
708     end
709   end
710 end
711
712 def doublePuts(out1,out2,msg)
713   out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
714   out2.puts msg
715 end
716
717 class Benchfile < File
718   @@counter = 0
719   
720   attr_reader :filename, :basename
721   
722   def initialize(name)
723     @basename, @filename = Benchfile.uniqueFilename(name)
724     super(@filename, "w")
725   end
726   
727   def self.uniqueFilename(name)
728     if name.is_a? Array
729       basename = name[0] + @@counter.to_s + name[1]
730     else
731       basename = name + @@counter.to_s
732     end
733     filename = BENCH_DATA_PATH + basename
734     @@counter += 1
735     raise "Benchfile #{filename} already exists" if FileTest.exist?(filename)
736     [basename, filename]
737   end
738   
739   def self.create(name)
740     file = Benchfile.new(name)
741     yield file
742     file.close
743     file.basename
744   end
745 end
746
747 $dataFiles={}
748 def ensureFile(key, filename)
749   unless $dataFiles[key]
750     $dataFiles[key] = Benchfile.create(key) {
751       | outp |
752       doublePuts($stderr,outp,IO::read(filename))
753     }
754   end
755   $dataFiles[key]
756 end
757
758 # Helper for files that cannot be renamed.
759 $absoluteFiles={}
760 def ensureAbsoluteFile(filename, basedir=nil)
761   return if $absoluteFiles[filename]
762   filename = Pathname.new(filename)
763
764   directory = Pathname.new('')
765   if basedir and filename.dirname != basedir
766     remainingPath = filename.dirname
767     while remainingPath != basedir
768       directory = remainingPath.basename + directory
769       remainingPath = remainingPath.dirname
770     end
771     if not $absoluteFiles[directory]
772       cmd = "mkdir -p #{Shellwords.shellescape((BENCH_DATA_PATH + directory).to_s)}"
773       $stderr.puts ">> #{cmd}" if $verbosity >= 2
774       raise unless system(cmd)
775       intermediateDirectory = Pathname.new(directory)
776       while intermediateDirectory.basename.to_s != "."
777         $absoluteFiles[intermediateDirectory] = true
778         intermediateDirectory = intermediateDirectory.dirname
779       end
780     end
781   end
782   
783   cmd = "cp #{Shellwords.shellescape(filename.to_s)} #{Shellwords.shellescape((BENCH_DATA_PATH + directory + filename.basename).to_s)}"
784   $stderr.puts ">> #{cmd}" if $verbosity >= 2
785   raise unless system(cmd)
786   $absoluteFiles[filename] = true
787 end
788
789 # Helper for large benchmarks with lots of files and directories.
790 def ensureBenchmarkFiles(rootdir)
791     toProcess = [rootdir]
792     while not toProcess.empty?
793       currdir = toProcess.pop
794       Dir.foreach(currdir.to_s) {
795         | filename |
796         path = currdir + filename
797         next if filename.match(/^\./)
798         toProcess.push(path) if File.directory?(path.to_s)
799         ensureAbsoluteFile(path, rootdir) if File.file?(path.to_s)
800       }
801     end
802 end
803
804 class JSCommand
805   attr_reader :js, :html
806   def initialize(js, html)
807     @js = js
808     @html = html
809   end
810 end
811
812 def loadCommandForFile(key, filename)
813   file = ensureFile(key, filename)
814   JSCommand.new("load(#{file.inspect});", "<script src=#{file.inspect}></script>")
815 end
816
817 def simpleCommand(command)
818   JSCommand.new(command, "<script type=\"text/javascript\">#{command}</script>")
819 end
820
821 # Benchmark that consists of a single file and must be loaded in its own global object each
822 # time (i.e. run()).
823 class SingleFileTimedBenchmarkParameters
824   attr_reader :benchPath
825   
826   def initialize(benchPath)
827     @benchPath = benchPath
828   end
829   
830   def kind
831     :singleFileTimedBenchmark
832   end
833 end
834
835 # Benchmark that consists of one or more data files that should be loaded globally, followed
836 # by a command to run the benchmark.
837 class MultiFileTimedBenchmarkParameters
838   attr_reader :dataPaths, :command
839
840   def initialize(dataPaths, command)
841     @dataPaths = dataPaths
842     @command = command
843   end
844   
845   def kind
846     :multiFileTimedBenchmark
847   end
848 end
849
850 # Benchmark that consists of one or more data files that should be loaded globally, followed
851 # by a command to run a short tick of the benchmark. The benchmark should be run for as many
852 # ticks as possible, for one quantum (quantum is 1000ms by default).
853 class ThroughputBenchmarkParameters
854   attr_reader :dataPaths, :setUpCommand, :command, :tearDownCommand, :doWarmup, :deterministic, :minimumIterations
855
856   def initialize(dataPaths, setUpCommand, command, tearDownCommand, doWarmup, deterministic, minimumIterations)
857     @dataPaths = dataPaths
858     @setUpCommand = setUpCommand
859     @command = command
860     @tearDownCommand = tearDownCommand
861     @doWarmup = doWarmup
862     @deterministic = deterministic
863     @minimumIterations = minimumIterations
864   end
865   
866   def kind
867     :throughputBenchmark
868   end
869 end
870
871 # Benchmark that can only run in DumpRenderTree or WebKitTestRunner, that has its own callback for reporting
872 # results. Other than that it's just like SingleFileTimedBenchmark.
873 class SingleFileTimedCallbackBenchmarkParameters
874   attr_reader :callbackDecl, :benchPath
875   
876   def initialize(callbackDecl, benchPath)
877     @callbackDecl = callbackDecl
878     @benchPath = benchPath
879   end
880   
881   def kind
882     :singleFileTimedCallbackBenchmark
883   end
884 end
885
886 def emitTimerFunctionCode(file)
887   case $timeMode
888   when :preciseTime
889     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
890     doublePuts($stderr,file,"   return preciseTime()*1000")
891     doublePuts($stderr,file,"}")
892   when :date
893     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
894     doublePuts($stderr,file,"   return Date.now()")
895     doublePuts($stderr,file,"}")
896   else
897     raise
898   end
899 end
900
901 def emitBenchRunCodeFile(name, plan, benchParams)
902   case plan.vm.vmType
903   when :jsc
904     Benchfile.create("bencher") {
905       | file |
906       emitTimerFunctionCode(file)
907       
908       if benchParams.kind == :multiFileTimedBenchmark
909         benchParams.dataPaths.each {
910           | path |
911           doublePuts($stderr,file,"load(#{path.inspect});")
912         }
913         doublePuts($stderr,file,"gc();")
914         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
915         doublePuts($stderr,file,"   var __before = __bencher_curTimeMS();")
916         $rerun.times {
917           doublePuts($stderr,file,"   #{benchParams.command.js}")
918         }
919         doublePuts($stderr,file,"   var __after = __bencher_curTimeMS();")
920         doublePuts($stderr,file,"   if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(__after-__before));");
921         doublePuts($stderr,file,"   gc();") unless plan.vm.shouldMeasureGC
922         doublePuts($stderr,file,"}")
923       elsif benchParams.kind == :throughputBenchmark
924         emitTimerFunctionCode(file)
925         benchParams.dataPaths.each {
926           | path |
927           doublePuts($stderr,file,"load(#{path.inspect});")
928         }
929         doublePuts($stderr,file,"#{benchParams.setUpCommand.js}")
930         if benchParams.doWarmup
931           warmup = $warmup
932         else
933           warmup = 0
934         end
935         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{warmup + $inner}; __bencher_index++) {")
936         doublePuts($stderr,file,"    var __before = __bencher_curTimeMS();")
937         doublePuts($stderr,file,"    var __after = __before;")
938         doublePuts($stderr,file,"    var __runs = 0;")
939         doublePuts($stderr,file,"    var __expected = #{$quantum};")
940         doublePuts($stderr,file,"    while (true) {")
941         $rerun.times {
942           doublePuts($stderr,file,"       #{benchParams.command.js}")
943         }
944         doublePuts($stderr,file,"       __runs++;")
945         doublePuts($stderr,file,"       __after = __bencher_curTimeMS();")
946         if benchParams.deterministic
947           doublePuts($stderr,file,"       if (true) {")
948         else
949           doublePuts($stderr,file,"       if (__after - __before >= __expected) {")
950         end
951         doublePuts($stderr,file,"           if (__runs >= #{benchParams.minimumIterations} || __bencher_index < #{warmup})")
952         doublePuts($stderr,file,"               break;")
953         doublePuts($stderr,file,"           __expected += #{$quantum}")
954         doublePuts($stderr,file,"       }")
955         doublePuts($stderr,file,"    }")
956         doublePuts($stderr,file,"    if (__bencher_index >= #{warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{warmup}) + \": Time: \"+((__after-__before)/__runs));")
957         doublePuts($stderr,file,"}")
958         doublePuts($stderr,file,"#{benchParams.tearDownCommand.js}")
959       else
960         raise unless benchParams.kind == :singleFileTimedBenchmark
961         doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
962         doublePuts($stderr,file,"   var __bencher_before = __bencher_curTimeMS();")
963         $rerun.times {
964           doublePuts($stderr,file,"   run(__bencher_what);")
965         }
966         doublePuts($stderr,file,"   var __bencher_after = __bencher_curTimeMS();")
967         doublePuts($stderr,file,"   return __bencher_after - __bencher_before;")
968         doublePuts($stderr,file,"}")
969         $warmup.times {
970           doublePuts($stderr,file,"__bencher_run(#{benchParams.benchPath.inspect})")
971           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
972         }
973         $inner.times {
974           | innerIndex |
975           doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchParams.benchPath.inspect}));")
976           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
977         }
978       end
979     }
980   when :dumpRenderTree, :webkitTestRunner
981     case $timeMode
982     when :preciseTime
983       curTime = "(testRunner.preciseTime()*1000)"
984     when :date
985       curTime = "(Date.now())"
986     else
987       raise
988     end
989
990     mainCode = Benchfile.create("bencher") {
991       | file |
992       doublePuts($stderr,file,"__bencher_count = 0;")
993       doublePuts($stderr,file,"function __bencher_doNext(result) {")
994       doublePuts($stderr,file,"    if (__bencher_count >= #{$warmup})")
995       doublePuts($stderr,file,"        debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);")
996       doublePuts($stderr,file,"    __bencher_count++;")
997       doublePuts($stderr,file,"    if (__bencher_count < #{$inner+$warmup})")
998       doublePuts($stderr,file,"        __bencher_runImpl(__bencher_doNext);")
999       doublePuts($stderr,file,"    else")
1000       doublePuts($stderr,file,"        quit();")
1001       doublePuts($stderr,file,"}")
1002       doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);")
1003     }
1004     
1005     cssCode = Benchfile.create("bencher-css") {
1006       | file |
1007       doublePuts($stderr,file,".pass {\n    font-weight: bold;\n    color: green;\n}\n.fail {\n    font-weight: bold;\n    color: red;\n}\n\#console {\n    white-space: pre-wrap;\n    font-family: monospace;\n}")
1008     }
1009     
1010     preCode = Benchfile.create("bencher-pre") {
1011       | file |
1012       doublePuts($stderr,file,"if (window.testRunner) {")
1013       doublePuts($stderr,file,"    testRunner.dumpAsText(window.enablePixelTesting);")
1014       doublePuts($stderr,file,"    testRunner.waitUntilDone();")
1015       doublePuts($stderr,file,"}")
1016       doublePuts($stderr,file,"")
1017       doublePuts($stderr,file,"function debug(msg)")
1018       doublePuts($stderr,file,"{")
1019       doublePuts($stderr,file,"    var span = document.createElement(\"span\");")
1020       doublePuts($stderr,file,"    document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace")
1021       doublePuts($stderr,file,"    span.innerHTML = msg + '<br />';")
1022       doublePuts($stderr,file,"}")
1023       doublePuts($stderr,file,"")
1024       doublePuts($stderr,file,"function quit() {")
1025       doublePuts($stderr,file,"    testRunner.notifyDone();")
1026       doublePuts($stderr,file,"}")
1027       doublePuts($stderr,file,"")
1028       doublePuts($stderr,file,"__bencher_continuation=null;")
1029       doublePuts($stderr,file,"")
1030       doublePuts($stderr,file,"function reportResult(result) {")
1031       doublePuts($stderr,file,"    __bencher_continuation(result);")
1032       doublePuts($stderr,file,"}")
1033       if benchParams.kind == :singleFileTimedCallbackBenchmark
1034         doublePuts($stderr,file,"")
1035         doublePuts($stderr,file,benchParams.callbackDecl)
1036       end
1037       doublePuts($stderr,file,"")
1038       doublePuts($stderr,file,"function __bencher_runImpl(continuation) {")
1039       doublePuts($stderr,file,"    function doit() {")
1040       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"\";")
1041       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";")
1042       doublePuts($stderr,file,"        var testFrame = document.getElementById(\"testframe\");")
1043       doublePuts($stderr,file,"        testFrame.contentDocument.open();")
1044       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");")
1045       if benchParams.kind == :throughputBenchmark or benchParams.kind == :multiFileTimedBenchmark
1046         benchParams.dataPaths.each {
1047           | path |
1048           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{path.inspect.inspect[1..-2]}></script>\");")
1049         }
1050       end
1051       if benchParams.kind == :throughputBenchmark
1052         if benchParams.doWarmup
1053           warmup = $warmup
1054         else
1055           warmup = 0
1056         end
1057         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">\");")
1058         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.setUpCommand.js}\");")
1059         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_before = #{curTime};\");")
1060         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_after = __bencher_before;\");")
1061         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_expected = #{$quantum};\");")
1062         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_runs = 0;\");")
1063         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"while (true) {\");")
1064         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    #{benchParams.command.js}\");")
1065         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_runs++;\");")
1066         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_after = #{curTime};\");")
1067         if benchParams.deterministic
1068           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (true) {\");")
1069         else
1070           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (__bencher_after - __bencher_before >= __bencher_expected) {\");")
1071         end
1072         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        if (__bencher_runs >= #{benchParams.minimumIterations} || window.parent.__bencher_count < #{warmup})\");")
1073         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"            break;\");")
1074         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        __bencher_expected += #{$quantum}\");")
1075         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    }\");")
1076         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"}\");")
1077         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.tearDownCommand.js}\");")
1078         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"window.parent.reportResult((__bencher_after - __bencher_before) / __bencher_runs);\");")
1079         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</script>\");")
1080       else
1081         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">var __bencher_before = #{curTime};</script>\");")
1082         if benchParams.kind == :multiFileTimedBenchmark
1083           doublePuts($stderr,file,"        testFrame.contentDocument.write(#{benchParams.command.html.inspect});")
1084         else
1085           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{benchParams.benchPath.inspect.inspect[1..-2]}></script>\");")
1086         end
1087         unless benchParams.kind == :singleFileTimedCallbackBenchmark
1088           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">window.parent.reportResult(#{curTime} - __bencher_before);</script>\");")
1089         end
1090       end
1091       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</body></html>\");")
1092       doublePuts($stderr,file,"        testFrame.contentDocument.close();")
1093       doublePuts($stderr,file,"    }")
1094       doublePuts($stderr,file,"    __bencher_continuation = continuation;")
1095       doublePuts($stderr,file,"    window.setTimeout(doit, 10);")
1096       doublePuts($stderr,file,"}")
1097     }
1098
1099     Benchfile.create(["bencher-htmldoc",".html"]) {
1100       | file |
1101       doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>")
1102     }
1103   else
1104     raise
1105   end
1106 end
1107
1108 def emitBenchRunCode(name, plan, benchParams)
1109   plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchParams), plan)
1110 end
1111
1112 class FileCreator
1113   def initialize(filename)
1114     @filename = filename
1115     @state = :empty
1116   end
1117   
1118   def puts(text)
1119     $script.print "echo #{Shellwords.shellescape(text)}"
1120     if @state == :empty
1121       $script.print " > "
1122       @state = :nonEmpty
1123     else
1124       $script.print " >> "
1125     end
1126     $script.puts "#{Shellwords.shellescape(@filename)}"
1127   end
1128   
1129   def close
1130     if @state == :empty
1131       $script.puts "rm -f #{Shellwords.shellescape(text)}"
1132       $script.puts "touch #{Shellwords.shellescape(text)}"
1133     end
1134   end
1135   
1136   def self.open(filename)
1137     outp = FileCreator.new(filename)
1138     yield outp
1139     outp.close
1140   end
1141 end
1142
1143 def emitSelfContainedBenchRunCode(name, plan, targetFile, configFile, benchmark)
1144   FileCreator.open(configFile) {
1145     | outp |
1146     outp.puts "__bencher_message = \"#{name}: #{plan.vm}: #{plan.iteration}: \";"
1147     outp.puts "__bencher_warmup = #{$warmup};"
1148     outp.puts "__bencher_inner = #{$inner};"
1149     outp.puts "__bencher_benchmark = #{benchmark.to_json};"
1150     case $timeMode
1151     when :preciseTime
1152       outp.puts "__bencher_curTime = (function(){ return testRunner.preciseTime() * 1000; });"
1153     when :date
1154       outp.puts "__bencher_curTime = (function(){ return Date.now(); });"
1155     else
1156       raise
1157     end
1158   }
1159   
1160   plan.vm.emitRunCode(targetFile, plan)
1161 end
1162
1163 def planForDescription(string, plans, benchFullname, vmName, iteration)
1164   raise "Unexpected benchmark full name: #{benchFullname.inspect}, string: #{string.inspect}" unless benchFullname =~ /\//
1165   suiteName = $~.pre_match
1166   return nil if suiteName == "WARMUP"
1167   benchName = $~.post_match
1168   result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration}
1169   raise "Unexpected result dimensions: #{result.inspect}, string: #{string.inspect}" unless result.size == 1
1170   result[0]
1171 end
1172
1173 class ParsedResult
1174   attr_reader :plan, :innerIndex, :time, :result
1175   
1176   def initialize(plan, innerIndex, time)
1177     @plan = plan
1178     @innerIndex = innerIndex
1179     if time == :crashed
1180       @result = :error
1181     else
1182       @time = time
1183       @result = :success
1184     end
1185     
1186     raise unless @plan.is_a? BenchPlan
1187     raise unless @innerIndex.is_a? Integer
1188     raise unless @time.is_a? Numeric or @result == :error
1189   end
1190   
1191   def benchmark
1192     plan.benchmark
1193   end
1194   
1195   def suite
1196     plan.suite
1197   end
1198   
1199   def vm
1200     plan.vm
1201   end
1202   
1203   def outerIndex
1204     plan.iteration
1205   end
1206   
1207   def self.create(plan, innerIndex, time)
1208     if plan
1209       ParsedResult.new(plan, innerIndex, time)
1210     else
1211       nil
1212     end
1213   end
1214   
1215   def self.parse(plans, string)
1216     if string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): Time: /
1217       benchFullname = $1
1218       vmName = $2
1219       outerIndex = $3.to_i
1220       innerIndex = $4.to_i
1221       time = $~.post_match.to_f
1222       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, time)
1223     elsif string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): CRASHED/
1224       benchFullname = $1
1225       vmName = $2
1226       outerIndex = $3.to_i
1227       innerIndex = $4.to_i
1228       time = $~.post_match.to_f
1229       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, :crashed)
1230     else
1231       nil
1232     end
1233   end
1234 end
1235
1236 class VM
1237   @@extraEnvSet = {}
1238   
1239   def initialize(origPath, name, nameKind, svnRevision)
1240     @origPath = origPath.to_s
1241     @path = origPath.to_s
1242     @name = name
1243     @nameKind = nameKind
1244     @extraEnv = {}
1245     
1246     if $forceVMKind
1247       @vmType = $forceVMKind
1248     else
1249       if @origPath =~ /DumpRenderTree$/
1250         @vmType = :dumpRenderTree
1251       elsif @origPath =~ /WebKitTestRunner$/
1252         @vmType = :webkitTestRunner
1253       else
1254         @vmType = :jsc
1255       end
1256     end
1257     
1258     @svnRevision = svnRevision
1259     
1260     # Try to detect information about the VM.
1261     if path =~ /\/WebKitBuild\/(Release|Debug)+\/([a-zA-Z]+)$/
1262       @checkoutPath = $~.pre_match
1263       # FIXME: Use some variant of this: 
1264       # <bdash>   def retrieve_revision
1265       # <bdash>     `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i
1266       # <bdash>   end
1267       unless @svnRevision
1268         begin
1269           Dir.chdir(@checkoutPath) {
1270             $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
1271             IO.popen("svn info", "r") {
1272               | inp |
1273               inp.each_line {
1274                 | line |
1275                 if line =~ /Revision: ([0-9]+)/
1276                   @svnRevision = $1
1277                 end
1278               }
1279             }
1280           }
1281           unless @svnRevision
1282             $stderr.puts "Warning: running svn info for #{name} silently failed."
1283           end
1284         rescue => e
1285           # Failed to detect svn revision.
1286           $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
1287         end
1288       end
1289     else
1290       $stderr.puts "Warning: could not identify checkout location for #{name}"
1291     end
1292     
1293     if @path =~ /\/Release\/([a-zA-Z]+)$/
1294       @libPath, @relativeBinPath = [$~.pre_match+"/Release"], "./#{$1}"
1295     elsif @path =~ /\/Debug\/([a-zA-Z]+)$/
1296       @libPath, @relativeBinPath = [$~.pre_match+"/Debug"], "./#{$1}"
1297     elsif @path =~ /\/Release\/bin\/([a-zA-Z]+)$/
1298       @libPath, @relativeBinPath = [$~.pre_match+"/Release/lib"], "./#{$1}"
1299     elsif @path =~ /\/Debug\/bin\/([a-zA-Z]+)$/
1300       @libPath, @relativeBinPath = [$~.pre_match+"/Debug/lib"], "./#{$1}"
1301     elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/
1302       @libPath = [$~.pre_match + "/Contents/Resources", $~.pre_match + "/Contents/Frameworks"]
1303     elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/
1304       @libPath, @relativeBinPath = [$~.pre_match], $&[1..-1]
1305     elsif @path =~ /(DumpRenderTree|webkitTestRunner|jsc)$/
1306       @libPath, @relativeBinPath = [$~.pre_match+"/"], "./#{$1}"
1307     end
1308     @libPath += $dependencies
1309   end
1310   
1311   def canCopyIntoBenchPath
1312     if @libPath and @relativeBinPath
1313       true
1314     else
1315       false
1316     end
1317   end
1318   
1319   def addExtraEnv(key, val)
1320     @extraEnv[key] = val
1321     @@extraEnvSet[key] = true
1322   end
1323   
1324   def copyIntoBenchPath
1325     raise unless canCopyIntoBenchPath
1326     basename, filename = Benchfile.uniqueFilename("vm")
1327     raise unless Dir.mkdir(filename)
1328     @libPath.each {
1329       | libPathPart |
1330       cmd = "cp -a #{Shellwords.shellescape(libPathPart)}/* #{Shellwords.shellescape(filename.to_s)}"
1331       $stderr.puts ">> #{cmd}" if $verbosity>=2
1332       raise unless system(cmd)
1333     }
1334     @path = "#{basename}/#{@relativeBinPath}"
1335     @libPath = [basename]
1336   end
1337   
1338   def to_s
1339     @name
1340   end
1341   
1342   def name
1343     @name
1344   end
1345   
1346   def shouldMeasureGC
1347     $measureGC == true or ($measureGC == name)
1348   end
1349   
1350   def origPath
1351     @origPath
1352   end
1353   
1354   def path
1355     @path
1356   end
1357   
1358   def nameKind
1359     @nameKind
1360   end
1361   
1362   def vmType
1363     @vmType
1364   end
1365   
1366   def checkoutPath
1367     @checkoutPath
1368   end
1369   
1370   def svnRevision
1371     @svnRevision
1372   end
1373   
1374   def extraEnv
1375     @extraEnv
1376   end
1377   
1378   def printFunction
1379     case @vmType
1380     when :jsc
1381       "print"
1382     when :dumpRenderTree, :webkitTestRunner
1383       "debug"
1384     else
1385       raise @vmType
1386     end
1387   end
1388   
1389   def emitRunCode(fileToRun, plan)
1390     myLibPath = @libPath
1391     myLibPath = [] unless myLibPath
1392     @@extraEnvSet.keys.each {
1393       | key |
1394       $script.puts "unset #{Shellwords.shellescape(key)}"
1395     }
1396     $script.puts "export DYLD_LIBRARY_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1397     $script.puts "export DYLD_FRAMEWORK_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1398     $script.puts "export LD_LIBRARY_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1399
1400     unless myLibPath.empty?
1401         primaryLibPath = myLibPath.first
1402         $script.puts "export TEST_RUNNER_TEST_PLUGIN_PATH=#{Shellwords.shellescape(Pathname.new(primaryLibPath).join('plugins').to_s)}"
1403         $script.puts "export TEST_RUNNER_INJECTED_BUNDLE_FILENAME=#{Shellwords.shellescape(Pathname.new(primaryLibPath).join('libTestRunnerInjectedBundle.so').to_s)}"
1404     end
1405
1406     @extraEnv.each_pair {
1407       | key, val |
1408       $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1409     }
1410     plan.environment.each_pair {
1411         | key, val |
1412         $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1413     }
1414     $script.puts "#{path} #{fileToRun} 2>&1 || {"
1415     $script.puts "    echo " + Shellwords.shellescape("#{name} failed to run!") + " 1>&2"
1416     $inner.times {
1417       | iteration |
1418       $script.puts "    echo " + Shellwords.shellescape("#{plan.prefix}: #{iteration}: CRASHED")
1419     }
1420     $script.puts "}"
1421     plan.environment.keys.each {
1422         | key |
1423         $script.puts "export #{Shellwords.shellescape(key)}="
1424     }
1425   end
1426 end
1427
1428 class StatsAccumulator
1429   def initialize
1430     @stats = []
1431     ($outer*$inner).times {
1432       @stats << Stats.new
1433     }
1434   end
1435   
1436   def statsForIteration(outerIteration, innerIteration)
1437     @stats[outerIteration*$inner + innerIteration]
1438   end
1439   
1440   def stats
1441     result = Stats.new
1442     @stats.each {
1443       | stat |
1444       if stat.ok?
1445         result.add(yield stat)
1446       else
1447         result.add(nil)
1448       end
1449     }
1450     result
1451   end
1452   
1453   def geometricMeanStats
1454     stats {
1455       | stat |
1456       stat.geometricMean
1457     }
1458   end
1459   
1460   def arithmeticMeanStats
1461     stats {
1462       | stat |
1463       stat.arithmeticMean
1464     }
1465   end
1466 end
1467
1468 module Benchmark
1469   attr_accessor :benchmarkSuite
1470   attr_reader :name
1471   
1472   def fullname
1473     benchmarkSuite.name + "/" + name
1474   end
1475   
1476   def to_s
1477     fullname
1478   end
1479   
1480   def weight
1481     1
1482   end
1483   
1484   def weightString
1485     raise unless weight.is_a? Fixnum
1486     raise unless weight >= 1
1487     if weight == 1
1488       ""
1489     else
1490       "x#{weight} "
1491     end
1492   end
1493 end
1494
1495 class SunSpiderBenchmark
1496   include Benchmark
1497   
1498   def initialize(name)
1499     @name = name
1500   end
1501   
1502   def emitRunCode(plan)
1503     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js")))
1504   end
1505 end
1506
1507 class LongSpiderBenchmark
1508   include Benchmark
1509   
1510   def initialize(name)
1511     @name = name
1512   end
1513   
1514   def emitRunCode(plan)
1515     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("LongSpider-#{@name}", "#{LONGSPIDER_PATH}/#{@name}.js")))
1516   end
1517 end
1518
1519 class V8Benchmark
1520   include Benchmark
1521   
1522   def initialize(name)
1523     @name = name
1524   end
1525   
1526   def emitRunCode(plan)
1527     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js")))
1528   end
1529 end
1530
1531 class V8RealBenchmark
1532   include Benchmark
1533   
1534   attr_reader :v8SuiteName
1535   
1536   def initialize(v8SuiteName, name, weight, minimumIterations)
1537     @v8SuiteName = v8SuiteName
1538     @name = name
1539     @weight = weight
1540     @minimumIterations = minimumIterations
1541   end
1542   
1543   def weight
1544     @weight
1545   end
1546   
1547   def emitRunCode(plan)
1548     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(["base", @v8SuiteName, "jsc-#{@name}"].collect{|v| ensureFile("V8Real-#{v}", "#{V8_REAL_PATH}/#{v}.js")}, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, false, @minimumIterations))
1549   end
1550 end
1551
1552 class OctaneBenchmark
1553   include Benchmark
1554   
1555   def initialize(files, name, weight, doWarmup, deterministic, minimumIterations)
1556     @files = files
1557     @name = name
1558     @weight = weight
1559     @doWarmup = doWarmup
1560     @deterministic = deterministic
1561     @minimumIterations = minimumIterations
1562   end
1563   
1564   def weight
1565     @weight
1566   end
1567   
1568   def emitRunCode(plan)
1569     files = []
1570     files += (["base"] + @files).collect {
1571       | v |
1572       ensureFile("Octane-#{v}", "#{OCTANE_PATH}/#{v}.js")
1573     }
1574     files += ["jsc-#{@name}"].collect {
1575       | v |
1576       ensureFile("Octane-#{v}", "#{OCTANE_WRAPPER_PATH}/#{v}.js")
1577     }
1578     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(files, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1579   end
1580 end
1581
1582 class KrakenBenchmark
1583   include Benchmark
1584   
1585   def initialize(name)
1586     @name = name
1587   end
1588   
1589   def emitRunCode(plan)
1590     emitBenchRunCode(fullname, plan, MultiFileTimedBenchmarkParameters.new([ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js")], loadCommandForFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js")))
1591   end
1592 end
1593
1594 class JSBenchBenchmark
1595   include Benchmark
1596   
1597   attr_reader :jsBenchMode
1598   
1599   def initialize(name, jsBenchMode)
1600     @name = name
1601     @jsBenchMode = jsBenchMode
1602   end
1603   
1604   def emitRunCode(plan)
1605     callbackDecl  = "function JSBNG_handleResult(result) {\n"
1606     callbackDecl += "    if (result.error) {\n"
1607     callbackDecl += "        console.log(\"Did not run benchmark correctly!\");\n"
1608     callbackDecl += "        quit();\n"
1609     callbackDecl += "    }\n"
1610     callbackDecl += "    reportResult(result.time);\n"
1611     callbackDecl += "}\n"
1612     emitBenchRunCode(fullname, plan, SingleFileTimedCallbackBenchmarkParameters.new(callbackDecl, ensureFile("JSBench-#{@name}", "#{JSBENCH_PATH}/#{@name}/#{@jsBenchMode}.js")))
1613   end
1614 end
1615
1616 class JSRegressBenchmark
1617   include Benchmark
1618   
1619   def initialize(name)
1620     @name = name
1621   end
1622   
1623   def emitRunCode(plan)
1624     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("JSRegress-#{@name}", "#{JSREGRESS_PATH}/#{@name}.js")))
1625   end
1626 end
1627
1628 class AsmBenchBenchmark
1629   include Benchmark
1630   
1631   def initialize(name)
1632     @name = name
1633   end
1634   
1635   def emitRunCode(plan)
1636     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("AsmBench-#{@name}", "#{ASMBENCH_PATH}/#{@name}.js")))
1637   end
1638 end
1639
1640
1641 class CompressionBenchBenchmark
1642   include Benchmark
1643   
1644   def initialize(files, name, model)
1645     @files = files
1646     @name = name;
1647     @name = name + "-" + model if !model.empty?
1648     @name = @name.gsub(" ", "-").downcase
1649     @scriptName = name
1650     @weight = 1
1651     @doWarmup = true
1652     @deterministic = true
1653     @minimumIterations = 1
1654     @model = model
1655   end
1656   
1657   def weight
1658     @weight
1659   end
1660   
1661   def emitRunCode(plan)
1662     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new((["base"] + @files + ["jsc-#{@scriptName}"]).collect{|v| ensureFile("Compression-#{v}", "#{COMPRESSIONBENCH_PATH}/#{v}.js")}, simpleCommand("jscSetUp('#{@model}');"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1663   end
1664 end
1665
1666 class DSPJSFiltrrBenchmark
1667   include Benchmark
1668   
1669   def initialize(name, filterKey)
1670     @name = name
1671     @filterKey = filterKey
1672   end
1673   
1674   def emitRunCode(plan)
1675     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr.js")
1676     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr_back.jpg")
1677     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-jquery.min.js")
1678     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-bencher.html")
1679     emitSelfContainedBenchRunCode(fullname, plan, "filtrr-bencher.html", "bencher-config.js", @filterKey)
1680   end
1681 end
1682
1683 class DSPJSVP8Benchmark
1684   include Benchmark
1685   
1686   def initialize
1687     @name = "route9-vp8"
1688   end
1689   
1690   def weight
1691     5
1692   end
1693   
1694   def emitRunCode(plan)
1695     ensureBenchmarkFiles(DSPJS_ROUTE9_PATH)
1696     emitSelfContainedBenchRunCode(fullname, plan, "route9-bencher.html", "bencher-config.js", "")
1697   end
1698 end
1699
1700 class DSPStarfieldBenchmark
1701   include Benchmark
1702
1703   def initialize
1704     @name = "starfield"
1705   end
1706   
1707   def weight
1708     5
1709   end
1710
1711   def emitRunCode(plan)
1712     ensureBenchmarkFiles(DSPJS_STARFIELD_PATH)
1713     emitSelfContainedBenchRunCode(fullname, plan, "starfield-bencher.html", "bencher-config.js", "")
1714   end
1715 end
1716
1717 class DSPJSJSLinuxBenchmark
1718   include Benchmark
1719   def initialize
1720     @name = "bellard-jslinux"
1721   end
1722
1723   def weight
1724     5
1725   end
1726
1727   def emitRunCode(plan)
1728     ensureBenchmarkFiles(DSPJS_JSLINUX_PATH)
1729     emitSelfContainedBenchRunCode(fullname, plan, "jslinux-bencher.html", "bencher-config.js", "")
1730   end
1731 end
1732
1733 class DSPJSQuake3Benchmark
1734   include Benchmark
1735
1736   def initialize
1737     @name = "zynaps-quake3"
1738   end
1739
1740   def weight
1741     5
1742   end
1743
1744   def emitRunCode(plan)
1745     ensureBenchmarkFiles(DSPJS_QUAKE3_PATH)
1746     emitSelfContainedBenchRunCode(fullname, plan, "quake-bencher.html", "bencher-config.js", "")
1747   end
1748 end
1749
1750 class DSPJSMandelbrotBenchmark
1751   include Benchmark
1752
1753   def initialize
1754     @name = "zynaps-mandelbrot"
1755   end
1756
1757   def weight
1758     5
1759   end
1760
1761   def emitRunCode(plan)
1762     ensureBenchmarkFiles(DSPJS_MANDELBROT_PATH)
1763     emitSelfContainedBenchRunCode(fullname, plan, "mandelbrot-bencher.html", "bencher-config.js", "")
1764   end
1765 end
1766
1767 class DSPJSAmmoJSASMBenchmark
1768   include Benchmark
1769
1770   def initialize
1771     @name = "ammojs-asm-js"
1772   end
1773
1774   def weight
1775     5
1776   end
1777
1778   def emitRunCode(plan)
1779     ensureBenchmarkFiles(DSPJS_AMMOJS_ASMJS_PATH)
1780     emitSelfContainedBenchRunCode(fullname, plan, "ammo-asmjs-bencher.html", "bencher-config.js", "")
1781   end
1782 end
1783
1784 class DSPJSAmmoJSRegularBenchmark
1785   include Benchmark
1786
1787   def initialize
1788     @name = "ammojs-regular-js"
1789   end
1790
1791   def weight
1792     5
1793   end
1794
1795   def emitRunCode(plan)
1796     ensureBenchmarkFiles(DSPJS_AMMOJS_REGULAR_PATH)
1797     emitSelfContainedBenchRunCode(fullname, plan, "ammo-regular-bencher.html", "bencher-config.js", "")
1798   end
1799 end
1800
1801 class BrowsermarkJSBenchmark
1802   include Benchmark
1803     
1804   def initialize(name)
1805     @name = name
1806   end
1807   
1808   def emitRunCode(plan)
1809     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new([ensureFile(name, "#{BROWSERMARK_JS_PATH}/#{name}/test.js"), ensureFile("browsermark-bencher", "#{BROWSERMARK_JS_PATH}/browsermark-bencher.js")], simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, 32))
1810   end
1811 end
1812
1813 class BrowsermarkDOMBenchmark
1814   include Benchmark
1815     
1816   def initialize(name)
1817     @name = name
1818   end
1819   
1820   def emitRunCode(plan)
1821     ensureBenchmarkFiles(BROWSERMARK_PATH)
1822     emitSelfContainedBenchRunCode(fullname, plan, "tests/benchmarks/dom/#{name}/index.html", "bencher-config.js", name)
1823   end
1824 end
1825
1826 class BenchmarkSuite
1827   def initialize(name, preferredMean, decimalShift)
1828     @name = name
1829     @preferredMean = preferredMean
1830     @benchmarks = []
1831     @subSuites = []
1832     @decimalShift = decimalShift
1833   end
1834   
1835   def name
1836     @name
1837   end
1838   
1839   def to_s
1840     @name
1841   end
1842   
1843   def decimalShift
1844     @decimalShift
1845   end
1846   
1847   def addIgnoringPattern(benchmark)
1848     benchmark.benchmarkSuite = self
1849     @benchmarks << benchmark
1850   end
1851   
1852   def add(benchmark)
1853     if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern
1854         addIgnoringPattern(benchmark)
1855     end
1856   end
1857   
1858   def addSubSuite(subSuite)
1859     @subSuites << subSuite
1860   end
1861   
1862   def benchmarks
1863     @benchmarks
1864   end
1865   
1866   def benchmarkForName(name)
1867     result = @benchmarks.select{|v| v.name == name}
1868     raise unless result.length == 1
1869     result[0]
1870   end
1871   
1872   def hasBenchmark(benchmark)
1873     array = @benchmarks.select{|v| v == benchmark}
1874     raise unless array.length == 1 or array.length == 0
1875     array.length == 1
1876   end
1877   
1878   def subSuites
1879     @subSuites
1880   end
1881   
1882   def suites
1883     [self] + @subSuites
1884   end
1885   
1886   def suitesWithBenchmark(benchmark)
1887     result = [self]
1888     @subSuites.each {
1889       | subSuite |
1890       if subSuite.hasBenchmark(benchmark)
1891         result << subSuite
1892       end
1893     }
1894     result
1895   end
1896   
1897   def empty?
1898     @benchmarks.empty?
1899   end
1900   
1901   def retain_if
1902     @benchmarks.delete_if {
1903       | benchmark |
1904       not yield benchmark
1905     }
1906   end
1907   
1908   def preferredMean
1909     @preferredMean
1910   end
1911   
1912   def computeMean(stat)
1913     if stat.ok?
1914       (stat.send @preferredMean) * (10 ** decimalShift)
1915     else
1916       nil
1917     end
1918   end
1919 end
1920
1921 class BenchRunPlan
1922   def initialize(benchmark, vm, iteration)
1923     @benchmark = benchmark
1924     @vm = vm
1925     @iteration = iteration
1926     @environment = {}
1927     if $environment.has_key?(vm.name)
1928       if $environment[vm.name].has_key?(benchmark.benchmarkSuite.name)
1929         if $environment[vm.name][benchmark.benchmarkSuite.name].has_key?(benchmark.name)
1930           @environment = $environment[vm.name][benchmark.benchmarkSuite.name][benchmark.name]
1931         end
1932       end
1933     end
1934   end
1935   
1936   def benchmark
1937     @benchmark
1938   end
1939   
1940   def suite
1941     @benchmark.benchmarkSuite
1942   end
1943   
1944   def vm
1945     @vm
1946   end
1947   
1948   def iteration
1949     @iteration
1950   end
1951  
1952   def environment
1953     @environment
1954   end
1955  
1956   def prefix
1957     "#{@benchmark.fullname}: #{vm.name}: #{iteration}"
1958   end
1959   
1960   def emitRunCode
1961     @benchmark.emitRunCode(self)
1962   end
1963   
1964   def to_s
1965     benchmark.to_s + "/" + vm.to_s
1966   end
1967 end
1968
1969 class BenchmarkOnVM
1970   def initialize(benchmark, suiteOnVM, subSuitesOnVM)
1971     @benchmark = benchmark
1972     @suiteOnVM = suiteOnVM
1973     @subSuitesOnVM = subSuitesOnVM
1974     @stats = Stats.new
1975   end
1976   
1977   def to_s
1978     "#{@benchmark} on #{@suiteOnVM.vm}"
1979   end
1980   
1981   def benchmark
1982     @benchmark
1983   end
1984   
1985   def vm
1986     @suiteOnVM.vm
1987   end
1988   
1989   def vmStats
1990     @suiteOnVM.vmStats
1991   end
1992   
1993   def suite
1994     @benchmark.benchmarkSuite
1995   end
1996   
1997   def suiteOnVM
1998     @suiteOnVM
1999   end
2000   
2001   def subSuitesOnVM
2002     @subSuitesOnVM
2003   end
2004   
2005   def stats
2006     @stats
2007   end
2008   
2009   def parseResult(result)
2010     raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm
2011     raise unless result.benchmark == @benchmark
2012     @stats.add(result.time)
2013   end
2014 end
2015
2016 class NamedStatsAccumulator < StatsAccumulator
2017   def initialize(name)
2018     super()
2019     @name = name
2020   end
2021   
2022   def reportingName
2023     @name
2024   end
2025 end
2026
2027 class SuiteOnVM < StatsAccumulator
2028   def initialize(vm, vmStats, suite)
2029     super()
2030     @vm = vm
2031     @vmStats = vmStats
2032     @suite = suite
2033     
2034     raise unless @vm.is_a? VM
2035     raise unless @vmStats.is_a? StatsAccumulator
2036     raise unless @suite.is_a? BenchmarkSuite
2037   end
2038   
2039   def to_s
2040     "#{@suite} on #{@vm}"
2041   end
2042   
2043   def suite
2044     @suite
2045   end
2046   
2047   def vm
2048     @vm
2049   end
2050
2051   def reportingName
2052     @vm.name
2053   end
2054   
2055   def vmStats
2056     raise unless @vmStats
2057     @vmStats
2058   end
2059 end
2060
2061 class SubSuiteOnVM < StatsAccumulator
2062   def initialize(vm, suite)
2063     super()
2064     @vm = vm
2065     @suite = suite
2066     raise unless @vm.is_a? VM
2067     raise unless @suite.is_a? BenchmarkSuite
2068   end
2069   
2070   def to_s
2071     "#{@suite} on #{@vm}"
2072   end
2073   
2074   def suite
2075     @suite
2076   end
2077   
2078   def vm
2079     @vm
2080   end
2081   
2082   def reportingName
2083     @vm.name
2084   end
2085 end
2086
2087 class BenchPlan
2088   def initialize(benchmarkOnVM, iteration)
2089     @benchmarkOnVM = benchmarkOnVM
2090     @iteration = iteration
2091   end
2092   
2093   def to_s
2094     "#{@benchmarkOnVM} \##{@iteration+1}"
2095   end
2096   
2097   def benchmarkOnVM
2098     @benchmarkOnVM
2099   end
2100   
2101   def benchmark
2102     @benchmarkOnVM.benchmark
2103   end
2104   
2105   def suite
2106     @benchmarkOnVM.suite
2107   end
2108   
2109   def vm
2110     @benchmarkOnVM.vm
2111   end
2112   
2113   def iteration
2114     @iteration
2115   end
2116   
2117   def parseResult(result)
2118     raise unless result.plan == self
2119     @benchmarkOnVM.parseResult(result)
2120     benchmark.weight.times {
2121       @benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time)
2122       @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time)
2123       @benchmarkOnVM.subSuitesOnVM.each {
2124         | subSuite |
2125         subSuite.statsForIteration(@iteration, result.innerIndex).add(result.time)
2126       }
2127     }
2128   end
2129 end
2130
2131 def lpad(str,chars)
2132   if str.length>chars
2133     str
2134   else
2135     "%#{chars}s"%(str)
2136   end
2137 end
2138
2139 def rpad(str,chars)
2140   while str.length < chars
2141     str+=" "
2142   end
2143   str
2144 end
2145
2146 def center(str,chars)
2147   while str.length<chars
2148     str+=" "
2149     if str.length<chars
2150       str=" "+str
2151     end
2152   end
2153   str
2154 end
2155
2156 def statsToStr(stats, decimalShift)
2157   if stats.error?
2158     lpad(center("ERROR", 10+10+2), 12+10+2)
2159   elsif $inner*$outer == 1
2160     string = numToStr(stats.mean, decimalShift)
2161     raise unless string =~ /\./
2162     left = $~.pre_match
2163     right = $~.post_match
2164     lpad(left, 13 - decimalShift) + "." + rpad(right, 10 + decimalShift)
2165   else
2166     lpad(numToStr(stats.mean, decimalShift), 12) + "+-" + rpad(numToStr(stats.confInt, decimalShift), 10)
2167   end
2168 end
2169
2170 def plural(num)
2171   if num == 1
2172     ""
2173   else
2174     "s"
2175   end
2176 end
2177
2178 def wrap(str, columns)
2179   array = str.split
2180   result = ""
2181   curLine = array.shift
2182   array.each {
2183     | curStr |
2184     if (curLine + " " + curStr).size > columns
2185       result += curLine + "\n"
2186       curLine = curStr
2187     else
2188       curLine += " " + curStr
2189     end
2190   }
2191   result + curLine + "\n"
2192 end
2193   
2194 def runAndGetResults
2195   results = nil
2196   Dir.chdir(BENCH_DATA_PATH) {
2197     $stderr.puts ">> sh ./runscript" if $verbosity >= 2
2198     raise "Script did not complete correctly: #{$?}" unless system("sh ./runscript > runlog")
2199     results = IO::read("runlog")
2200   }
2201   raise unless results
2202   results
2203 end
2204
2205 def parseAndDisplayResults(results)
2206   vmStatses = []
2207   $vms.each {
2208     | vm |
2209     vmStatses << NamedStatsAccumulator.new(vm.name)
2210   }
2211   
2212   suitesOnVMs = []
2213   suitesOnVMsForSuite = {}
2214   subSuitesOnVMsForSubSuite = {}
2215   $suites.each {
2216     | suite |
2217     suitesOnVMsForSuite[suite] = []
2218     suite.subSuites.each {
2219       | subSuite |
2220       subSuitesOnVMsForSubSuite[subSuite] = []
2221     }
2222   }
2223   suitesOnVMsForVM = {}
2224   $vms.each {
2225     | vm |
2226     suitesOnVMsForVM[vm] = []
2227   }
2228   
2229   benchmarksOnVMs = []
2230   benchmarksOnVMsForBenchmark = {}
2231   $benchmarks.each {
2232     | benchmark |
2233     benchmarksOnVMsForBenchmark[benchmark] = []
2234   }
2235   
2236   $vms.each_with_index {
2237     | vm, vmIndex |
2238     vmStats = vmStatses[vmIndex]
2239     $suites.each {
2240       | suite |
2241       suiteOnVM = SuiteOnVM.new(vm, vmStats, suite)
2242       subSuitesOnVM = suite.subSuites.map {
2243         | subSuite |
2244         result = SubSuiteOnVM.new(vm, subSuite)
2245         subSuitesOnVMsForSubSuite[subSuite] << result
2246         result
2247       }
2248       suitesOnVMs << suiteOnVM
2249       suitesOnVMsForSuite[suite] << suiteOnVM
2250       suitesOnVMsForVM[vm] << suiteOnVM
2251       suite.benchmarks.each {
2252         | benchmark |
2253         subSuitesOnVMForThisBenchmark = []
2254         subSuitesOnVM.each {
2255           | subSuiteOnVM |
2256           if subSuiteOnVM.suite.hasBenchmark(benchmark)
2257             subSuitesOnVMForThisBenchmark << subSuiteOnVM
2258           end
2259         }
2260         benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM, subSuitesOnVMForThisBenchmark)
2261         benchmarksOnVMs << benchmarkOnVM
2262         benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
2263       }
2264     }
2265   }
2266   
2267   plans = []
2268   benchmarksOnVMs.each {
2269     | benchmarkOnVM |
2270     $outer.times {
2271       | iteration |
2272       plans << BenchPlan.new(benchmarkOnVM, iteration)
2273     }
2274   }
2275
2276   hostname = nil
2277   hwmodel = nil
2278   results.each_line {
2279     | line |
2280     line.chomp!
2281     if line =~ /HOSTNAME:([^.]+)/
2282       hostname = $1
2283     elsif line =~ /HARDWARE:hw\.model: /
2284       hwmodel = $~.post_match.chomp
2285     else
2286       result = ParsedResult.parse(plans, line)
2287       if result
2288         result.plan.parseResult(result)
2289       end
2290     end
2291   }
2292   
2293   # Compute the geomean of the preferred means of results on a SuiteOnVM
2294   overallResults = []
2295   $vms.each {
2296     | vm |
2297     result = Stats.new
2298     $outer.times {
2299       | outerIndex |
2300       $inner.times {
2301         | innerIndex |
2302         curResult = Stats.new
2303         suitesOnVMsForVM[vm].each {
2304           | suiteOnVM |
2305           # For a given iteration, suite, and VM, compute the suite's preferred mean
2306           # over the data collected for all benchmarks in that suite. We'll have one
2307           # sample per benchmark. For example on V8 this will be the geomean of 1
2308           # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
2309           # splay.
2310           curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
2311         }
2312         
2313         # curResult now holds 1 sample for each of the means computed in the above
2314         # loop. Compute the geomean over this, and store it.
2315         if curResult.ok?
2316           result.add(curResult.geometricMean)
2317         else
2318           result.add(nil)
2319         end
2320       }
2321     }
2322
2323     # $overallResults will have a Stats for each VM. That Stats object will hold
2324     # $inner*$outer geomeans, allowing us to compute the arithmetic mean and
2325     # confidence interval of the geomeans of preferred means. Convoluted, but
2326     # useful and probably sound.
2327     overallResults << result
2328   }
2329   
2330   if $verbosity >= 2
2331     benchmarksOnVMs.each {
2332       | benchmarkOnVM |
2333       $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
2334     }
2335     
2336     $vms.each_with_index {
2337       | vm, vmIndex |
2338       vmStats = vmStatses[vmIndex]
2339       $stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}"
2340       $stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}"
2341     }
2342   end
2343
2344   if $outputName
2345     reportName = $outputName
2346   else
2347     reportName =
2348       (if ($vms.collect {
2349              | vm |
2350              vm.nameKind
2351            }.index :auto)
2352          ""
2353        else
2354          text = $vms.collect {
2355            | vm |
2356            vm.to_s
2357          }.join("_") + "_"
2358          if text.size >= 40
2359            ""
2360          else
2361            text
2362          end
2363        end) +
2364       ($suites.collect {
2365          | suite |
2366          suite.to_s
2367        }.join("")) + "_" +
2368       (if hostname
2369          hostname + "_"
2370        else
2371          ""
2372        end)+
2373       (begin
2374          time = Time.now
2375          "%04d%02d%02d_%02d%02d" %
2376            [ time.year, time.month, time.day,
2377              time.hour, time.min ]
2378        end)
2379   end
2380
2381   unless $brief
2382     puts "Generating benchmark report at #{Dir.pwd}/#{reportName}_report.txt"
2383     puts "And raw data at #{Dir.pwd}/#{reportName}.json"
2384   end
2385   
2386   outp = $stdout
2387   json = {}
2388   begin
2389     outp = File.open(reportName + "_report.txt","w")
2390   rescue => e
2391     $stderr.puts "Error: could not save report to #{reportName}_report.txt: #{e}"
2392     $stderr.puts
2393   end
2394   
2395   def createVMsString
2396     result = ""
2397     result += "   " if $allSuites.size > 1
2398     result += rpad("", $benchpad + $weightpad)
2399     result += " "
2400     $vms.size.times {
2401       | index |
2402       if index != 0
2403         result += " "+NoChange.new(0).shortForm
2404       end
2405       result += lpad(center($vms[index].name, 10+10+2), 12+10+2)
2406     }
2407     result += "    "
2408     if $vms.size >= 3
2409       result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
2410     elsif $vms.size >= 2
2411       result += " "*26
2412     end
2413     result
2414   end
2415   
2416   def andJoin(list)
2417     if list.size == 1
2418       list[0].to_s
2419     elsif list.size == 2
2420       "#{list[0]} and #{list[1]}"
2421     else
2422       "#{list[0..-2].join(', ')}, and #{list[-1]}"
2423     end
2424   end
2425   
2426   json["vms"] = $vms.collect{|v| v.name}
2427   json["suites"] = {}
2428   json["runlog"] = results
2429   
2430   columns = [createVMsString.size, 78].max
2431   
2432   outp.print "Benchmark report for "
2433   outp.print andJoin($suites)
2434   if hostname
2435     outp.print " on #{hostname}"
2436   end
2437   if hwmodel
2438     outp.print " (#{hwmodel})"
2439   end
2440   outp.puts "."
2441   outp.puts
2442   
2443   outp.puts "VMs tested:"
2444   $vms.each {
2445     | vm |
2446     outp.print "\"#{vm.name}\" at #{vm.origPath}"
2447     if vm.svnRevision
2448       outp.print " (r#{vm.svnRevision})"
2449     end
2450     outp.puts
2451     vm.extraEnv.each_pair {
2452       | key, val |
2453       outp.puts "    export #{key}=#{val}"
2454     }
2455   }
2456   
2457   outp.puts
2458   
2459   outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
2460                  "with #{$outer} VM invocation#{plural($outer)} per benchmark."+
2461                  (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
2462                                       "total time of those iterations, for each sample.")
2463                   else "" end)+
2464                  (if $measureGC == true then (" No manual garbage collection invocations were "+
2465                                               "emitted.")
2466                   elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
2467                                          "all VMs except #{$measureGC}.")
2468                   else (" Emitted a call to gc() between sample measurements.") end)+
2469                  (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
2470                                         "began with the very first iteration.")
2471                   else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
2472                         "invocation for warm-up.") end)+
2473                  (case $timeMode
2474                   when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
2475                                           "microsecond-level timing.")
2476                   when :date then (" Used the portable Date.now() method to get millisecond-"+
2477                                    "level timing.")
2478                   else raise end)+
2479                  " Reporting benchmark execution times with 95% confidence "+
2480                  "intervals in milliseconds.",
2481                  columns)
2482   
2483   outp.puts
2484   
2485   def printVMs(outp)
2486     outp.puts createVMsString
2487   end
2488   
2489   def summaryStats(outp, json, accumulators, name, decimalShift, &proc)
2490     resultingJson = {}
2491     outp.print "   " if $allSuites.size > 1
2492     outp.print rpad(name, $benchpad + $weightpad)
2493     outp.print " "
2494     accumulators.size.times {
2495       | index |
2496       if index != 0
2497         outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
2498       end
2499       outp.print statsToStr(accumulators[index].stats(&proc), decimalShift)
2500       resultingJson[accumulators[index].reportingName] = accumulators[index].stats(&proc).jsonMap
2501     }
2502     if accumulators.size>=2
2503       outp.print("    "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm)
2504     end
2505     outp.puts
2506     json[name] = resultingJson
2507   end
2508
2509   def allSummaryStats(outp, json, accumulators, preferredMean, decimalShift)
2510     meanLabel = '<' + preferredMean.to_s.sub(/Mean$/, '') + '>'
2511     summaryStats(outp, json, accumulators, meanLabel, decimalShift) {
2512       | stat |
2513       stat.send(preferredMean)
2514     }
2515   end
2516   
2517   $suites.each {
2518     | suite |
2519     suiteJson = {}
2520     subSuiteJsons = {}
2521     suite.subSuites.each {
2522       | subSuite |
2523       subSuiteJsons[subSuite] = {}
2524     }
2525     
2526     printVMs(outp)
2527     if $allSuites.size > 1
2528       outp.puts(andJoin(suite.suites.map{|v| v.name}) + ":")
2529     else
2530       outp.puts
2531     end
2532     suite.benchmarks.each {
2533       | benchmark |
2534       benchmarkJson = {}
2535       outp.print "   " if $allSuites.size > 1
2536       outp.print rpad(benchmark.name, $benchpad) + rpad(benchmark.weightString, $weightpad)
2537       if benchmark.name.size > $benchNameClip
2538         outp.puts
2539         outp.print "   " if $allSuites.size > 1
2540         outp.print((" " * $benchpad) + (" " * $weightpad))
2541       end
2542       outp.print " "
2543       myConfigs = benchmarksOnVMsForBenchmark[benchmark]
2544       myConfigs.size.times {
2545         | index |
2546         if index != 0
2547           outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
2548         end
2549         outp.print statsToStr(myConfigs[index].stats, suite.decimalShift)
2550         benchmarkJson[myConfigs[index].vm.name] = myConfigs[index].stats.jsonMap
2551       }
2552       if $vms.size>=2
2553         outp.print("    "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
2554       end
2555       outp.puts
2556       suiteJson[benchmark.name] = benchmarkJson
2557       suite.subSuites.each {
2558         | subSuite |
2559         if subSuite.hasBenchmark(benchmark)
2560           subSuiteJsons[subSuite][benchmark.name] = benchmarkJson
2561         end
2562       }
2563     }
2564     outp.puts
2565     unless suite.subSuites.empty?
2566       suite.subSuites.each {
2567         | subSuite |
2568         outp.puts "#{subSuite.name}:"
2569         allSummaryStats(outp, subSuiteJsons[subSuite], subSuitesOnVMsForSubSuite[subSuite], subSuite.preferredMean, subSuite.decimalShift)
2570         outp.puts
2571       }
2572       outp.puts "#{suite.name} including #{andJoin(suite.subSuites.map{|v| v.name})}:"
2573     end
2574     allSummaryStats(outp, suiteJson, suitesOnVMsForSuite[suite], suite.preferredMean, suite.decimalShift)
2575     outp.puts if $allSuites.size > 1
2576     
2577     json["suites"][suite.name] = suiteJson
2578     suite.subSuites.each {
2579       | subSuite |
2580       json["suites"][subSuite.name] = subSuiteJsons[subSuite]
2581     }
2582   }
2583   
2584   if $suites.size > 1
2585     scaledResultJson = {}
2586     printVMs(outp)
2587     outp.puts "Geomean of preferred means:"
2588     outp.print "   "
2589     outp.print rpad("<scaled-result>", $benchpad + $weightpad)
2590     outp.print " "
2591     $vms.size.times {
2592       | index |
2593       if index != 0
2594         outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm
2595       end
2596       outp.print statsToStr(overallResults[index], 0)
2597       scaledResultJson[$vms[index].name] = overallResults[index].jsonMap
2598     }
2599     if overallResults.size>=2
2600       outp.print("    "+overallResults[-1].compareTo(overallResults[0]).longForm)
2601     end
2602     outp.puts
2603     
2604     json["<scaled-result>"] = scaledResultJson
2605   end
2606   outp.puts
2607   
2608   if outp != $stdout
2609     outp.close
2610   end
2611   
2612   if outp != $stdout and not $brief
2613     puts
2614     File.open(reportName + "_report.txt") {
2615       | inp |
2616       puts inp.read
2617     }
2618   end
2619   
2620   if $brief
2621     puts(overallResults.collect{|stats| stats.mean}.join("\t"))
2622     puts(overallResults.collect{|stats| stats.confInt}.join("\t"))
2623   end
2624   
2625   File.open(reportName + ".json", "w") {
2626     | outp |
2627     outp.puts json.to_json
2628   }
2629 end
2630
2631 begin
2632   $sawBenchOptions = false
2633   
2634   def resetBenchOptionsIfNecessary
2635     unless $sawBenchOptions
2636       $includeSunSpider = false
2637       $includeLongSpider = false
2638       $includeV8 = false
2639       $includeKraken = false
2640       $includeJSBench = false
2641       $includeJSRegress = false
2642       $includeAsmBench = false
2643       $includeDSPJS = false
2644       $includeBrowsermarkJS = false
2645       $includeBrowsermarkDOM = false
2646       $includeOctane = false
2647       $includeCompressionBench = false
2648       $sawBenchOptions = true
2649     end
2650   end
2651   
2652   GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
2653                  ['--inner', GetoptLong::REQUIRED_ARGUMENT],
2654                  ['--outer', GetoptLong::REQUIRED_ARGUMENT],
2655                  ['--warmup', GetoptLong::REQUIRED_ARGUMENT],
2656                  ['--no-ss-warmup', GetoptLong::NO_ARGUMENT],
2657                  ['--quantum', GetoptLong::REQUIRED_ARGUMENT],
2658                  ['--minimum', GetoptLong::REQUIRED_ARGUMENT],
2659                  ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
2660                  ['--sunspider', GetoptLong::NO_ARGUMENT],
2661                  ['--longspider', GetoptLong::NO_ARGUMENT],
2662                  ['--v8-spider', GetoptLong::NO_ARGUMENT],
2663                  ['--kraken', GetoptLong::NO_ARGUMENT],
2664                  ['--js-bench', GetoptLong::NO_ARGUMENT],
2665                  ['--js-regress', GetoptLong::NO_ARGUMENT],
2666                  ['--asm-bench', GetoptLong::NO_ARGUMENT],
2667                  ['--dsp', GetoptLong::NO_ARGUMENT],
2668                  ['--browsermark-js', GetoptLong::NO_ARGUMENT],
2669                  ['--browsermark-dom', GetoptLong::NO_ARGUMENT],
2670                  ['--octane', GetoptLong::NO_ARGUMENT],
2671                  ['--compression-bench', GetoptLong::NO_ARGUMENT],
2672                  ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
2673                  ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
2674                  ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
2675                  ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
2676                  ['--dont-copy-vms', GetoptLong::NO_ARGUMENT],
2677                  ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
2678                  ['--brief', GetoptLong::NO_ARGUMENT],
2679                  ['--silent', GetoptLong::NO_ARGUMENT],
2680                  ['--remote', GetoptLong::REQUIRED_ARGUMENT],
2681                  ['--local', GetoptLong::NO_ARGUMENT],
2682                  ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
2683                  ['--slave', GetoptLong::NO_ARGUMENT],
2684                  ['--prepare-only', GetoptLong::NO_ARGUMENT],
2685                  ['--analyze', GetoptLong::REQUIRED_ARGUMENT],
2686                  ['--vms', GetoptLong::REQUIRED_ARGUMENT],
2687                  ['--output-name', GetoptLong::REQUIRED_ARGUMENT],
2688                  ['--environment', GetoptLong::REQUIRED_ARGUMENT],
2689                  ['--dependencies', GetoptLong::REQUIRED_ARGUMENT],
2690                  ['--config', GetoptLong::REQUIRED_ARGUMENT],
2691                  ['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
2692     | opt, arg |
2693     case opt
2694     when '--rerun'
2695       $rerun = intArg(opt,arg,1,nil)
2696     when '--inner'
2697       $inner = intArg(opt,arg,1,nil)
2698     when '--outer'
2699       $outer = intArg(opt,arg,1,nil)
2700     when '--warmup'
2701       $warmup = intArg(opt,arg,0,nil)
2702     when '--no-ss-warmup'
2703       $sunSpiderWarmup = false
2704     when '--quantum'
2705       $quantum = intArg(opt,arg,1,nil)
2706     when '--minimum'
2707       $minimum = intArg(opt,arg,1,nil)
2708     when '--timing-mode'
2709       if arg.upcase == "PRECISETIME"
2710         $timeMode = :preciseTime
2711       elsif arg.upcase == "DATE"
2712         $timeMode = :date
2713       elsif arg.upcase == "AUTO"
2714         $timeMode = :auto
2715       else
2716         quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
2717                   "Invalid argument for command-line option")
2718       end
2719     when '--force-vm-kind'
2720       if arg.upcase == "JSC"
2721         $forceVMKind = :jsc
2722       elsif arg.upcase == "DUMPRENDERTREE"
2723         $forceVMKind = :dumpRenderTree
2724       elsif arg.upcase == "WEBKITTESTRUNNER"
2725         $forceVMKind = :webkitTestRunner
2726       elsif arg.upcase == "AUTO"
2727         $forceVMKind = nil
2728       else
2729         quickFail("Expected 'jsc', 'DumpRenderTree', or 'WebKitTestRunner' for --force-vm-kind, but got '#{arg}'.",
2730                   "Invalid argument for command-line option")
2731       end
2732     when '--force-vm-copy'
2733       $needToCopyVMs = true
2734     when '--dont-copy-vms'
2735       $dontCopyVMs = true
2736     when '--sunspider'
2737       resetBenchOptionsIfNecessary
2738       $includeSunSpider = true
2739     when '--longspider'
2740       resetBenchOptionsIfNecessary
2741       $includeLongSpider = true
2742     when '--v8-spider'
2743       resetBenchOptionsIfNecessary
2744       $includeV8 = true
2745     when '--kraken'
2746       resetBenchOptionsIfNecessary
2747       $includeKraken = true
2748     when '--js-bench'
2749       resetBenchOptionsIfNecessary
2750       $includeJSBench = true
2751     when '--js-regress'
2752       resetBenchOptionsIfNecessary
2753       $includeJSRegress = true
2754     when '--asm-bench'
2755       resetBenchOptionsIfNecessary
2756       $includeAsmBench = true
2757     when '--dsp'
2758       resetBenchOptionsIfNecessary
2759       $includeDSPJS = true
2760     when '--browsermark-js'
2761       resetBenchOptionsIfNecessary
2762       $includeBrowsermarkJS = true
2763     when '--browsermark-dom'
2764       resetBenchOptionsIfNecessary
2765       $includeBrowsermarkDOM = true
2766     when '--octane'
2767       resetBenchOptionsIfNecessary
2768       $includeOctane = true
2769     when '--compression-bench'
2770       resetBenchOptionsIfNecessary
2771       $includeCompressionBench = true
2772     when '--benchmarks'
2773       $benchmarkPattern = Regexp.new(arg)
2774     when '--measure-gc'
2775       if arg == ''
2776         $measureGC = true
2777       else
2778         $measureGC = arg
2779       end
2780     when '--verbose'
2781       $verbosity += 1
2782     when '--brief'
2783       $brief = true
2784     when '--silent'
2785       $silent = true
2786     when '--remote'
2787       $remoteHosts += arg.split(',')
2788       $needToCopyVMs = true
2789     when '--ssh-options'
2790       $sshOptions << arg
2791     when '--local'
2792       $alsoLocal = true
2793     when '--prepare-only'
2794       $run = false
2795     when '--analyze'
2796       $prepare = false
2797       $run = false
2798       $analyze << arg
2799     when '--output-name'
2800       $outputName = arg
2801     when '--vms'
2802       JSON::parse(IO::read(arg)).each {
2803         | vmDescription |
2804         path = Pathname.new(vmDescription["path"]).realpath
2805         if vmDescription["name"]
2806           name = vmDescription["name"]
2807           nameKind = :given
2808         else
2809           name = "Conf\##{$vms.length+1}"
2810           nameKind = :auto
2811         end
2812         vm = VM.new(path, name, nameKind, nil)
2813         if vmDescription["env"]
2814           vmDescription["env"].each_pair {
2815             | key, val |
2816             vm.addExtraEnv(key, val)
2817           }
2818         end
2819         $vms << vm
2820       }
2821     when '--environment'
2822       $environment = JSON::parse(IO::read(arg))
2823     when '--dependencies'
2824       $dependencies.push(Pathname.new(arg).realpath)
2825     when '--config'
2826       $configPath = Pathname.new(arg)
2827     when '--help'
2828       usage
2829     else
2830       raise "bad option: #{opt}"
2831     end
2832   }
2833   
2834   # Figure out the configuration
2835   if $configPath.file?
2836     config = JSON::parse(IO::read($configPath.to_s))
2837   else
2838     config = {}
2839   end
2840
2841   def pathname_if_exist config
2842     if config
2843       Pathname.new config
2844     else
2845       config
2846     end
2847   end
2848
2849   OCTANE_PATH = pathname_if_exist(config["OctanePath"])
2850   BROWSERMARK_PATH = pathname_if_exist(config["BrowserMarkPath"])
2851   BROWSERMARK_JS_PATH = pathname_if_exist(config["BrowserMarkJSPath"])
2852   BROWSERMARK_DOM_PATH = pathname_if_exist(config["BrowserMarkDOMPath"])
2853   ASMBENCH_PATH = pathname_if_exist(config["AsmBenchPath"])
2854   COMPRESSIONBENCH_PATH = pathname_if_exist(config["CompressionBenchPath"])
2855   DSPJS_FILTRR_PATH = pathname_if_exist(config["DSPJSFiltrrPath"])
2856   DSPJS_ROUTE9_PATH = pathname_if_exist(config["DSPJSRoute9Path"])
2857   DSPJS_STARFIELD_PATH = pathname_if_exist(config["DSPJSStarfieldPath"])
2858   DSPJS_QUAKE3_PATH = pathname_if_exist(config["DSPJSQuake3Path"])
2859   DSPJS_MANDELBROT_PATH = pathname_if_exist(config["DSPJSMandelbrotPath"])
2860   DSPJS_JSLINUX_PATH = pathname_if_exist(config["DSPJSLinuxPath"])
2861   DSPJS_AMMOJS_ASMJS_PATH = pathname_if_exist(config["DSPJSAmmoJSAsmJSPath"])
2862   DSPJS_AMMOJS_REGULAR_PATH = pathname_if_exist(config["DSPJSAmmoJSRegularPath"])
2863   JSBENCH_PATH = pathname_if_exist(config["JSBenchPath"])
2864   KRAKEN_PATH = pathname_if_exist(config["KrakenPath"])
2865   
2866   # If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option.
2867   if $dontCopyVMs
2868     $needToCopyVMs = false
2869   end
2870   
2871   ARGV.each {
2872     | vm |
2873     if vm =~ /([a-zA-Z0-9_ .]+):/
2874       name = $1
2875       nameKind = :given
2876       vm = $~.post_match
2877     else
2878       name = "Conf\##{$vms.length+1}"
2879       nameKind = :auto
2880     end
2881     envs = []
2882     while vm =~ /([a-zA-Z0-9_]+)=([a-zA-Z0-9_:.\/-]+):/
2883       envs << [$1, $2]
2884       vm = $~.post_match
2885     end
2886     $stderr.puts "#{name}: #{vm}" if $verbosity >= 1
2887     vm = VM.new(Pathname.new(vm).realpath, name, nameKind, nil)
2888     envs.each {
2889       | pair |
2890       vm.addExtraEnv(pair[0], pair[1])
2891     }
2892     $vms << vm
2893   }
2894   
2895   if $vms.empty?
2896     quickFail("Please specify at least on configuraiton on the command line.",
2897               "Insufficient arguments")
2898   end
2899   
2900   $vms.each {
2901     | vm |
2902     if vm.vmType == :jsc
2903       $allDRT = false
2904     end
2905   }
2906   
2907   SUNSPIDER = BenchmarkSuite.new("SunSpider", :arithmeticMean, 0)
2908   WARMUP = BenchmarkSuite.new("WARMUP", :arithmeticMean, 0)
2909   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
2910    "access-fannkuch", "access-nbody", "access-nsieve",
2911    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
2912    "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
2913    "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
2914    "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
2915    "string-base64", "string-fasta", "string-tagcloud",
2916    "string-unpack-code", "string-validate-input"].each {
2917     | name |
2918     SUNSPIDER.add SunSpiderBenchmark.new(name)
2919     WARMUP.addIgnoringPattern SunSpiderBenchmark.new(name)
2920   }
2921
2922   LONGSPIDER = BenchmarkSuite.new("LongSpider", :geometricMean, 0)
2923   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
2924    "access-fannkuch", "access-nbody", "access-nsieve",
2925    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-nsieve-bits",
2926    "controlflow-recursive", "crypto-aes", "crypto-md5", "crypto-sha1",
2927    "date-format-tofte", "date-format-xparb", "hash-map", "math-cordic",
2928    "math-partial-sums", "math-spectral-norm", "string-base64",
2929    "string-fasta", "string-tagcloud"].each {
2930     | name |
2931     LONGSPIDER.add LongSpiderBenchmark.new(name)
2932   }
2933
2934   V8 = BenchmarkSuite.new("V8Spider", :geometricMean, 0)
2935   ["crypto", "deltablue", "earley-boyer", "raytrace",
2936    "regexp", "richards", "splay"].each {
2937     | name |
2938     V8.add V8Benchmark.new(name)
2939   }
2940
2941   OCTANE = BenchmarkSuite.new("Octane", :geometricMean, 1)
2942   [[["crypto"], "encrypt", 1, true, false, 32],
2943    [["crypto"], "decrypt", 1, true, false, 32],
2944    [["deltablue"], "deltablue", 2, true, false, 32],
2945    [["earley-boyer"], "earley", 1, true, false, 32],
2946    [["earley-boyer"], "boyer", 1, true, false, 32],
2947    [["navier-stokes"], "navier-stokes", 2, true, false, 16],
2948    [["raytrace"], "raytrace", 2, true, false, 32],
2949    [["richards"], "richards", 2, true, false, 32],
2950    [["splay"], "splay", 2, true, false, 32],
2951    [["regexp"], "regexp", 2, true, false, 16],
2952    [["pdfjs"], "pdfjs", 2, false, false, 4],
2953    [["mandreel"], "mandreel", 2, false, false, 4],
2954    [["gbemu-part1", "gbemu-part2"], "gbemu", 2, false, false, 4],
2955    [["code-load"], "closure", 1, false, false, 16],
2956    [["code-load"], "jquery", 1, false, false, 16],
2957    [["box2d"], "box2d", 2, false, false, 8],
2958    [["zlib", "zlib-data"], "zlib", 2, false, true, 3],
2959    [["typescript", "typescript-input", "typescript-compiler"], "typescript", 2, false, true, 1]].each {
2960     | args |
2961     OCTANE.add OctaneBenchmark.new(*args)
2962   }
2963
2964   KRAKEN = BenchmarkSuite.new("Kraken", :arithmeticMean, -1)
2965   ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
2966    "audio-oscillator", "imaging-darkroom", "imaging-desaturate",
2967    "imaging-gaussian-blur", "json-parse-financial",
2968    "json-stringify-tinderbox", "stanford-crypto-aes",
2969    "stanford-crypto-ccm", "stanford-crypto-pbkdf2",
2970    "stanford-crypto-sha256-iterative"].each {
2971     | name |
2972     KRAKEN.add KrakenBenchmark.new(name)
2973   }
2974   
2975   JSBENCH = BenchmarkSuite.new("JSBench", :arithmeticMean, 0)
2976   [["amazon-chrome", "urem"], ["amazon-chrome-win", "urem"], ["amazon-firefox", "urm"],
2977    ["amazon-firefox-win", "urm"], ["amazon-safari", "urem"], ["facebook-chrome", "urem"],
2978    ["facebook-chrome-win", "urem"], ["facebook-firefox", "urem"],
2979    ["facebook-firefox-win", "urem"], ["facebook-safari", "urem"],
2980    ["google-chrome", "urem"], ["google-chrome-win", "urem"], ["google-firefox", "uem"],
2981    ["google-firefox-win", "urem"], ["google-safari", "urem"], ["twitter-chrome", "urem"],
2982    ["twitter-chrome-win", "rem"], ["twitter-firefox", "urem"],
2983    ["twitter-firefox-win", "urem"], ["twitter-safari", "urem"], ["yahoo-chrome", "urem"],
2984    ["yahoo-chrome-win", "urem"], ["yahoo-firefox", "urem"], ["yahoo-firefox-win", "urem"],
2985    ["yahoo-safari", "urem"]].each {
2986     | nameAndMode |
2987     JSBENCH.add JSBenchBenchmark.new(*nameAndMode)
2988   }
2989   
2990   JSREGRESS = BenchmarkSuite.new("JSRegress", :geometricMean, 0)
2991   Dir.foreach(JSREGRESS_PATH) {
2992     | filename |
2993     if filename =~ /\.js$/
2994       name = $~.pre_match
2995       JSREGRESS.add JSRegressBenchmark.new(name)
2996     end
2997   }
2998   
2999   ASMBENCH = BenchmarkSuite.new("AsmBench", :geometricMean, 0)
3000   if ASMBENCH_PATH
3001     Dir.foreach(ASMBENCH_PATH) {
3002       | filename |
3003       if filename =~ /\.js$/
3004         name = $~.pre_match
3005         ASMBENCH.add AsmBenchBenchmark.new(name)
3006       end
3007     }
3008   end
3009
3010   COMPRESSIONBENCH = BenchmarkSuite.new("CompressionBench", :geometricMean, 0)
3011   [[["huffman", "compression-data"], "huffman", ""],
3012    [["arithmetic", "compression-data"], "arithmetic", "Simple"],
3013    [["arithmetic", "compression-data"], "arithmetic", "Precise"],
3014    [["arithmetic", "compression-data"], "arithmetic", "Complex Precise"],
3015    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 0"],
3016    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 1"],
3017    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 2"],
3018    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 1"],
3019    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 2"],
3020    [["lz-string", "compression-data"], "lz-string", ""]
3021   ].each {
3022     | args |
3023     COMPRESSIONBENCH.add CompressionBenchBenchmark.new(*args)
3024   }
3025
3026   DSPJS = BenchmarkSuite.new("DSP", :geometricMean, 0)
3027   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-posterize-tint", "e2")
3028   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-contrast-sat-bright", "e5")
3029   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-sat-adj-contr-mult", "e7")
3030   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-blur-overlay-sat-contr", "e8")
3031   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sat-blur-mult-sharpen-contr", "e9")
3032   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sepia-bias", "e10")
3033   DSPJS.add DSPJSVP8Benchmark.new
3034   DSPJS.add DSPStarfieldBenchmark.new
3035   DSPJS.add DSPJSJSLinuxBenchmark.new
3036   DSPJS.add DSPJSQuake3Benchmark.new
3037   DSPJS.add DSPJSMandelbrotBenchmark.new
3038   DSPJS.add DSPJSAmmoJSASMBenchmark.new
3039   DSPJS.add DSPJSAmmoJSRegularBenchmark.new
3040
3041   BROWSERMARK_JS = BenchmarkSuite.new("BrowsermarkJS", :geometricMean, 1)
3042   ["array_blur", "array_weighted", "string_chat", "string_filter", "string_weighted"].each {
3043     | name |
3044       BROWSERMARK_JS.add BrowsermarkJSBenchmark.new(name)
3045   }
3046   
3047   BROWSERMARK_DOM = BenchmarkSuite.new("BrowsermarkDOM", :geometricMean, 1)
3048   ["advanced_search", "create_source", "dynamic_create", "search"].each {
3049     | name |
3050       BROWSERMARK_DOM.add BrowsermarkDOMBenchmark.new(name)
3051   }
3052
3053   $suites = []
3054   
3055   if $includeSunSpider and not SUNSPIDER.empty?
3056     $suites << SUNSPIDER
3057   end
3058   
3059   if $includeLongSpider and not LONGSPIDER.empty?
3060     $suites << LONGSPIDER
3061   end
3062   
3063   if $includeV8 and not V8.empty?
3064     $suites << V8
3065   end
3066   
3067   if $includeOctane and not OCTANE.empty?
3068     if OCTANE_PATH
3069       $suites << OCTANE
3070     else
3071       $stderr.puts "Warning: refusing to run Octane because \"OctanePath\" isn't set in #{$configPath}."
3072     end
3073   end
3074   
3075   if $includeKraken and not KRAKEN.empty?
3076     if KRAKEN_PATH
3077       $suites << KRAKEN
3078     else
3079       $stderr.puts "Warning: refusing to run Kraken because \"KrakenPath\" isn't set in #{$configPath}."
3080     end
3081   end
3082   
3083   if $includeJSBench and not JSBENCH.empty?
3084     if $allDRT
3085       if JSBENCH_PATH
3086         $suites << JSBENCH
3087       else
3088         $stderr.puts "Warning: refusing to run JSBench because \"JSBenchPath\" isn't set in #{$configPath}"
3089       end
3090     else
3091       $stderr.puts "Warning: refusing to run JSBench because not all VMs are DumpRenderTree or WebKitTestRunner."
3092     end
3093   end
3094   
3095   if $includeJSRegress and not JSREGRESS.empty?
3096     $suites << JSREGRESS
3097   end
3098   
3099   if $includeAsmBench and not ASMBENCH.empty?
3100     if ASMBENCH_PATH
3101       $suites << ASMBENCH
3102     else
3103       $stderr.puts "Warning: refusing to run AsmBench because \"AsmBenchPath\" isn't set in #{$configPath}."
3104     end
3105   end
3106   
3107   if $includeDSPJS and not DSPJS.empty?
3108     if $allDRT
3109       if DSPJS_FILTRR_PATH and DSPJS_ROUTE9_PATH and DSPJS_STARFIELD_PATH and DSPJS_QUAKE3_PATH and DSPJS_MANDELBROT_PATH and DSPJS_JSLINUX_PATH and DSPJS_AMMOJS_ASMJS_PATH and DSPJS_AMMOJS_REGULAR_PATH
3110         $suites << DSPJS
3111       else
3112         $stderr.puts "Warning: refusing to run DSPJS because one of the following isn't set in #{$configPath}: \"DSPJSFiltrrPath\", \"DSPJSRoute9Path\", \"DSPJSStarfieldPath\", \"DSPJSQuake3Path\", \"DSPJSMandelbrotPath\", \"DSPJSLinuxPath\", \"DSPJSAmmoJSAsmJSPath\", \"DSPJSAmmoJSRegularPath\"."
3113       end
3114     else
3115       $stderr.puts "Warning: refusing to run DSPJS because not all VMs are DumpRenderTree or WebKitTestRunner."
3116     end
3117   end
3118   
3119   if $includeBrowsermarkJS and not BROWSERMARK_JS.empty?
3120     if BROWSERMARK_PATH and BROWSERMARK_JS_PATH
3121       $suites << BROWSERMARK_JS
3122     else
3123       $stderr.puts "Warning: refusing to run Browsermark-JS because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\" or \"BrowserMarkJSPath\"."
3124     end
3125   end
3126
3127   if $includeBrowsermarkDOM and not BROWSERMARK_DOM.empty?
3128     if $allDRT
3129       if BROWSERMARK_PATH and BROWSERMARK_JS_PATH and BROWSERMARK_DOM_PATH
3130         $suites << BROWSERMARK_DOM
3131       else
3132         $stderr.puts "Warning: refusing to run Browsermark-DOM because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\", \"BrowserMarkJSPath\", or \"BrowserMarkDOMPath\"."
3133       end
3134     else
3135       $stderr.puts "Warning: refusing to run Browsermark-DOM because not all VMs are DumpRenderTree or WebKitTestRunner."
3136     end
3137   end
3138
3139   if $includeCompressionBench and not COMPRESSIONBENCH.empty?
3140     if COMPRESSIONBENCH_PATH
3141       $suites << COMPRESSIONBENCH
3142     else
3143       $stderr.puts "Warning: refusing to run CompressionBench because \"CompressionBenchPath\" isn't set in #{$configPath}"
3144     end
3145   end
3146
3147   $allSuites = $suites.map{|v| v.suites}.flatten(1)
3148   
3149   $benchmarks = []
3150   $suites.each {
3151     | suite |
3152     $benchmarks += suite.benchmarks
3153   }
3154   
3155   if $suites.empty? or $benchmarks.empty?
3156     $stderr.puts "No benchmarks found.  Bailing out."
3157     exit 1
3158   end
3159   
3160   if $outer*$inner == 1
3161     $stderr.puts "Warning: will only collect one sample per benchmark/VM.  Confidence interval calculation will fail."
3162   end
3163   
3164   $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
3165   
3166   $runPlans = []
3167   $vms.each {
3168     | vm |
3169     $benchmarks.each {
3170       | benchmark |
3171       $outer.times {
3172         | iteration |
3173         $runPlans << BenchRunPlan.new(benchmark, vm, iteration)
3174       }
3175     }
3176   }
3177   
3178   $runPlans.shuffle!
3179   
3180   if $sunSpiderWarmup
3181     warmupPlans = []
3182     $vms.each {
3183       | vm |
3184       WARMUP.benchmarks.each {
3185         | benchmark |
3186         warmupPlans << BenchRunPlan.new(benchmark, vm, 0)
3187       }
3188     }
3189     
3190     $runPlans = warmupPlans.shuffle + $runPlans
3191   end
3192   
3193   $suitepad = $suites.collect {
3194     | suite |
3195     suite.to_s.size
3196   }.max + 1
3197   
3198   $planpad = $runPlans.collect {
3199     | plan |
3200     plan.to_s.size
3201   }.max + 1
3202   
3203   maxBenchNameLength =
3204     ($benchmarks + ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
3205     | benchmark |
3206     if benchmark.respond_to? :name
3207       benchmark.name.size
3208     else
3209       benchmark.size
3210     end
3211   }.max
3212   $benchNameClip = 40
3213   $benchpad = [maxBenchNameLength, $benchNameClip].min + 1
3214   
3215   $weightpad = $benchmarks.collect {
3216     | benchmark |
3217     benchmark.weightString.size
3218   }.max
3219
3220   $vmpad = $vms.collect {
3221     | vm |
3222     vm.to_s.size
3223   }.max + 1
3224   
3225   $analyze.each_with_index {
3226     | filename, index |
3227     if index >= 1
3228       puts
3229     end
3230     parseAndDisplayResults(IO::read(filename))
3231   }
3232   
3233   if not $prepare and not $run
3234     exit 0
3235   end
3236   
3237   if FileTest.exist? BENCH_DATA_PATH
3238     cmd = "rm -rf #{BENCH_DATA_PATH}"
3239     $stderr.puts ">> #{cmd}" if $verbosity >= 2
3240     raise unless system cmd
3241   end
3242   
3243   Dir.mkdir BENCH_DATA_PATH
3244   
3245   if $needToCopyVMs
3246     canCopyIntoBenchPath = true
3247     $vms.each {
3248       | vm |
3249       canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath
3250     }
3251     
3252     if canCopyIntoBenchPath
3253       $vms.each {
3254         | vm |
3255         $stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..."
3256         vm.copyIntoBenchPath
3257       }
3258       $stderr.puts "All VMs are in place."
3259     else
3260       $stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it."
3261     end
3262   end
3263   
3264   if $measureGC and $measureGC != true
3265     found = false
3266     $vms.each {
3267       | vm |
3268       if vm.name == $measureGC
3269         found = true
3270       end
3271     }
3272     unless found
3273       $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
3274     end
3275   end
3276   
3277   if $prepare
3278     File.open("#{BENCH_DATA_PATH}/runscript", "w") {
3279       | file |
3280       file.puts "echo \"HOSTNAME:\\c\""
3281       file.puts "hostname"
3282       file.puts "echo"
3283       file.puts "echo \"HARDWARE:\\c\""
3284       file.puts "/usr/sbin/sysctl hw.model"
3285       file.puts "echo"
3286       file.puts "set -e"
3287       $script = file
3288       $runPlans.each_with_index {
3289         | plan, idx |
3290         if $verbosity == 0 and not $silent
3291           text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s
3292           text2 = plan.to_s
3293           file.puts("echo " + Shellwords.shellescape("\r#{text1} #{rpad(text2,$planpad)}") + "\"\\c\" 1>&2")
3294           file.puts("echo " + Shellwords.shellescape("\r#{text1} #{text2}") + "\"\\c\" 1>&2")
3295         end
3296         plan.emitRunCode
3297       }
3298       if $verbosity == 0 and not $silent
3299         file.puts("echo " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}") + "\"\\c\" 1>&2")
3300         file.puts("echo " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size}") + " 1>&2")
3301       end
3302     }
3303   end
3304   
3305   if $run
3306     unless $remoteHosts.empty?
3307       $stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0
3308       Dir.chdir(TEMP_PATH) {
3309         cmd = "tar -czf payload.tar.gz benchdata"
3310         $stderr.puts ">> #{cmd}" if $verbosity>=2
3311         raise unless system(cmd)
3312       }
3313       
3314       def grokHost(host)
3315         if host =~ /:([0-9]+)$/
3316           "-p " + $1 + " " + Shellwords.shellescape($~.pre_match)
3317         else
3318           Shellwords.shellescape(host)
3319         end
3320       end
3321       
3322       def sshRead(host, command)
3323         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3324         $stderr.puts ">> #{cmd}" if $verbosity>=2
3325         result = ""
3326         IO.popen(cmd, "r") {
3327           | inp |
3328           inp.each_line {
3329             | line |
3330             $stderr.puts "#{host}: #{line}" if $verbosity>=2
3331             result += line
3332           }
3333         }
3334         raise "#{$?}" unless $?.success?
3335         result
3336       end
3337       
3338       def sshWrite(host, command, data)
3339         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3340         $stderr.puts ">> #{cmd}" if $verbosity>=2
3341         IO.popen(cmd, "w") {
3342           | outp |
3343           outp.write(data)
3344         }
3345         raise "#{$?}" unless $?.success?
3346       end
3347       
3348       $remoteHosts.each {
3349         | host |
3350         $stderr.puts "Sending benchmark payload to #{host}..." if $verbosity==0
3351         
3352         remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"]
3353         raise unless remoteTempPath
3354         
3355         sshWrite(host, "cd #{Shellwords.shellescape(remoteTempPath)} && rm -rf benchdata && tar -xz", IO::read("#{TEMP_PATH}/payload.tar.gz"))
3356         
3357         $stderr.puts "Running on #{host}..." if $verbosity==0
3358         
3359         parseAndDisplayResults(sshRead(host, "cd #{Shellwords.shellescape(remoteTempPath + '/benchdata')} && sh runscript"))
3360       }
3361     end
3362     
3363     if not $remoteHosts.empty? and $alsoLocal
3364       $stderr.puts "Running locally..."
3365     end
3366     
3367     if $remoteHosts.empty? or $alsoLocal
3368       parseAndDisplayResults(runAndGetResults)
3369     end
3370   end
3371   
3372   if $prepare and not $run and $analyze.empty?
3373     puts wrap("Benchmarking script and data are in #{BENCH_DATA_PATH}. You can run "+
3374               "the benchmarks and get the results by doing:", 78)
3375     puts
3376     puts "cd #{BENCH_DATA_PATH}"
3377     puts "sh runscript > results.txt"
3378     puts
3379     puts wrap("Then you can analyze the results by running bencher with the same arguments "+
3380               "as now, but replacing --prepare-only with --analyze results.txt.", 78)
3381   end
3382 rescue => e
3383   fail(e)
3384 end
3385