Tools/Scripts/run-jsc-benchmarks

   1 #!/usr/bin/env ruby
   2
   3 # Copyright (C) 2011, 2012, 2013, 2014 Apple Inc. All rights reserved.
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions
   7 # are met:
   8 # 1. Redistributions of source code must retain the above copyright
   9 #    notice, this list of conditions and the following disclaimer.
  10 # 2. Redistributions in binary form must reproduce the above copyright
  11 #    notice, this list of conditions and the following disclaimer in the
  12 #    documentation and/or other materials provided with the distribution.
  13 #
  14 # THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
  15 # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  16 # THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  17 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
  18 # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  19 # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  20 # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  21 # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  22 # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  23 # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  24 # THE POSSIBILITY OF SUCH DAMAGE.
  25
  26 require 'rubygems'
  27
  28 require 'getoptlong'
  29 require 'pathname'
  30 require 'shellwords'
  31 require 'socket'
  32
  33 begin
  34   require 'json'
  35 rescue LoadError => e
  36   $stderr.puts "It does not appear that you have the 'json' package installed.  Try running 'sudo gem install json'."
  37   exit 1
  38 end
  39
  40 SCRIPT_PATH = Pathname.new(__FILE__).realpath
  41
  42 raise unless SCRIPT_PATH.dirname.basename.to_s == "Scripts"
  43 raise unless SCRIPT_PATH.dirname.dirname.basename.to_s == "Tools"
  44
  45 OPENSOURCE_PATH = SCRIPT_PATH.dirname.dirname.dirname
  46
  47 SUNSPIDER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "SunSpider" + "tests" + "sunspider-1.0"
  48 LONGSPIDER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "LongSpider"
  49 V8_PATH = OPENSOURCE_PATH + "PerformanceTests" + "SunSpider" + "tests" + "v8-v6"
  50 JSREGRESS_PATH = OPENSOURCE_PATH + "LayoutTests" + "js" + "regress" + "script-tests"
  51 OCTANE_WRAPPER_PATH = OPENSOURCE_PATH + "PerformanceTests" + "Octane" + "wrappers"
  52
  53 TEMP_PATH = OPENSOURCE_PATH + "BenchmarkTemp"
  54
  55 if TEMP_PATH.exist?
  56   raise unless TEMP_PATH.directory?
  57 else
  58   Dir.mkdir(TEMP_PATH)
  59 end
  60
  61 BENCH_DATA_PATH = TEMP_PATH + "benchdata"
  62
  63 IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933,
  64             0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684,
  65             0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227,
  66             0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674,
  67             0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652,
  68             0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604,
  69             0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338,
  70             0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538,
  71             0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185,
  72             0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694,
  73             0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244,
  74             0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067,
  75             0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323,
  76             0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129,
  77             0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574,
  78             0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722,
  79             0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624,
  80             0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321,
  81             0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843,
  82             0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217,
  83             0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464,
  84             0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599,
  85             0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638,
  86             0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592,
  87             0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471,
  88             0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283,
  89             0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037,
  90             0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738,
  91             0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481,
  92             0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085,
  93             0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651,
  94             0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183,
  95             0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683,
  96             0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154,
  97             0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599,
  98             0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019,
  99             0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417,
 100             0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795,
 101             0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153,
 102             0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494,
 103             0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818,
 104             0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128,
 105             0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423,
 106             0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704,
 107             0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974,
 108             0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232,
 109             0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479,
 110             0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716,
 111             0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943,
 112             0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192,
 113             0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401,
 114             0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963,
 115             0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822,
 116             0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007,
 117             0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186,
 118             0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358,
 119             0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525,
 120             0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686,
 121             0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841,
 122             0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991,
 123             0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137,
 124             0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278,
 125             0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414,
 126             0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547,
 127             0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675,
 128             0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799,
 129             0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192,
 130             0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037,
 131             0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151,
 132             0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262,
 133             0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369,
 134             0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489,
 135             0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259,
 136             0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689,
 137             0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785,
 138             0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879,
 139             0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297,
 140             0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059,
 141             0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145,
 142             0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242,
 143             0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324,
 144             0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404,
 145             0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483,
 146             0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559,
 147             0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634,
 148             0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707,
 149             0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778,
 150             0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848,
 151             0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916,
 152             0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983,
 153             0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057,
 154             0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121,
 155             0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183,
 156             0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244,
 157             0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304,
 158             0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363,
 159             0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442,
 160             0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476,
 161             0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539,
 162             0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946,
 163             0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653,
 164             0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712,
 165             0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762,
 166             0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812,
 167             0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867,
 168             0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915,
 169             0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962,
 170             0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007,
 171             0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059,
 172             0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103,
 173             0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146,
 174             0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189,
 175             0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231,
 176             0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272,
 177             0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312,
 178             0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352,
 179             0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391,
 180             0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429,
 181             0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467,
 182             0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504,
 183             0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546,
 184             0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582,
 185             0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617,
 186             0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652,
 187             0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686,
 188             0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724,
 189             0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757,
 190             0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579,
 191             0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822,
 192             0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853,
 193             0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885,
 194             0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915,
 195             0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595,
 196             0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979,
 197             0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013,
 198             0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041,
 199             0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074,
 200             0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102,
 201             0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129,
 202             0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156,
 203             0.99616, 0.996164]
 204
 205 # Run-time configuration parameters (can be set with command-line options)
 206
 207 $rerun=1
 208 $inner=1
 209 $warmup=1
 210 $outer=4
 211 $quantum=1000
 212 $includeSunSpider=true
 213 $includeLongSpider=true
 214 $includeV8=true
 215 $includeKraken=true
 216 $includeJSBench=true
 217 $includeJSRegress=true
 218 $includeAsmBench=true
 219 $includeDSPJS=true
 220 $includeBrowsermarkJS=false
 221 $includeBrowsermarkDOM=false
 222 $includeOctane=true
 223 $includeCompressionBench = true
 224 $measureGC=false
 225 $benchmarkPattern=nil
 226 $verbosity=0
 227 $timeMode=:preciseTime
 228 $forceVMKind=nil
 229 $brief=false
 230 $silent=false
 231 $remoteHosts=[]
 232 $alsoLocal=false
 233 $sshOptions=[]
 234 $vms = []
 235 $environment = {}
 236 $needToCopyVMs = false
 237 $dontCopyVMs = false
 238 $allDRT = true
 239 $outputName = nil
 240 $sunSpiderWarmup = true
 241 $configPath = Pathname.new(ENV["HOME"]) + ".run-jsc-benchmarks"
 242
 243 $prepare = true
 244 $run = true
 245 $analyze = []
 246
 247 # Helpful functions and classes
 248
 249 def smallUsage
 250   puts "Use the --help option to get basic usage information."
 251   exit 1
 252 end
 253
 254 def usage
 255   puts "run-jsc-benchmarks [options] <vm1> [<vm2> ...]"
 256   puts
 257   puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
 258   puts "benchmarks, and reports detailed statistics.  What makes run-jsc-benchmarks"
 259   puts "special is that each benchmark/VM configuration is run in a single VM invocation,"
 260   puts "and the invocations are run in random order.  This minimizes systematics due to"
 261   puts "one benchmark polluting the running time of another.  The fine-grained"
 262   puts "interleaving of VM invocations further minimizes systematics due to changes in"
 263   puts "the performance or behavior of your machine."
 264   puts
 265   puts "Run-jsc-benchmarks is highly configurable.  You can compare as many VMs as you"
 266   puts "like.  You can change the amount of warm-up iterations, number of iterations"
 267   puts "executed per VM invocation, and the number of VM invocations per benchmark."
 268   puts
 269   puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
 270   puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
 271   puts "the executable and <name> is the name that you would like to give the"
 272   puts "configuration for the purposeof reporting.  If no name is given, a generic name"
 273   puts "of the form Conf#<n> will be ascribed to the configuration automatically."
 274   puts
 275   puts "It's also possible to specify per-VM environment variables. For example, you"
 276   puts "might specify a VM like Foo:JSC_useJIT=false:/path/to/jsc, in which case the"
 277   puts "harness will set the JSC_useJIT environment variable to false just before running"
 278   puts "the given VM. Note that the harness will not unset the environment variable, so"
 279   puts "you must ensure that your other VMs will use the opposite setting"
 280   puts "(JSC_useJIT=true in this case)."
 281   puts
 282   puts "Options:"
 283   puts "--rerun <n>          Set the number of iterations of the benchmark that"
 284   puts "                     contribute to the measured run time.  Default is #{$rerun}."
 285   puts "--inner <n>          Set the number of inner (per-runtime-invocation)"
 286   puts "                     iterations.  Default is #{$inner}."
 287   puts "--outer <n>          Set the number of runtime invocations for each benchmark."
 288   puts "                     Default is #{$outer}."
 289   puts "--warmup <n>         Set the number of warm-up runs per invocation.  Default"
 290   puts "                     is #{$warmup}. This has a different effect on different kinds"
 291   puts "                     benchmarks. Some benchmarks have no notion of warm-up."
 292   puts "--no-ss-warmup       Disable SunSpider-based warm-up runs."
 293   puts "--quantum <n>        Set the duration in milliseconds for which an iteration of"
 294   puts "                     a throughput benchmark should be run.  Default is #{$quantum}."
 295   puts "--timing-mode        Set the way that time is measured.  Possible values"
 296   puts "                     are 'preciseTime' and 'date'.  Default is 'preciseTime'."
 297   puts "--force-vm-kind      Turn off auto-detection of VM kind, and assume that it is"
 298   puts "                     the one specified.  Valid arguments are 'jsc', "
 299   puts "                     'DumpRenderTree', or 'WebKitTestRunner'."
 300   puts "--force-vm-copy      Force VM builds to be copied to the working directory."
 301   puts "                     This may reduce pathologies resulting from path names."
 302   puts "--dont-copy-vms      Don't copy VMs even when doing a remote benchmarking run;"
 303   puts "                     instead assume that they are already there."
 304   puts "--sunspider          Only run SunSpider."
 305   puts "--v8-spider          Only run V8."
 306   puts "--kraken             Only run Kraken."
 307   puts "--js-bench           Only run JSBench."
 308   puts "--js-regress         Only run JSRegress."
 309   puts "--dsp                Only run DSP."
 310   puts "--asm-bench          Only run AsmBench."
 311   puts "--browsermark-js     Only run browsermark-js."
 312   puts "--browsermark-dom    Only run browsermark-dom."
 313   puts "--octane             Only run Octane."
 314   puts "--compression-bench  Only run compression bench"
 315   puts "                     The default is to run all benchmarks. The above options can"
 316   puts "                     be combined to run any subset (so --sunspider --dsp will run"
 317   puts "                     both SunSpider and DSP)."
 318   puts "--benchmarks         Only run benchmarks matching the given regular expression."
 319   puts "--measure-gc         Turn off manual calls to gc(), so that GC time is measured."
 320   puts "                     Works best with large values of --inner.  You can also say"
 321   puts "                     --measure-gc <conf>, which turns this on for one"
 322   puts "                     configuration only."
 323   puts "--verbose or -v      Print more stuff."
 324   puts "--brief              Print only the final result for each VM."
 325   puts "--silent             Don't print progress. This might slightly reduce some"
 326   puts "                     performance perturbation."
 327   puts "--remote <sshhosts>  Perform performance measurements remotely, on the given"
 328   puts "                     SSH host(s). Easiest way to use this is to specify the SSH"
 329   puts "                     user@host string. However, you can also supply a comma-"
 330   puts "                     separated list of SSH hosts. Alternatively, you can use this"
 331   puts "                     option multiple times to specify multiple hosts. This"
 332   puts "                     automatically copies the WebKit release builds of the VMs"
 333   puts "                     you specified to all of the hosts."
 334   puts "--ssh-options        Pass additional options to SSH."
 335   puts "--local              Also do a local benchmark run even when doing --remote."
 336   puts "--vms                Use a JSON file to specify which VMs to run, as opposed to"
 337   puts "                     specifying them on the command line."
 338   puts "--prepare-only       Only prepare the runscript (a shell script that"
 339   puts "                     invokes the VMs to run benchmarks) but don't run it."
 340   puts "--analyze            Only read the output of the runscript but don't do anything"
 341   puts "                     else. This requires passing the same arguments that you"
 342   puts "                     passed when running --prepare-only."
 343   puts "--output-name        Base of the filenames to put results into. Will write a file"
 344   puts "                     called <base>_report.txt and <base>.json.  By default this"
 345   puts "                     name is automatically synthesized from the machine name,"
 346   puts "                     date, set of benchmarks run, and set of configurations."
 347   puts "--environment        JSON file that specifies the environment variables that should"
 348   puts "                     be used for particular VMs and benchmarks."
 349   puts "--config <path>      Specify the path of the configuration file. Defaults to"
 350   puts "                     ~/.run-jsc-benchmarks"
 351   puts "--help or -h         Display this message."
 352   puts
 353   puts "Example:"
 354   puts "run-jsc-benchmarks TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
 355   exit 1
 356 end
 357
 358 def fail(reason)
 359   if reason.respond_to? :backtrace
 360     puts "FAILED: #{reason.inspect}"
 361     puts "Stack trace:"
 362     puts reason.backtrace.join("\n")
 363   else
 364     puts "FAILED: #{reason.inspect}"
 365   end
 366   smallUsage
 367 end
 368
 369 def quickFail(r1,r2)
 370   $stderr.puts "#{$0}: #{r1}"
 371   puts
 372   fail(r2)
 373 end
 374
 375 def intArg(argName,arg,min,max)
 376   result=arg.to_i
 377   unless result.to_s == arg
 378     quickFail("Expected an integer value for #{argName}, but got #{arg}.",
 379               "Invalid argument for command-line option")
 380   end
 381   if min and result<min
 382     quickFail("Argument for #{argName} cannot be smaller than #{min}.",
 383               "Invalid argument for command-line option")
 384   end
 385   if max and result>max
 386     quickFail("Argument for #{argName} cannot be greater than #{max}.",
 387               "Invalid argument for command-line option")
 388   end
 389   result
 390 end
 391
 392 def computeMean(array)
 393   sum=0.0
 394   array.each {
 395     | value |
 396     sum += value
 397   }
 398   sum/array.length
 399 end
 400
 401 def computeGeometricMean(array)
 402   sum = 0.0
 403   array.each {
 404     | value |
 405     sum += Math.log(value)
 406   }
 407   Math.exp(sum * (1.0/array.length))
 408 end
 409
 410 def computeHarmonicMean(array)
 411   1.0 / computeMean(array.collect{ | value | 1.0 / value })
 412 end
 413
 414 def computeStdDev(array)
 415   case array.length
 416   when 0
 417     0.0/0.0
 418   when 1
 419     0.0
 420   else
 421     begin
 422       mean=computeMean(array)
 423       sum=0.0
 424       array.each {
 425         | value |
 426         sum += (value-mean)**2
 427       }
 428       Math.sqrt(sum/(array.length-1))
 429     rescue
 430       0.0/0.0
 431     end
 432   end
 433 end
 434
 435 class Array
 436   def shuffle!
 437     size.downto(1) { |n| push delete_at(rand(n)) }
 438     self
 439   end
 440 end
 441
 442 def inverseBetaRegularized(n)
 443   IBR_LOOKUP[n-1]
 444 end
 445
 446 def numToStr(num, decimalShift)
 447   ("%." + (4 + decimalShift).to_s + "f") % (num.to_f)
 448 end
 449
 450 class CantSay
 451   def initialize
 452   end
 453
 454   def shortForm
 455     " "
 456   end
 457
 458   def longForm
 459     ""
 460   end
 461
 462   def to_s
 463     ""
 464   end
 465 end
 466
 467 class NoChange
 468   attr_reader :amountFaster
 469
 470   def initialize(amountFaster)
 471     @amountFaster = amountFaster
 472   end
 473
 474   def shortForm
 475     " "
 476   end
 477
 478   def longForm
 479     "  might be #{numToStr(@amountFaster, 0)}x faster"
 480   end
 481
 482   def to_s
 483     if @amountFaster < 1.01
 484       ""
 485     else
 486       longForm
 487     end
 488   end
 489 end
 490
 491 class Faster
 492   attr_reader :amountFaster
 493
 494   def initialize(amountFaster)
 495     @amountFaster = amountFaster
 496   end
 497
 498   def shortForm
 499     "^"
 500   end
 501
 502   def longForm
 503     "^ definitely #{numToStr(@amountFaster, 0)}x faster"
 504   end
 505
 506   def to_s
 507     longForm
 508   end
 509 end
 510
 511 class Slower
 512   attr_reader :amountSlower
 513
 514   def initialize(amountSlower)
 515     @amountSlower = amountSlower
 516   end
 517
 518   def shortForm
 519     "!"
 520   end
 521
 522   def longForm
 523     "! definitely #{numToStr(@amountSlower, 0)}x slower"
 524   end
 525
 526   def to_s
 527     longForm
 528   end
 529 end
 530
 531 class MayBeSlower
 532   attr_reader :amountSlower
 533
 534   def initialize(amountSlower)
 535     @amountSlower = amountSlower
 536   end
 537
 538   def shortForm
 539     "?"
 540   end
 541
 542   def longForm
 543     "? might be #{numToStr(@amountSlower, 0)}x slower"
 544   end
 545
 546   def to_s
 547     if @amountSlower < 1.01
 548       "?"
 549     else
 550       longForm
 551     end
 552   end
 553 end
 554
 555 def jsonSanitize(value)
 556   if value.is_a? Fixnum
 557     value
 558   elsif value.is_a? Float
 559     if value.nan? or value.infinite?
 560       value.to_s
 561     else
 562       value
 563     end
 564   elsif value.is_a? Array
 565     value.map{|v| jsonSanitize(v)}
 566   elsif value.nil?
 567     value
 568   else
 569     raise "Unrecognized value #{value.inspect}"
 570   end
 571 end
 572
 573 class Stats
 574   def initialize
 575     @array = []
 576   end
 577
 578   def add(value)
 579     if not value or not @array
 580       @array = nil
 581     elsif value.is_a? Float
 582       if value.nan? or value.infinite?
 583         @array = nil
 584       else
 585         @array << value
 586       end
 587     elsif value.is_a? Stats
 588       add(value.array)
 589     elsif value.respond_to? :each
 590       value.each {
 591         | v |
 592         add(v)
 593       }
 594     else
 595       @array << value.to_f
 596     end
 597   end
 598
 599   def status
 600     if @array
 601       :ok
 602     else
 603       :error
 604     end
 605   end
 606
 607   def error?
 608     # TODO: We're probably still not handling this case correctly.
 609     not @array or @array.empty?
 610   end
 611
 612   def ok?
 613     not not @array
 614   end
 615
 616   def array
 617     @array
 618   end
 619
 620   def sum
 621     result=0
 622     @array.each {
 623       | value |
 624       result += value
 625     }
 626     result
 627   end
 628
 629   def min
 630     @array.min
 631   end
 632
 633   def max
 634     @array.max
 635   end
 636
 637   def size
 638     @array.length
 639   end
 640
 641   def mean
 642     computeMean(array)
 643   end
 644
 645   def arithmeticMean
 646     mean
 647   end
 648
 649   def stdDev
 650     computeStdDev(array)
 651   end
 652
 653   def stdErr
 654     stdDev/Math.sqrt(size)
 655   end
 656
 657   # Computes a 95% Student's t distribution confidence interval
 658   def confInt
 659     if size < 2
 660       0.0/0.0
 661     else
 662       raise if size > 1000
 663       Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
 664     end
 665   end
 666
 667   def lower
 668     mean-confInt
 669   end
 670
 671   def upper
 672     mean+confInt
 673   end
 674
 675   def geometricMean
 676     computeGeometricMean(array)
 677   end
 678
 679   def harmonicMean
 680     computeHarmonicMean(array)
 681   end
 682
 683   def compareTo(other)
 684     return CantSay.new unless ok? and other.ok?
 685
 686     if upper < other.lower
 687       Faster.new(other.mean/mean)
 688     elsif lower > other.upper
 689       Slower.new(mean/other.mean)
 690     elsif mean > other.mean
 691       MayBeSlower.new(mean/other.mean)
 692     else
 693       NoChange.new(other.mean/mean)
 694     end
 695   end
 696
 697   def to_s
 698     "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
 699   end
 700
 701   def jsonMap
 702     if ok?
 703       {"data"=>jsonSanitize(@array), "mean"=>jsonSanitize(mean), "confInt"=>jsonSanitize(confInt)}
 704     else
 705       "ERROR"
 706     end
 707   end
 708 end
 709
 710 def doublePuts(out1,out2,msg)
 711   out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
 712   out2.puts msg
 713 end
 714
 715 class Benchfile < File
 716   @@counter = 0
 717
 718   attr_reader :filename, :basename
 719
 720   def initialize(name)
 721     @basename, @filename = Benchfile.uniqueFilename(name)
 722     super(@filename, "w")
 723   end
 724
 725   def self.uniqueFilename(name)
 726     if name.is_a? Array
 727       basename = name[0] + @@counter.to_s + name[1]
 728     else
 729       basename = name + @@counter.to_s
 730     end
 731     filename = BENCH_DATA_PATH + basename
 732     @@counter += 1
 733     raise "Benchfile #{filename} already exists" if FileTest.exist?(filename)
 734     [basename, filename]
 735   end
 736
 737   def self.create(name)
 738     file = Benchfile.new(name)
 739     yield file
 740     file.close
 741     file.basename
 742   end
 743 end
 744
 745 $dataFiles={}
 746 def ensureFile(key, filename)
 747   unless $dataFiles[key]
 748     $dataFiles[key] = Benchfile.create(key) {
 749       | outp |
 750       doublePuts($stderr,outp,IO::read(filename))
 751     }
 752   end
 753   $dataFiles[key]
 754 end
 755
 756 # Helper for files that cannot be renamed.
 757 $absoluteFiles={}
 758 def ensureAbsoluteFile(filename, basedir=nil)
 759   return if $absoluteFiles[filename]
 760   filename = Pathname.new(filename)
 761
 762   directory = Pathname.new('')
 763   if basedir and filename.dirname != basedir
 764     remainingPath = filename.dirname
 765     while remainingPath != basedir
 766       directory = remainingPath.basename + directory
 767       remainingPath = remainingPath.dirname
 768     end
 769     if not $absoluteFiles[directory]
 770       cmd = "mkdir -p #{Shellwords.shellescape((BENCH_DATA_PATH + directory).to_s)}"
 771       $stderr.puts ">> #{cmd}" if $verbosity >= 2
 772       raise unless system(cmd)
 773       intermediateDirectory = Pathname.new(directory)
 774       while intermediateDirectory.basename.to_s != "."
 775         $absoluteFiles[intermediateDirectory] = true
 776         intermediateDirectory = intermediateDirectory.dirname
 777       end
 778     end
 779   end
 780
 781   cmd = "cp #{Shellwords.shellescape(filename.to_s)} #{Shellwords.shellescape((BENCH_DATA_PATH + directory + filename.basename).to_s)}"
 782   $stderr.puts ">> #{cmd}" if $verbosity >= 2
 783   raise unless system(cmd)
 784   $absoluteFiles[filename] = true
 785 end
 786
 787 # Helper for large benchmarks with lots of files and directories.
 788 def ensureBenchmarkFiles(rootdir)
 789     toProcess = [rootdir]
 790     while not toProcess.empty?
 791       currdir = toProcess.pop
 792       Dir.foreach(currdir.to_s) {
 793         | filename |
 794         path = currdir + filename
 795         next if filename.match(/^\./)
 796         toProcess.push(path) if File.directory?(path.to_s)
 797         ensureAbsoluteFile(path, rootdir) if File.file?(path.to_s)
 798       }
 799     end
 800 end
 801
 802 class JSCommand
 803   attr_reader :js, :html
 804   def initialize(js, html)
 805     @js = js
 806     @html = html
 807   end
 808 end
 809
 810 def loadCommandForFile(key, filename)
 811   file = ensureFile(key, filename)
 812   JSCommand.new("load(#{file.inspect});", "<script src=#{file.inspect}></script>")
 813 end
 814
 815 def simpleCommand(command)
 816   JSCommand.new(command, "<script type=\"text/javascript\">#{command}</script>")
 817 end
 818
 819 # Benchmark that consists of a single file and must be loaded in its own global object each
 820 # time (i.e. run()).
 821 class SingleFileTimedBenchmarkParameters
 822   attr_reader :benchPath
 823
 824   def initialize(benchPath)
 825     @benchPath = benchPath
 826   end
 827
 828   def kind
 829     :singleFileTimedBenchmark
 830   end
 831 end
 832
 833 # Benchmark that consists of one or more data files that should be loaded globally, followed
 834 # by a command to run the benchmark.
 835 class MultiFileTimedBenchmarkParameters
 836   attr_reader :dataPaths, :command
 837
 838   def initialize(dataPaths, command)
 839     @dataPaths = dataPaths
 840     @command = command
 841   end
 842
 843   def kind
 844     :multiFileTimedBenchmark
 845   end
 846 end
 847
 848 # Benchmark that consists of one or more data files that should be loaded globally, followed
 849 # by a command to run a short tick of the benchmark. The benchmark should be run for as many
 850 # ticks as possible, for one quantum (quantum is 1000ms by default).
 851 class ThroughputBenchmarkParameters
 852   attr_reader :dataPaths, :setUpCommand, :command, :tearDownCommand, :doWarmup, :deterministic, :minimumIterations
 853
 854   def initialize(dataPaths, setUpCommand, command, tearDownCommand, doWarmup, deterministic, minimumIterations)
 855     @dataPaths = dataPaths
 856     @setUpCommand = setUpCommand
 857     @command = command
 858     @tearDownCommand = tearDownCommand
 859     @doWarmup = doWarmup
 860     @deterministic = deterministic
 861     @minimumIterations = minimumIterations
 862   end
 863
 864   def kind
 865     :throughputBenchmark
 866   end
 867 end
 868
 869 # Benchmark that can only run in DumpRenderTree or WebKitTestRunner, that has its own callback for reporting
 870 # results. Other than that it's just like SingleFileTimedBenchmark.
 871 class SingleFileTimedCallbackBenchmarkParameters
 872   attr_reader :callbackDecl, :benchPath
 873
 874   def initialize(callbackDecl, benchPath)
 875     @callbackDecl = callbackDecl
 876     @benchPath = benchPath
 877   end
 878
 879   def kind
 880     :singleFileTimedCallbackBenchmark
 881   end
 882 end
 883
 884 def emitTimerFunctionCode(file)
 885   case $timeMode
 886   when :preciseTime
 887     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
 888     doublePuts($stderr,file,"   return preciseTime()*1000")
 889     doublePuts($stderr,file,"}")
 890   when :date
 891     doublePuts($stderr,file,"function __bencher_curTimeMS() {")
 892     doublePuts($stderr,file,"   return Date.now()")
 893     doublePuts($stderr,file,"}")
 894   else
 895     raise
 896   end
 897 end
 898
 899 def emitBenchRunCodeFile(name, plan, benchParams)
 900   case plan.vm.vmType
 901   when :jsc
 902     Benchfile.create("bencher") {
 903       | file |
 904       emitTimerFunctionCode(file)
 905
 906       if benchParams.kind == :multiFileTimedBenchmark
 907         benchParams.dataPaths.each {
 908           | path |
 909           doublePuts($stderr,file,"load(#{path.inspect});")
 910         }
 911         doublePuts($stderr,file,"gc();")
 912         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
 913         doublePuts($stderr,file,"   var __before = __bencher_curTimeMS();")
 914         $rerun.times {
 915           doublePuts($stderr,file,"   #{benchParams.command.js}")
 916         }
 917         doublePuts($stderr,file,"   var __after = __bencher_curTimeMS();")
 918         doublePuts($stderr,file,"   if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(__after-__before));");
 919         doublePuts($stderr,file,"   gc();") unless plan.vm.shouldMeasureGC
 920         doublePuts($stderr,file,"}")
 921       elsif benchParams.kind == :throughputBenchmark
 922         emitTimerFunctionCode(file)
 923         benchParams.dataPaths.each {
 924           | path |
 925           doublePuts($stderr,file,"load(#{path.inspect});")
 926         }
 927         doublePuts($stderr,file,"#{benchParams.setUpCommand.js}")
 928         if benchParams.doWarmup
 929           warmup = $warmup
 930         else
 931           warmup = 0
 932         end
 933         doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{warmup + $inner}; __bencher_index++) {")
 934         doublePuts($stderr,file,"    var __before = __bencher_curTimeMS();")
 935         doublePuts($stderr,file,"    var __after = __before;")
 936         doublePuts($stderr,file,"    var __runs = 0;")
 937         doublePuts($stderr,file,"    var __expected = #{$quantum};")
 938         doublePuts($stderr,file,"    while (true) {")
 939         $rerun.times {
 940           doublePuts($stderr,file,"       #{benchParams.command.js}")
 941         }
 942         doublePuts($stderr,file,"       __runs++;")
 943         doublePuts($stderr,file,"       __after = __bencher_curTimeMS();")
 944         if benchParams.deterministic
 945           doublePuts($stderr,file,"       if (true) {")
 946         else
 947           doublePuts($stderr,file,"       if (__after - __before >= __expected) {")
 948         end
 949         doublePuts($stderr,file,"           if (__runs >= #{benchParams.minimumIterations} || __bencher_index < #{warmup})")
 950         doublePuts($stderr,file,"               break;")
 951         doublePuts($stderr,file,"           __expected += #{$quantum}")
 952         doublePuts($stderr,file,"       }")
 953         doublePuts($stderr,file,"    }")
 954         doublePuts($stderr,file,"    if (__bencher_index >= #{warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{warmup}) + \": Time: \"+((__after-__before)/__runs));")
 955         doublePuts($stderr,file,"}")
 956         doublePuts($stderr,file,"#{benchParams.tearDownCommand.js}")
 957       else
 958         raise unless benchParams.kind == :singleFileTimedBenchmark
 959         doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
 960         doublePuts($stderr,file,"   var __bencher_before = __bencher_curTimeMS();")
 961         $rerun.times {
 962           doublePuts($stderr,file,"   run(__bencher_what);")
 963         }
 964         doublePuts($stderr,file,"   var __bencher_after = __bencher_curTimeMS();")
 965         doublePuts($stderr,file,"   return __bencher_after - __bencher_before;")
 966         doublePuts($stderr,file,"}")
 967         $warmup.times {
 968           doublePuts($stderr,file,"__bencher_run(#{benchParams.benchPath.inspect})")
 969           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
 970         }
 971         $inner.times {
 972           | innerIndex |
 973           doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchParams.benchPath.inspect}));")
 974           doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
 975         }
 976       end
 977     }
 978   when :dumpRenderTree, :webkitTestRunner
 979     case $timeMode
 980     when :preciseTime
 981       curTime = "(testRunner.preciseTime()*1000)"
 982     when :date
 983       curTime = "(Date.now())"
 984     else
 985       raise
 986     end
 987
 988     mainCode = Benchfile.create("bencher") {
 989       | file |
 990       doublePuts($stderr,file,"__bencher_count = 0;")
 991       doublePuts($stderr,file,"function __bencher_doNext(result) {")
 992       doublePuts($stderr,file,"    if (__bencher_count >= #{$warmup})")
 993       doublePuts($stderr,file,"        debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);")
 994       doublePuts($stderr,file,"    __bencher_count++;")
 995       doublePuts($stderr,file,"    if (__bencher_count < #{$inner+$warmup})")
 996       doublePuts($stderr,file,"        __bencher_runImpl(__bencher_doNext);")
 997       doublePuts($stderr,file,"    else")
 998       doublePuts($stderr,file,"        quit();")
 999       doublePuts($stderr,file,"}")
1000       doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);")
1001     }
1002
1003     cssCode = Benchfile.create("bencher-css") {
1004       | file |
1005       doublePuts($stderr,file,".pass {\n    font-weight: bold;\n    color: green;\n}\n.fail {\n    font-weight: bold;\n    color: red;\n}\n\#console {\n    white-space: pre-wrap;\n    font-family: monospace;\n}")
1006     }
1007
1008     preCode = Benchfile.create("bencher-pre") {
1009       | file |
1010       doublePuts($stderr,file,"if (window.testRunner) {")
1011       doublePuts($stderr,file,"    testRunner.dumpAsText(window.enablePixelTesting);")
1012       doublePuts($stderr,file,"    testRunner.waitUntilDone();")
1013       doublePuts($stderr,file,"}")
1014       doublePuts($stderr,file,"")
1015       doublePuts($stderr,file,"function debug(msg)")
1016       doublePuts($stderr,file,"{")
1017       doublePuts($stderr,file,"    var span = document.createElement(\"span\");")
1018       doublePuts($stderr,file,"    document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace")
1019       doublePuts($stderr,file,"    span.innerHTML = msg + '<br />';")
1020       doublePuts($stderr,file,"}")
1021       doublePuts($stderr,file,"")
1022       doublePuts($stderr,file,"function quit() {")
1023       doublePuts($stderr,file,"    testRunner.notifyDone();")
1024       doublePuts($stderr,file,"}")
1025       doublePuts($stderr,file,"")
1026       doublePuts($stderr,file,"__bencher_continuation=null;")
1027       doublePuts($stderr,file,"")
1028       doublePuts($stderr,file,"function reportResult(result) {")
1029       doublePuts($stderr,file,"    __bencher_continuation(result);")
1030       doublePuts($stderr,file,"}")
1031       if benchParams.kind == :singleFileTimedCallbackBenchmark
1032         doublePuts($stderr,file,"")
1033         doublePuts($stderr,file,benchParams.callbackDecl)
1034       end
1035       doublePuts($stderr,file,"")
1036       doublePuts($stderr,file,"function __bencher_runImpl(continuation) {")
1037       doublePuts($stderr,file,"    function doit() {")
1038       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"\";")
1039       doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";")
1040       doublePuts($stderr,file,"        var testFrame = document.getElementById(\"testframe\");")
1041       doublePuts($stderr,file,"        testFrame.contentDocument.open();")
1042       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");")
1043       if benchParams.kind == :throughputBenchmark or benchParams.kind == :multiFileTimedBenchmark
1044         benchParams.dataPaths.each {
1045           | path |
1046           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{path.inspect.inspect[1..-2]}></script>\");")
1047         }
1048       end
1049       if benchParams.kind == :throughputBenchmark
1050         if benchParams.doWarmup
1051           warmup = $warmup
1052         else
1053           warmup = 0
1054         end
1055         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">\");")
1056         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.setUpCommand.js}\");")
1057         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_before = #{curTime};\");")
1058         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_after = __bencher_before;\");")
1059         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_expected = #{$quantum};\");")
1060         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"var __bencher_runs = 0;\");")
1061         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"while (true) {\");")
1062         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    #{benchParams.command.js}\");")
1063         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_runs++;\");")
1064         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    __bencher_after = #{curTime};\");")
1065         if benchParams.deterministic
1066           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (true) {\");")
1067         else
1068           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    if (__bencher_after - __bencher_before >= __bencher_expected) {\");")
1069         end
1070         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        if (__bencher_runs >= #{benchParams.minimumIterations} || window.parent.__bencher_count < #{warmup})\");")
1071         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"            break;\");")
1072         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"        __bencher_expected += #{$quantum}\");")
1073         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"    }\");")
1074         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"}\");")
1075         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"#{benchParams.tearDownCommand.js}\");")
1076         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"window.parent.reportResult((__bencher_after - __bencher_before) / __bencher_runs);\");")
1077         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</script>\");")
1078       else
1079         doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">var __bencher_before = #{curTime};</script>\");")
1080         if benchParams.kind == :multiFileTimedBenchmark
1081           doublePuts($stderr,file,"        testFrame.contentDocument.write(#{benchParams.command.html.inspect});")
1082         else
1083           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=#{benchParams.benchPath.inspect.inspect[1..-2]}></script>\");")
1084         end
1085         unless benchParams.kind == :singleFileTimedCallbackBenchmark
1086           doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">window.parent.reportResult(#{curTime} - __bencher_before);</script>\");")
1087         end
1088       end
1089       doublePuts($stderr,file,"        testFrame.contentDocument.write(\"</body></html>\");")
1090       doublePuts($stderr,file,"        testFrame.contentDocument.close();")
1091       doublePuts($stderr,file,"    }")
1092       doublePuts($stderr,file,"    __bencher_continuation = continuation;")
1093       doublePuts($stderr,file,"    window.setTimeout(doit, 10);")
1094       doublePuts($stderr,file,"}")
1095     }
1096
1097     Benchfile.create(["bencher-htmldoc",".html"]) {
1098       | file |
1099       doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>")
1100     }
1101   else
1102     raise
1103   end
1104 end
1105
1106 def emitBenchRunCode(name, plan, benchParams)
1107   plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchParams), plan)
1108 end
1109
1110 class FileCreator
1111   def initialize(filename)
1112     @filename = filename
1113     @state = :empty
1114   end
1115
1116   def puts(text)
1117     $script.print "echo #{Shellwords.shellescape(text)}"
1118     if @state == :empty
1119       $script.print " > "
1120       @state = :nonEmpty
1121     else
1122       $script.print " >> "
1123     end
1124     $script.puts "#{Shellwords.shellescape(@filename)}"
1125   end
1126
1127   def close
1128     if @state == :empty
1129       $script.puts "rm -f #{Shellwords.shellescape(text)}"
1130       $script.puts "touch #{Shellwords.shellescape(text)}"
1131     end
1132   end
1133
1134   def self.open(filename)
1135     outp = FileCreator.new(filename)
1136     yield outp
1137     outp.close
1138   end
1139 end
1140
1141 def emitSelfContainedBenchRunCode(name, plan, targetFile, configFile, benchmark)
1142   FileCreator.open(configFile) {
1143     | outp |
1144     outp.puts "__bencher_message = \"#{name}: #{plan.vm}: #{plan.iteration}: \";"
1145     outp.puts "__bencher_warmup = #{$warmup};"
1146     outp.puts "__bencher_inner = #{$inner};"
1147     outp.puts "__bencher_benchmark = #{benchmark.to_json};"
1148     case $timeMode
1149     when :preciseTime
1150       outp.puts "__bencher_curTime = (function(){ return testRunner.preciseTime() * 1000; });"
1151     when :date
1152       outp.puts "__bencher_curTime = (function(){ return Date.now(); });"
1153     else
1154       raise
1155     end
1156   }
1157
1158   plan.vm.emitRunCode(targetFile, plan)
1159 end
1160
1161 def planForDescription(string, plans, benchFullname, vmName, iteration)
1162   raise "Unexpected benchmark full name: #{benchFullname.inspect}, string: #{string.inspect}" unless benchFullname =~ /\//
1163   suiteName = $~.pre_match
1164   return nil if suiteName == "WARMUP"
1165   benchName = $~.post_match
1166   result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration}
1167   raise "Unexpected result dimensions: #{result.inspect}, string: #{string.inspect}" unless result.size == 1
1168   result[0]
1169 end
1170
1171 class ParsedResult
1172   attr_reader :plan, :innerIndex, :time, :result
1173
1174   def initialize(plan, innerIndex, time)
1175     @plan = plan
1176     @innerIndex = innerIndex
1177     if time == :crashed
1178       @result = :error
1179     else
1180       @time = time
1181       @result = :success
1182     end
1183
1184     raise unless @plan.is_a? BenchPlan
1185     raise unless @innerIndex.is_a? Integer
1186     raise unless @time.is_a? Numeric or @result == :error
1187   end
1188
1189   def benchmark
1190     plan.benchmark
1191   end
1192
1193   def suite
1194     plan.suite
1195   end
1196
1197   def vm
1198     plan.vm
1199   end
1200
1201   def outerIndex
1202     plan.iteration
1203   end
1204
1205   def self.create(plan, innerIndex, time)
1206     if plan
1207       ParsedResult.new(plan, innerIndex, time)
1208     else
1209       nil
1210     end
1211   end
1212
1213   def self.parse(plans, string)
1214     if string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): Time: /
1215       benchFullname = $1
1216       vmName = $2
1217       outerIndex = $3.to_i
1218       innerIndex = $4.to_i
1219       time = $~.post_match.to_f
1220       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, time)
1221     elsif string =~ /([a-zA-Z0-9\/_.-]+): ([a-zA-Z0-9_#. ]+): ([0-9]+): ([0-9]+): CRASHED/
1222       benchFullname = $1
1223       vmName = $2
1224       outerIndex = $3.to_i
1225       innerIndex = $4.to_i
1226       time = $~.post_match.to_f
1227       ParsedResult.create(planForDescription(string, plans, benchFullname, vmName, outerIndex), innerIndex, :crashed)
1228     else
1229       nil
1230     end
1231   end
1232 end
1233
1234 class VM
1235   @@extraEnvSet = {}
1236
1237   def initialize(origPath, name, nameKind, svnRevision)
1238     @origPath = origPath.to_s
1239     @path = origPath.to_s
1240     @name = name
1241     @nameKind = nameKind
1242     @extraEnv = {}
1243
1244     if $forceVMKind
1245       @vmType = $forceVMKind
1246     else
1247       if @origPath =~ /DumpRenderTree$/
1248         @vmType = :dumpRenderTree
1249       elsif @origPath =~ /WebKitTestRunner$/
1250         @vmType = :webkitTestRunner
1251       else
1252         @vmType = :jsc
1253       end
1254     end
1255
1256     @svnRevision = svnRevision
1257
1258     # Try to detect information about the VM.
1259     if path =~ /\/WebKitBuild\/(Release|Debug)+\/([a-zA-Z]+)$/
1260       @checkoutPath = $~.pre_match
1261       # FIXME: Use some variant of this:
1262       # <bdash>   def retrieve_revision
1263       # <bdash>     `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i
1264       # <bdash>   end
1265       unless @svnRevision
1266         begin
1267           Dir.chdir(@checkoutPath) {
1268             $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
1269             IO.popen("svn info", "r") {
1270               | inp |
1271               inp.each_line {
1272                 | line |
1273                 if line =~ /Revision: ([0-9]+)/
1274                   @svnRevision = $1
1275                 end
1276               }
1277             }
1278           }
1279           unless @svnRevision
1280             $stderr.puts "Warning: running svn info for #{name} silently failed."
1281           end
1282         rescue => e
1283           # Failed to detect svn revision.
1284           $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
1285         end
1286       end
1287     else
1288       $stderr.puts "Warning: could not identify checkout location for #{name}"
1289     end
1290
1291     if @path =~ /\/Release\/([a-zA-Z]+)$/
1292       @libPath, @relativeBinPath = [$~.pre_match+"/Release"], "./#{$1}"
1293     elsif @path =~ /\/Debug\/([a-zA-Z]+)$/
1294       @libPath, @relativeBinPath = [$~.pre_match+"/Debug"], "./#{$1}"
1295     elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/
1296       @libPath = [$~.pre_match + "/Contents/Resources", $~.pre_match + "/Contents/Frameworks"]
1297     elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/
1298       @libPath, @relativeBinPath = [$~.pre_match], $&[1..-1]
1299     elsif @path =~ /(DumpRenderTree|webkitTestRunner|jsc)$/
1300       @libPath, @relativeBinPath = [$~.pre_match+"/"], "./#{$1}"
1301     end
1302   end
1303
1304   def canCopyIntoBenchPath
1305     if @libPath and @relativeBinPath
1306       true
1307     else
1308       false
1309     end
1310   end
1311
1312   def addExtraEnv(key, val)
1313     @extraEnv[key] = val
1314     @@extraEnvSet[key] = true
1315   end
1316
1317   def copyIntoBenchPath
1318     raise unless canCopyIntoBenchPath
1319     basename, filename = Benchfile.uniqueFilename("vm")
1320     raise unless Dir.mkdir(filename)
1321     @libPath.each {
1322       | libPathPart |
1323       cmd = "cp -a #{Shellwords.shellescape(libPathPart)}/* #{Shellwords.shellescape(filename.to_s)}"
1324       $stderr.puts ">> #{cmd}" if $verbosity>=2
1325       raise unless system(cmd)
1326     }
1327     @path = "#{basename}/#{@relativeBinPath}"
1328     @libPath = [basename]
1329   end
1330
1331   def to_s
1332     @name
1333   end
1334
1335   def name
1336     @name
1337   end
1338
1339   def shouldMeasureGC
1340     $measureGC == true or ($measureGC == name)
1341   end
1342
1343   def origPath
1344     @origPath
1345   end
1346
1347   def path
1348     @path
1349   end
1350
1351   def nameKind
1352     @nameKind
1353   end
1354
1355   def vmType
1356     @vmType
1357   end
1358
1359   def checkoutPath
1360     @checkoutPath
1361   end
1362
1363   def svnRevision
1364     @svnRevision
1365   end
1366
1367   def extraEnv
1368     @extraEnv
1369   end
1370
1371   def printFunction
1372     case @vmType
1373     when :jsc
1374       "print"
1375     when :dumpRenderTree, :webkitTestRunner
1376       "debug"
1377     else
1378       raise @vmType
1379     end
1380   end
1381
1382   def emitRunCode(fileToRun, plan)
1383     myLibPath = @libPath
1384     myLibPath = [] unless myLibPath
1385     @@extraEnvSet.keys.each {
1386       | key |
1387       $script.puts "unset #{Shellwords.shellescape(key)}"
1388     }
1389     $script.puts "export DYLD_LIBRARY_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1390     $script.puts "export DYLD_FRAMEWORK_PATH=#{Shellwords.shellescape(myLibPath.join(':').to_s)}"
1391     @extraEnv.each_pair {
1392       | key, val |
1393       $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1394     }
1395     plan.environment.each_pair {
1396         | key, val |
1397         $script.puts "export #{Shellwords.shellescape(key)}=#{Shellwords.shellescape(val)}"
1398     }
1399     $script.puts "#{path} #{fileToRun} 2>&1 || {"
1400     $script.puts "    echo " + Shellwords.shellescape("#{name} failed to run!") + " 1>&2"
1401     $inner.times {
1402       | iteration |
1403       $script.puts "    echo " + Shellwords.shellescape("#{plan.prefix}: #{iteration}: CRASHED")
1404     }
1405     $script.puts "}"
1406     plan.environment.keys.each {
1407         | key |
1408         $script.puts "export #{Shellwords.shellescape(key)}="
1409     }
1410   end
1411 end
1412
1413 class StatsAccumulator
1414   def initialize
1415     @stats = []
1416     ($outer*$inner).times {
1417       @stats << Stats.new
1418     }
1419   end
1420
1421   def statsForIteration(outerIteration, innerIteration)
1422     @stats[outerIteration*$inner + innerIteration]
1423   end
1424
1425   def stats
1426     result = Stats.new
1427     @stats.each {
1428       | stat |
1429       if stat.ok?
1430         result.add(yield stat)
1431       else
1432         result.add(nil)
1433       end
1434     }
1435     result
1436   end
1437
1438   def geometricMeanStats
1439     stats {
1440       | stat |
1441       stat.geometricMean
1442     }
1443   end
1444
1445   def arithmeticMeanStats
1446     stats {
1447       | stat |
1448       stat.arithmeticMean
1449     }
1450   end
1451 end
1452
1453 module Benchmark
1454   attr_accessor :benchmarkSuite
1455   attr_reader :name
1456
1457   def fullname
1458     benchmarkSuite.name + "/" + name
1459   end
1460
1461   def to_s
1462     fullname
1463   end
1464
1465   def weight
1466     1
1467   end
1468
1469   def weightString
1470     raise unless weight.is_a? Fixnum
1471     raise unless weight >= 1
1472     if weight == 1
1473       ""
1474     else
1475       "x#{weight} "
1476     end
1477   end
1478 end
1479
1480 class SunSpiderBenchmark
1481   include Benchmark
1482
1483   def initialize(name)
1484     @name = name
1485   end
1486
1487   def emitRunCode(plan)
1488     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js")))
1489   end
1490 end
1491
1492 class LongSpiderBenchmark
1493   include Benchmark
1494
1495   def initialize(name)
1496     @name = name
1497   end
1498
1499   def emitRunCode(plan)
1500     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("LongSpider-#{@name}", "#{LONGSPIDER_PATH}/#{@name}.js")))
1501   end
1502 end
1503
1504 class V8Benchmark
1505   include Benchmark
1506
1507   def initialize(name)
1508     @name = name
1509   end
1510
1511   def emitRunCode(plan)
1512     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js")))
1513   end
1514 end
1515
1516 class V8RealBenchmark
1517   include Benchmark
1518
1519   attr_reader :v8SuiteName
1520
1521   def initialize(v8SuiteName, name, weight, minimumIterations)
1522     @v8SuiteName = v8SuiteName
1523     @name = name
1524     @weight = weight
1525     @minimumIterations = minimumIterations
1526   end
1527
1528   def weight
1529     @weight
1530   end
1531
1532   def emitRunCode(plan)
1533     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(["base", @v8SuiteName, "jsc-#{@name}"].collect{|v| ensureFile("V8Real-#{v}", "#{V8_REAL_PATH}/#{v}.js")}, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, false, @minimumIterations))
1534   end
1535 end
1536
1537 class OctaneBenchmark
1538   include Benchmark
1539
1540   def initialize(files, name, weight, doWarmup, deterministic, minimumIterations)
1541     @files = files
1542     @name = name
1543     @weight = weight
1544     @doWarmup = doWarmup
1545     @deterministic = deterministic
1546     @minimumIterations = minimumIterations
1547   end
1548
1549   def weight
1550     @weight
1551   end
1552
1553   def emitRunCode(plan)
1554     files = []
1555     files += (["base"] + @files).collect {
1556       | v |
1557       ensureFile("Octane-#{v}", "#{OCTANE_PATH}/#{v}.js")
1558     }
1559     files += ["jsc-#{@name}"].collect {
1560       | v |
1561       ensureFile("Octane-#{v}", "#{OCTANE_WRAPPER_PATH}/#{v}.js")
1562     }
1563     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new(files, simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1564   end
1565 end
1566
1567 class KrakenBenchmark
1568   include Benchmark
1569
1570   def initialize(name)
1571     @name = name
1572   end
1573
1574   def emitRunCode(plan)
1575     emitBenchRunCode(fullname, plan, MultiFileTimedBenchmarkParameters.new([ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js")], loadCommandForFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js")))
1576   end
1577 end
1578
1579 class JSBenchBenchmark
1580   include Benchmark
1581
1582   attr_reader :jsBenchMode
1583
1584   def initialize(name, jsBenchMode)
1585     @name = name
1586     @jsBenchMode = jsBenchMode
1587   end
1588
1589   def emitRunCode(plan)
1590     callbackDecl  = "function JSBNG_handleResult(result) {\n"
1591     callbackDecl += "    if (result.error) {\n"
1592     callbackDecl += "        console.log(\"Did not run benchmark correctly!\");\n"
1593     callbackDecl += "        quit();\n"
1594     callbackDecl += "    }\n"
1595     callbackDecl += "    reportResult(result.time);\n"
1596     callbackDecl += "}\n"
1597     emitBenchRunCode(fullname, plan, SingleFileTimedCallbackBenchmarkParameters.new(callbackDecl, ensureFile("JSBench-#{@name}", "#{JSBENCH_PATH}/#{@name}/#{@jsBenchMode}.js")))
1598   end
1599 end
1600
1601 class JSRegressBenchmark
1602   include Benchmark
1603
1604   def initialize(name)
1605     @name = name
1606   end
1607
1608   def emitRunCode(plan)
1609     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("JSRegress-#{@name}", "#{JSREGRESS_PATH}/#{@name}.js")))
1610   end
1611 end
1612
1613 class AsmBenchBenchmark
1614   include Benchmark
1615
1616   def initialize(name)
1617     @name = name
1618   end
1619
1620   def emitRunCode(plan)
1621     emitBenchRunCode(fullname, plan, SingleFileTimedBenchmarkParameters.new(ensureFile("AsmBench-#{@name}", "#{ASMBENCH_PATH}/#{@name}.js")))
1622   end
1623 end
1624
1625
1626 class CompressionBenchBenchmark
1627   include Benchmark
1628
1629   def initialize(files, name, model)
1630     @files = files
1631     @name = name;
1632     @name = name + "-" + model if !model.empty?
1633     @name = @name.gsub(" ", "-").downcase
1634     @scriptName = name
1635     @weight = 1
1636     @doWarmup = true
1637     @deterministic = true
1638     @minimumIterations = 1
1639     @model = model
1640   end
1641
1642   def weight
1643     @weight
1644   end
1645
1646   def emitRunCode(plan)
1647     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new((["base"] + @files + ["jsc-#{@scriptName}"]).collect{|v| ensureFile("Compression-#{v}", "#{COMPRESSIONBENCH_PATH}/#{v}.js")}, simpleCommand("jscSetUp('#{@model}');"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), @doWarmup, @deterministic, @minimumIterations))
1648   end
1649 end
1650
1651 class DSPJSFiltrrBenchmark
1652   include Benchmark
1653
1654   def initialize(name, filterKey)
1655     @name = name
1656     @filterKey = filterKey
1657   end
1658
1659   def emitRunCode(plan)
1660     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr.js")
1661     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr_back.jpg")
1662     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-jquery.min.js")
1663     ensureAbsoluteFile(DSPJS_FILTRR_PATH + "filtrr-bencher.html")
1664     emitSelfContainedBenchRunCode(fullname, plan, "filtrr-bencher.html", "bencher-config.js", @filterKey)
1665   end
1666 end
1667
1668 class DSPJSVP8Benchmark
1669   include Benchmark
1670
1671   def initialize
1672     @name = "route9-vp8"
1673   end
1674
1675   def weight
1676     5
1677   end
1678
1679   def emitRunCode(plan)
1680     ensureBenchmarkFiles(DSPJS_ROUTE9_PATH)
1681     emitSelfContainedBenchRunCode(fullname, plan, "route9-bencher.html", "bencher-config.js", "")
1682   end
1683 end
1684
1685 class DSPStarfieldBenchmark
1686   include Benchmark
1687
1688   def initialize
1689     @name = "starfield"
1690   end
1691
1692   def weight
1693     5
1694   end
1695
1696   def emitRunCode(plan)
1697     ensureBenchmarkFiles(DSPJS_STARFIELD_PATH)
1698     emitSelfContainedBenchRunCode(fullname, plan, "starfield-bencher.html", "bencher-config.js", "")
1699   end
1700 end
1701
1702 class DSPJSJSLinuxBenchmark
1703   include Benchmark
1704   def initialize
1705     @name = "bellard-jslinux"
1706   end
1707
1708   def weight
1709     5
1710   end
1711
1712   def emitRunCode(plan)
1713     ensureBenchmarkFiles(DSPJS_JSLINUX_PATH)
1714     emitSelfContainedBenchRunCode(fullname, plan, "jslinux-bencher.html", "bencher-config.js", "")
1715   end
1716 end
1717
1718 class DSPJSQuake3Benchmark
1719   include Benchmark
1720
1721   def initialize
1722     @name = "zynaps-quake3"
1723   end
1724
1725   def weight
1726     5
1727   end
1728
1729   def emitRunCode(plan)
1730     ensureBenchmarkFiles(DSPJS_QUAKE3_PATH)
1731     emitSelfContainedBenchRunCode(fullname, plan, "quake-bencher.html", "bencher-config.js", "")
1732   end
1733 end
1734
1735 class DSPJSMandelbrotBenchmark
1736   include Benchmark
1737
1738   def initialize
1739     @name = "zynaps-mandelbrot"
1740   end
1741
1742   def weight
1743     5
1744   end
1745
1746   def emitRunCode(plan)
1747     ensureBenchmarkFiles(DSPJS_MANDELBROT_PATH)
1748     emitSelfContainedBenchRunCode(fullname, plan, "mandelbrot-bencher.html", "bencher-config.js", "")
1749   end
1750 end
1751
1752 class DSPJSAmmoJSASMBenchmark
1753   include Benchmark
1754
1755   def initialize
1756     @name = "ammojs-asm-js"
1757   end
1758
1759   def weight
1760     5
1761   end
1762
1763   def emitRunCode(plan)
1764     ensureBenchmarkFiles(DSPJS_AMMOJS_ASMJS_PATH)
1765     emitSelfContainedBenchRunCode(fullname, plan, "ammo-asmjs-bencher.html", "bencher-config.js", "")
1766   end
1767 end
1768
1769 class DSPJSAmmoJSRegularBenchmark
1770   include Benchmark
1771
1772   def initialize
1773     @name = "ammojs-regular-js"
1774   end
1775
1776   def weight
1777     5
1778   end
1779
1780   def emitRunCode(plan)
1781     ensureBenchmarkFiles(DSPJS_AMMOJS_REGULAR_PATH)
1782     emitSelfContainedBenchRunCode(fullname, plan, "ammo-regular-bencher.html", "bencher-config.js", "")
1783   end
1784 end
1785
1786 class BrowsermarkJSBenchmark
1787   include Benchmark
1788
1789   def initialize(name)
1790     @name = name
1791   end
1792
1793   def emitRunCode(plan)
1794     emitBenchRunCode(fullname, plan, ThroughputBenchmarkParameters.new([ensureFile(name, "#{BROWSERMARK_JS_PATH}/#{name}/test.js"), ensureFile("browsermark-bencher", "#{BROWSERMARK_JS_PATH}/browsermark-bencher.js")], simpleCommand("jscSetUp();"), simpleCommand("jscRun();"), simpleCommand("jscTearDown();"), true, 32))
1795   end
1796 end
1797
1798 class BrowsermarkDOMBenchmark
1799   include Benchmark
1800
1801   def initialize(name)
1802     @name = name
1803   end
1804
1805   def emitRunCode(plan)
1806     ensureBenchmarkFiles(BROWSERMARK_PATH)
1807     emitSelfContainedBenchRunCode(fullname, plan, "tests/benchmarks/dom/#{name}/index.html", "bencher-config.js", name)
1808   end
1809 end
1810
1811 class BenchmarkSuite
1812   def initialize(name, preferredMean, decimalShift)
1813     @name = name
1814     @preferredMean = preferredMean
1815     @benchmarks = []
1816     @subSuites = []
1817     @decimalShift = decimalShift
1818   end
1819
1820   def name
1821     @name
1822   end
1823
1824   def to_s
1825     @name
1826   end
1827
1828   def decimalShift
1829     @decimalShift
1830   end
1831
1832   def addIgnoringPattern(benchmark)
1833     benchmark.benchmarkSuite = self
1834     @benchmarks << benchmark
1835   end
1836
1837   def add(benchmark)
1838     if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern
1839         addIgnoringPattern(benchmark)
1840     end
1841   end
1842
1843   def addSubSuite(subSuite)
1844     @subSuites << subSuite
1845   end
1846
1847   def benchmarks
1848     @benchmarks
1849   end
1850
1851   def benchmarkForName(name)
1852     result = @benchmarks.select{|v| v.name == name}
1853     raise unless result.length == 1
1854     result[0]
1855   end
1856
1857   def hasBenchmark(benchmark)
1858     array = @benchmarks.select{|v| v == benchmark}
1859     raise unless array.length == 1 or array.length == 0
1860     array.length == 1
1861   end
1862
1863   def subSuites
1864     @subSuites
1865   end
1866
1867   def suites
1868     [self] + @subSuites
1869   end
1870
1871   def suitesWithBenchmark(benchmark)
1872     result = [self]
1873     @subSuites.each {
1874       | subSuite |
1875       if subSuite.hasBenchmark(benchmark)
1876         result << subSuite
1877       end
1878     }
1879     result
1880   end
1881
1882   def empty?
1883     @benchmarks.empty?
1884   end
1885
1886   def retain_if
1887     @benchmarks.delete_if {
1888       | benchmark |
1889       not yield benchmark
1890     }
1891   end
1892
1893   def preferredMean
1894     @preferredMean
1895   end
1896
1897   def computeMean(stat)
1898     if stat.ok?
1899       (stat.send @preferredMean) * (10 ** decimalShift)
1900     else
1901       nil
1902     end
1903   end
1904 end
1905
1906 class BenchRunPlan
1907   def initialize(benchmark, vm, iteration)
1908     @benchmark = benchmark
1909     @vm = vm
1910     @iteration = iteration
1911     @environment = {}
1912     if $environment.has_key?(vm.name)
1913       if $environment[vm.name].has_key?(benchmark.benchmarkSuite.name)
1914         if $environment[vm.name][benchmark.benchmarkSuite.name].has_key?(benchmark.name)
1915           @environment = $environment[vm.name][benchmark.benchmarkSuite.name][benchmark.name]
1916         end
1917       end
1918     end
1919   end
1920
1921   def benchmark
1922     @benchmark
1923   end
1924
1925   def suite
1926     @benchmark.benchmarkSuite
1927   end
1928
1929   def vm
1930     @vm
1931   end
1932
1933   def iteration
1934     @iteration
1935   end
1936
1937   def environment
1938     @environment
1939   end
1940
1941   def prefix
1942     "#{@benchmark.fullname}: #{vm.name}: #{iteration}"
1943   end
1944
1945   def emitRunCode
1946     @benchmark.emitRunCode(self)
1947   end
1948
1949   def to_s
1950     benchmark.to_s + "/" + vm.to_s
1951   end
1952 end
1953
1954 class BenchmarkOnVM
1955   def initialize(benchmark, suiteOnVM, subSuitesOnVM)
1956     @benchmark = benchmark
1957     @suiteOnVM = suiteOnVM
1958     @subSuitesOnVM = subSuitesOnVM
1959     @stats = Stats.new
1960   end
1961
1962   def to_s
1963     "#{@benchmark} on #{@suiteOnVM.vm}"
1964   end
1965
1966   def benchmark
1967     @benchmark
1968   end
1969
1970   def vm
1971     @suiteOnVM.vm
1972   end
1973
1974   def vmStats
1975     @suiteOnVM.vmStats
1976   end
1977
1978   def suite
1979     @benchmark.benchmarkSuite
1980   end
1981
1982   def suiteOnVM
1983     @suiteOnVM
1984   end
1985
1986   def subSuitesOnVM
1987     @subSuitesOnVM
1988   end
1989
1990   def stats
1991     @stats
1992   end
1993
1994   def parseResult(result)
1995     raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm
1996     raise unless result.benchmark == @benchmark
1997     @stats.add(result.time)
1998   end
1999 end
2000
2001 class NamedStatsAccumulator < StatsAccumulator
2002   def initialize(name)
2003     super()
2004     @name = name
2005   end
2006
2007   def reportingName
2008     @name
2009   end
2010 end
2011
2012 class SuiteOnVM < StatsAccumulator
2013   def initialize(vm, vmStats, suite)
2014     super()
2015     @vm = vm
2016     @vmStats = vmStats
2017     @suite = suite
2018
2019     raise unless @vm.is_a? VM
2020     raise unless @vmStats.is_a? StatsAccumulator
2021     raise unless @suite.is_a? BenchmarkSuite
2022   end
2023
2024   def to_s
2025     "#{@suite} on #{@vm}"
2026   end
2027
2028   def suite
2029     @suite
2030   end
2031
2032   def vm
2033     @vm
2034   end
2035
2036   def reportingName
2037     @vm.name
2038   end
2039
2040   def vmStats
2041     raise unless @vmStats
2042     @vmStats
2043   end
2044 end
2045
2046 class SubSuiteOnVM < StatsAccumulator
2047   def initialize(vm, suite)
2048     super()
2049     @vm = vm
2050     @suite = suite
2051     raise unless @vm.is_a? VM
2052     raise unless @suite.is_a? BenchmarkSuite
2053   end
2054
2055   def to_s
2056     "#{@suite} on #{@vm}"
2057   end
2058
2059   def suite
2060     @suite
2061   end
2062
2063   def vm
2064     @vm
2065   end
2066
2067   def reportingName
2068     @vm.name
2069   end
2070 end
2071
2072 class BenchPlan
2073   def initialize(benchmarkOnVM, iteration)
2074     @benchmarkOnVM = benchmarkOnVM
2075     @iteration = iteration
2076   end
2077
2078   def to_s
2079     "#{@benchmarkOnVM} \##{@iteration+1}"
2080   end
2081
2082   def benchmarkOnVM
2083     @benchmarkOnVM
2084   end
2085
2086   def benchmark
2087     @benchmarkOnVM.benchmark
2088   end
2089
2090   def suite
2091     @benchmarkOnVM.suite
2092   end
2093
2094   def vm
2095     @benchmarkOnVM.vm
2096   end
2097
2098   def iteration
2099     @iteration
2100   end
2101
2102   def parseResult(result)
2103     raise unless result.plan == self
2104     @benchmarkOnVM.parseResult(result)
2105     benchmark.weight.times {
2106       @benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time)
2107       @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time)
2108       @benchmarkOnVM.subSuitesOnVM.each {
2109         | subSuite |
2110         subSuite.statsForIteration(@iteration, result.innerIndex).add(result.time)
2111       }
2112     }
2113   end
2114 end
2115
2116 def lpad(str,chars)
2117   if str.length>chars
2118     str
2119   else
2120     "%#{chars}s"%(str)
2121   end
2122 end
2123
2124 def rpad(str,chars)
2125   while str.length < chars
2126     str+=" "
2127   end
2128   str
2129 end
2130
2131 def center(str,chars)
2132   while str.length<chars
2133     str+=" "
2134     if str.length<chars
2135       str=" "+str
2136     end
2137   end
2138   str
2139 end
2140
2141 def statsToStr(stats, decimalShift)
2142   if stats.error?
2143     lpad(center("ERROR", 10+10+2), 12+10+2)
2144   elsif $inner*$outer == 1
2145     string = numToStr(stats.mean, decimalShift)
2146     raise unless string =~ /\./
2147     left = $~.pre_match
2148     right = $~.post_match
2149     lpad(left, 13 - decimalShift) + "." + rpad(right, 10 + decimalShift)
2150   else
2151     lpad(numToStr(stats.mean, decimalShift), 12) + "+-" + rpad(numToStr(stats.confInt, decimalShift), 10)
2152   end
2153 end
2154
2155 def plural(num)
2156   if num == 1
2157     ""
2158   else
2159     "s"
2160   end
2161 end
2162
2163 def wrap(str, columns)
2164   array = str.split
2165   result = ""
2166   curLine = array.shift
2167   array.each {
2168     | curStr |
2169     if (curLine + " " + curStr).size > columns
2170       result += curLine + "\n"
2171       curLine = curStr
2172     else
2173       curLine += " " + curStr
2174     end
2175   }
2176   result + curLine + "\n"
2177 end
2178
2179 def runAndGetResults
2180   results = nil
2181   Dir.chdir(BENCH_DATA_PATH) {
2182     $stderr.puts ">> sh ./runscript" if $verbosity >= 2
2183     raise "Script did not complete correctly: #{$?}" unless system("sh ./runscript > runlog")
2184     results = IO::read("runlog")
2185   }
2186   raise unless results
2187   results
2188 end
2189
2190 def parseAndDisplayResults(results)
2191   vmStatses = []
2192   $vms.each {
2193     | vm |
2194     vmStatses << NamedStatsAccumulator.new(vm.name)
2195   }
2196
2197   suitesOnVMs = []
2198   suitesOnVMsForSuite = {}
2199   subSuitesOnVMsForSubSuite = {}
2200   $suites.each {
2201     | suite |
2202     suitesOnVMsForSuite[suite] = []
2203     suite.subSuites.each {
2204       | subSuite |
2205       subSuitesOnVMsForSubSuite[subSuite] = []
2206     }
2207   }
2208   suitesOnVMsForVM = {}
2209   $vms.each {
2210     | vm |
2211     suitesOnVMsForVM[vm] = []
2212   }
2213
2214   benchmarksOnVMs = []
2215   benchmarksOnVMsForBenchmark = {}
2216   $benchmarks.each {
2217     | benchmark |
2218     benchmarksOnVMsForBenchmark[benchmark] = []
2219   }
2220
2221   $vms.each_with_index {
2222     | vm, vmIndex |
2223     vmStats = vmStatses[vmIndex]
2224     $suites.each {
2225       | suite |
2226       suiteOnVM = SuiteOnVM.new(vm, vmStats, suite)
2227       subSuitesOnVM = suite.subSuites.map {
2228         | subSuite |
2229         result = SubSuiteOnVM.new(vm, subSuite)
2230         subSuitesOnVMsForSubSuite[subSuite] << result
2231         result
2232       }
2233       suitesOnVMs << suiteOnVM
2234       suitesOnVMsForSuite[suite] << suiteOnVM
2235       suitesOnVMsForVM[vm] << suiteOnVM
2236       suite.benchmarks.each {
2237         | benchmark |
2238         subSuitesOnVMForThisBenchmark = []
2239         subSuitesOnVM.each {
2240           | subSuiteOnVM |
2241           if subSuiteOnVM.suite.hasBenchmark(benchmark)
2242             subSuitesOnVMForThisBenchmark << subSuiteOnVM
2243           end
2244         }
2245         benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM, subSuitesOnVMForThisBenchmark)
2246         benchmarksOnVMs << benchmarkOnVM
2247         benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
2248       }
2249     }
2250   }
2251
2252   plans = []
2253   benchmarksOnVMs.each {
2254     | benchmarkOnVM |
2255     $outer.times {
2256       | iteration |
2257       plans << BenchPlan.new(benchmarkOnVM, iteration)
2258     }
2259   }
2260
2261   hostname = nil
2262   hwmodel = nil
2263   results.each_line {
2264     | line |
2265     line.chomp!
2266     if line =~ /HOSTNAME:([^.]+)/
2267       hostname = $1
2268     elsif line =~ /HARDWARE:hw\.model: /
2269       hwmodel = $~.post_match.chomp
2270     else
2271       result = ParsedResult.parse(plans, line)
2272       if result
2273         result.plan.parseResult(result)
2274       end
2275     end
2276   }
2277
2278   # Compute the geomean of the preferred means of results on a SuiteOnVM
2279   overallResults = []
2280   $vms.each {
2281     | vm |
2282     result = Stats.new
2283     $outer.times {
2284       | outerIndex |
2285       $inner.times {
2286         | innerIndex |
2287         curResult = Stats.new
2288         suitesOnVMsForVM[vm].each {
2289           | suiteOnVM |
2290           # For a given iteration, suite, and VM, compute the suite's preferred mean
2291           # over the data collected for all benchmarks in that suite. We'll have one
2292           # sample per benchmark. For example on V8 this will be the geomean of 1
2293           # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
2294           # splay.
2295           curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
2296         }
2297
2298         # curResult now holds 1 sample for each of the means computed in the above
2299         # loop. Compute the geomean over this, and store it.
2300         if curResult.ok?
2301           result.add(curResult.geometricMean)
2302         else
2303           result.add(nil)
2304         end
2305       }
2306     }
2307
2308     # $overallResults will have a Stats for each VM. That Stats object will hold
2309     # $inner*$outer geomeans, allowing us to compute the arithmetic mean and
2310     # confidence interval of the geomeans of preferred means. Convoluted, but
2311     # useful and probably sound.
2312     overallResults << result
2313   }
2314
2315   if $verbosity >= 2
2316     benchmarksOnVMs.each {
2317       | benchmarkOnVM |
2318       $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
2319     }
2320
2321     $vms.each_with_index {
2322       | vm, vmIndex |
2323       vmStats = vmStatses[vmIndex]
2324       $stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}"
2325       $stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}"
2326     }
2327   end
2328
2329   if $outputName
2330     reportName = $outputName
2331   else
2332     reportName =
2333       (if ($vms.collect {
2334              | vm |
2335              vm.nameKind
2336            }.index :auto)
2337          ""
2338        else
2339          text = $vms.collect {
2340            | vm |
2341            vm.to_s
2342          }.join("_") + "_"
2343          if text.size >= 40
2344            ""
2345          else
2346            text
2347          end
2348        end) +
2349       ($suites.collect {
2350          | suite |
2351          suite.to_s
2352        }.join("")) + "_" +
2353       (if hostname
2354          hostname + "_"
2355        else
2356          ""
2357        end)+
2358       (begin
2359          time = Time.now
2360          "%04d%02d%02d_%02d%02d" %
2361            [ time.year, time.month, time.day,
2362              time.hour, time.min ]
2363        end)
2364   end
2365
2366   unless $brief
2367     puts "Generating benchmark report at #{Dir.pwd}/#{reportName}_report.txt"
2368     puts "And raw data at #{Dir.pwd}/#{reportName}.json"
2369   end
2370
2371   outp = $stdout
2372   json = {}
2373   begin
2374     outp = File.open(reportName + "_report.txt","w")
2375   rescue => e
2376     $stderr.puts "Error: could not save report to #{reportName}_report.txt: #{e}"
2377     $stderr.puts
2378   end
2379
2380   def createVMsString
2381     result = ""
2382     result += "   " if $allSuites.size > 1
2383     result += rpad("", $benchpad + $weightpad)
2384     result += " "
2385     $vms.size.times {
2386       | index |
2387       if index != 0
2388         result += " "+NoChange.new(0).shortForm
2389       end
2390       result += lpad(center($vms[index].name, 10+10+2), 12+10+2)
2391     }
2392     result += "    "
2393     if $vms.size >= 3
2394       result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
2395     elsif $vms.size >= 2
2396       result += " "*26
2397     end
2398     result
2399   end
2400
2401   def andJoin(list)
2402     if list.size == 1
2403       list[0].to_s
2404     elsif list.size == 2
2405       "#{list[0]} and #{list[1]}"
2406     else
2407       "#{list[0..-2].join(', ')}, and #{list[-1]}"
2408     end
2409   end
2410
2411   json["vms"] = $vms.collect{|v| v.name}
2412   json["suites"] = {}
2413   json["runlog"] = results
2414
2415   columns = [createVMsString.size, 78].max
2416
2417   outp.print "Benchmark report for "
2418   outp.print andJoin($suites)
2419   if hostname
2420     outp.print " on #{hostname}"
2421   end
2422   if hwmodel
2423     outp.print " (#{hwmodel})"
2424   end
2425   outp.puts "."
2426   outp.puts
2427
2428   outp.puts "VMs tested:"
2429   $vms.each {
2430     | vm |
2431     outp.print "\"#{vm.name}\" at #{vm.origPath}"
2432     if vm.svnRevision
2433       outp.print " (r#{vm.svnRevision})"
2434     end
2435     outp.puts
2436     vm.extraEnv.each_pair {
2437       | key, val |
2438       outp.puts "    export #{key}=#{val}"
2439     }
2440   }
2441
2442   outp.puts
2443
2444   outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
2445                  "with #{$outer} VM invocation#{plural($outer)} per benchmark."+
2446                  (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
2447                                       "total time of those iterations, for each sample.")
2448                   else "" end)+
2449                  (if $measureGC == true then (" No manual garbage collection invocations were "+
2450                                               "emitted.")
2451                   elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
2452                                          "all VMs except #{$measureGC}.")
2453                   else (" Emitted a call to gc() between sample measurements.") end)+
2454                  (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
2455                                         "began with the very first iteration.")
2456                   else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
2457                         "invocation for warm-up.") end)+
2458                  (case $timeMode
2459                   when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
2460                                           "microsecond-level timing.")
2461                   when :date then (" Used the portable Date.now() method to get millisecond-"+
2462                                    "level timing.")
2463                   else raise end)+
2464                  " Reporting benchmark execution times with 95% confidence "+
2465                  "intervals in milliseconds.",
2466                  columns)
2467
2468   outp.puts
2469
2470   def printVMs(outp)
2471     outp.puts createVMsString
2472   end
2473
2474   def summaryStats(outp, json, accumulators, name, decimalShift, &proc)
2475     resultingJson = {}
2476     outp.print "   " if $allSuites.size > 1
2477     outp.print rpad(name, $benchpad + $weightpad)
2478     outp.print " "
2479     accumulators.size.times {
2480       | index |
2481       if index != 0
2482         outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
2483       end
2484       outp.print statsToStr(accumulators[index].stats(&proc), decimalShift)
2485       resultingJson[accumulators[index].reportingName] = accumulators[index].stats(&proc).jsonMap
2486     }
2487     if accumulators.size>=2
2488       outp.print("    "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm)
2489     end
2490     outp.puts
2491     json[name] = resultingJson
2492   end
2493
2494   def allSummaryStats(outp, json, accumulators, preferredMean, decimalShift)
2495     meanLabel = '<' + preferredMean.to_s.sub(/Mean$/, '') + '>'
2496     summaryStats(outp, json, accumulators, meanLabel, decimalShift) {
2497       | stat |
2498       stat.send(preferredMean)
2499     }
2500   end
2501
2502   $suites.each {
2503     | suite |
2504     suiteJson = {}
2505     subSuiteJsons = {}
2506     suite.subSuites.each {
2507       | subSuite |
2508       subSuiteJsons[subSuite] = {}
2509     }
2510
2511     printVMs(outp)
2512     if $allSuites.size > 1
2513       outp.puts(andJoin(suite.suites.map{|v| v.name}) + ":")
2514     else
2515       outp.puts
2516     end
2517     suite.benchmarks.each {
2518       | benchmark |
2519       benchmarkJson = {}
2520       outp.print "   " if $allSuites.size > 1
2521       outp.print rpad(benchmark.name, $benchpad) + rpad(benchmark.weightString, $weightpad)
2522       if benchmark.name.size > $benchNameClip
2523         outp.puts
2524         outp.print "   " if $allSuites.size > 1
2525         outp.print((" " * $benchpad) + (" " * $weightpad))
2526       end
2527       outp.print " "
2528       myConfigs = benchmarksOnVMsForBenchmark[benchmark]
2529       myConfigs.size.times {
2530         | index |
2531         if index != 0
2532           outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
2533         end
2534         outp.print statsToStr(myConfigs[index].stats, suite.decimalShift)
2535         benchmarkJson[myConfigs[index].vm.name] = myConfigs[index].stats.jsonMap
2536       }
2537       if $vms.size>=2
2538         outp.print("    "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
2539       end
2540       outp.puts
2541       suiteJson[benchmark.name] = benchmarkJson
2542       suite.subSuites.each {
2543         | subSuite |
2544         if subSuite.hasBenchmark(benchmark)
2545           subSuiteJsons[subSuite][benchmark.name] = benchmarkJson
2546         end
2547       }
2548     }
2549     outp.puts
2550     unless suite.subSuites.empty?
2551       suite.subSuites.each {
2552         | subSuite |
2553         outp.puts "#{subSuite.name}:"
2554         allSummaryStats(outp, subSuiteJsons[subSuite], subSuitesOnVMsForSubSuite[subSuite], subSuite.preferredMean, subSuite.decimalShift)
2555         outp.puts
2556       }
2557       outp.puts "#{suite.name} including #{andJoin(suite.subSuites.map{|v| v.name})}:"
2558     end
2559     allSummaryStats(outp, suiteJson, suitesOnVMsForSuite[suite], suite.preferredMean, suite.decimalShift)
2560     outp.puts if $allSuites.size > 1
2561
2562     json["suites"][suite.name] = suiteJson
2563     suite.subSuites.each {
2564       | subSuite |
2565       json["suites"][subSuite.name] = subSuiteJsons[subSuite]
2566     }
2567   }
2568
2569   if $suites.size > 1
2570     printVMs(outp)
2571     outp.puts "All benchmarks:"
2572     allSummaryStats(outp, json, vmStatses, nil, 0)
2573
2574     scaledResultJson = {}
2575
2576     outp.puts
2577     printVMs(outp)
2578     outp.puts "Geomean of preferred means:"
2579     outp.print "   "
2580     outp.print rpad("<scaled-result>", $benchpad + $weightpad)
2581     outp.print " "
2582     $vms.size.times {
2583       | index |
2584       if index != 0
2585         outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm
2586       end
2587       outp.print statsToStr(overallResults[index], 0)
2588       scaledResultJson[$vms[index].name] = overallResults[index].jsonMap
2589     }
2590     if overallResults.size>=2
2591       outp.print("    "+overallResults[-1].compareTo(overallResults[0]).longForm)
2592     end
2593     outp.puts
2594
2595     json["<scaled-result>"] = scaledResultJson
2596   end
2597   outp.puts
2598
2599   if outp != $stdout
2600     outp.close
2601   end
2602
2603   if outp != $stdout and not $brief
2604     puts
2605     File.open(reportName + "_report.txt") {
2606       | inp |
2607       puts inp.read
2608     }
2609   end
2610
2611   if $brief
2612     puts(overallResults.collect{|stats| stats.mean}.join("\t"))
2613     puts(overallResults.collect{|stats| stats.confInt}.join("\t"))
2614   end
2615
2616   File.open(reportName + ".json", "w") {
2617     | outp |
2618     outp.puts json.to_json
2619   }
2620 end
2621
2622 begin
2623   $sawBenchOptions = false
2624
2625   def resetBenchOptionsIfNecessary
2626     unless $sawBenchOptions
2627       $includeSunSpider = false
2628       $includeLongSpider = false
2629       $includeV8 = false
2630       $includeKraken = false
2631       $includeJSBench = false
2632       $includeJSRegress = false
2633       $includeAsmBench = false
2634       $includeDSPJS = false
2635       $includeBrowsermarkJS = false
2636       $includeBrowsermarkDOM = false
2637       $includeOctane = false
2638       $includeCompressionBench = false
2639       $sawBenchOptions = true
2640     end
2641   end
2642
2643   GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
2644                  ['--inner', GetoptLong::REQUIRED_ARGUMENT],
2645                  ['--outer', GetoptLong::REQUIRED_ARGUMENT],
2646                  ['--warmup', GetoptLong::REQUIRED_ARGUMENT],
2647                  ['--no-ss-warmup', GetoptLong::NO_ARGUMENT],
2648                  ['--quantum', GetoptLong::REQUIRED_ARGUMENT],
2649                  ['--minimum', GetoptLong::REQUIRED_ARGUMENT],
2650                  ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
2651                  ['--sunspider', GetoptLong::NO_ARGUMENT],
2652                  ['--longspider', GetoptLong::NO_ARGUMENT],
2653                  ['--v8-spider', GetoptLong::NO_ARGUMENT],
2654                  ['--kraken', GetoptLong::NO_ARGUMENT],
2655                  ['--js-bench', GetoptLong::NO_ARGUMENT],
2656                  ['--js-regress', GetoptLong::NO_ARGUMENT],
2657                  ['--asm-bench', GetoptLong::NO_ARGUMENT],
2658                  ['--dsp', GetoptLong::NO_ARGUMENT],
2659                  ['--browsermark-js', GetoptLong::NO_ARGUMENT],
2660                  ['--browsermark-dom', GetoptLong::NO_ARGUMENT],
2661                  ['--octane', GetoptLong::NO_ARGUMENT],
2662                  ['--compression-bench', GetoptLong::NO_ARGUMENT],
2663                  ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
2664                  ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
2665                  ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
2666                  ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
2667                  ['--dont-copy-vms', GetoptLong::NO_ARGUMENT],
2668                  ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
2669                  ['--brief', GetoptLong::NO_ARGUMENT],
2670                  ['--silent', GetoptLong::NO_ARGUMENT],
2671                  ['--remote', GetoptLong::REQUIRED_ARGUMENT],
2672                  ['--local', GetoptLong::NO_ARGUMENT],
2673                  ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
2674                  ['--slave', GetoptLong::NO_ARGUMENT],
2675                  ['--prepare-only', GetoptLong::NO_ARGUMENT],
2676                  ['--analyze', GetoptLong::REQUIRED_ARGUMENT],
2677                  ['--vms', GetoptLong::REQUIRED_ARGUMENT],
2678                  ['--output-name', GetoptLong::REQUIRED_ARGUMENT],
2679                  ['--environment', GetoptLong::REQUIRED_ARGUMENT],
2680                  ['--config', GetoptLong::REQUIRED_ARGUMENT],
2681                  ['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
2682     | opt, arg |
2683     case opt
2684     when '--rerun'
2685       $rerun = intArg(opt,arg,1,nil)
2686     when '--inner'
2687       $inner = intArg(opt,arg,1,nil)
2688     when '--outer'
2689       $outer = intArg(opt,arg,1,nil)
2690     when '--warmup'
2691       $warmup = intArg(opt,arg,0,nil)
2692     when '--no-ss-warmup'
2693       $sunSpiderWarmup = false
2694     when '--quantum'
2695       $quantum = intArg(opt,arg,1,nil)
2696     when '--minimum'
2697       $minimum = intArg(opt,arg,1,nil)
2698     when '--timing-mode'
2699       if arg.upcase == "PRECISETIME"
2700         $timeMode = :preciseTime
2701       elsif arg.upcase == "DATE"
2702         $timeMode = :date
2703       elsif arg.upcase == "AUTO"
2704         $timeMode = :auto
2705       else
2706         quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
2707                   "Invalid argument for command-line option")
2708       end
2709     when '--force-vm-kind'
2710       if arg.upcase == "JSC"
2711         $forceVMKind = :jsc
2712       elsif arg.upcase == "DUMPRENDERTREE"
2713         $forceVMKind = :dumpRenderTree
2714       elsif arg.upcase == "WEBKITTESTRUNNER"
2715         $forceVMKind = :webkitTestRunner
2716       elsif arg.upcase == "AUTO"
2717         $forceVMKind = nil
2718       else
2719         quickFail("Expected 'jsc', 'DumpRenderTree', or 'WebKitTestRunner' for --force-vm-kind, but got '#{arg}'.",
2720                   "Invalid argument for command-line option")
2721       end
2722     when '--force-vm-copy'
2723       $needToCopyVMs = true
2724     when '--dont-copy-vms'
2725       $dontCopyVMs = true
2726     when '--sunspider'
2727       resetBenchOptionsIfNecessary
2728       $includeSunSpider = true
2729     when '--longspider'
2730       resetBenchOptionsIfNecessary
2731       $includeLongSpider = true
2732     when '--v8-spider'
2733       resetBenchOptionsIfNecessary
2734       $includeV8 = true
2735     when '--kraken'
2736       resetBenchOptionsIfNecessary
2737       $includeKraken = true
2738     when '--js-bench'
2739       resetBenchOptionsIfNecessary
2740       $includeJSBench = true
2741     when '--js-regress'
2742       resetBenchOptionsIfNecessary
2743       $includeJSRegress = true
2744     when '--asm-bench'
2745       resetBenchOptionsIfNecessary
2746       $includeAsmBench = true
2747     when '--dsp'
2748       resetBenchOptionsIfNecessary
2749       $includeDSPJS = true
2750     when '--browsermark-js'
2751       resetBenchOptionsIfNecessary
2752       $includeBrowsermarkJS = true
2753     when '--browsermark-dom'
2754       resetBenchOptionsIfNecessary
2755       $includeBrowsermarkDOM = true
2756     when '--octane'
2757       resetBenchOptionsIfNecessary
2758       $includeOctane = true
2759     when '--compression-bench'
2760       resetBenchOptionsIfNecessary
2761       $includeCompressionBench = true
2762     when '--benchmarks'
2763       $benchmarkPattern = Regexp.new(arg)
2764     when '--measure-gc'
2765       if arg == ''
2766         $measureGC = true
2767       else
2768         $measureGC = arg
2769       end
2770     when '--verbose'
2771       $verbosity += 1
2772     when '--brief'
2773       $brief = true
2774     when '--silent'
2775       $silent = true
2776     when '--remote'
2777       $remoteHosts += arg.split(',')
2778       $needToCopyVMs = true
2779     when '--ssh-options'
2780       $sshOptions << arg
2781     when '--local'
2782       $alsoLocal = true
2783     when '--prepare-only'
2784       $run = false
2785     when '--analyze'
2786       $prepare = false
2787       $run = false
2788       $analyze << arg
2789     when '--output-name'
2790       $outputName = arg
2791     when '--vms'
2792       JSON::parse(IO::read(arg)).each {
2793         | vmDescription |
2794         path = Pathname.new(vmDescription["path"]).realpath
2795         if vmDescription["name"]
2796           name = vmDescription["name"]
2797           nameKind = :given
2798         else
2799           name = "Conf\##{$vms.length+1}"
2800           nameKind = :auto
2801         end
2802         vm = VM.new(path, name, nameKind, nil)
2803         if vmDescription["env"]
2804           vmDescription["env"].each_pair {
2805             | key, val |
2806             vm.addExtraEnv(key, val)
2807           }
2808         end
2809         $vms << vm
2810       }
2811     when '--environment'
2812       $environment = JSON::parse(IO::read(arg))
2813     when '--config'
2814       $configPath = Pathname.new(arg)
2815     when '--help'
2816       usage
2817     else
2818       raise "bad option: #{opt}"
2819     end
2820   }
2821
2822   # Figure out the configuration
2823   if $configPath.file?
2824     config = JSON::parse(IO::read($configPath.to_s))
2825   else
2826     config = {}
2827   end
2828   OCTANE_PATH = config["OctanePath"]
2829   BROWSERMARK_PATH = config["BrowserMarkPath"]
2830   BROWSERMARK_JS_PATH = config["BrowserMarkJSPath"]
2831   BROWSERMARK_DOM_PATH = config["BrowserMarkDOMPath"]
2832   ASMBENCH_PATH = config["AsmBenchPath"]
2833   COMPRESSIONBENCH_PATH = config["CompressionBenchPath"]
2834   DSPJS_FILTRR_PATH = config["DSPJSFiltrrPath"]
2835   DSPJS_ROUTE9_PATH = config["DSPJSRoute9Path"]
2836   DSPJS_STARFIELD_PATH = config["DSPJSStarfieldPath"]
2837   DSPJS_QUAKE3_PATH = config["DSPJSQuake3Path"]
2838   DSPJS_MANDELBROT_PATH = config["DSPJSMandelbrotPath"]
2839   DSPJS_JSLINUX_PATH = config["DSPJSLinuxPath"]
2840   DSPJS_AMMOJS_ASMJS_PATH = config["DSPJSAmmoJSAsmJSPath"]
2841   DSPJS_AMMOJS_REGULAR_PATH = config["DSPJSAmmoJSRegularPath"]
2842   JSBENCH_PATH = config["JSBenchPath"]
2843   KRAKEN_PATH = config["KrakenPath"]
2844
2845   # If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option.
2846   if $dontCopyVMs
2847     $needToCopyVMs = false
2848   end
2849
2850   ARGV.each {
2851     | vm |
2852     if vm =~ /([a-zA-Z0-9_ ]+):/
2853       name = $1
2854       nameKind = :given
2855       vm = $~.post_match
2856     else
2857       name = "Conf\##{$vms.length+1}"
2858       nameKind = :auto
2859     end
2860     envs = []
2861     while vm =~ /([a-zA-Z0-9_]+)=([a-zA-Z0-9_:]+):/
2862       envs << [$1, $2]
2863       vm = $~.post_match
2864     end
2865     $stderr.puts "#{name}: #{vm}" if $verbosity >= 1
2866     vm = VM.new(Pathname.new(vm).realpath, name, nameKind, nil)
2867     envs.each {
2868       | pair |
2869       vm.addExtraEnv(pair[0], pair[1])
2870     }
2871     $vms << vm
2872   }
2873
2874   if $vms.empty?
2875     quickFail("Please specify at least on configuraiton on the command line.",
2876               "Insufficient arguments")
2877   end
2878
2879   $vms.each {
2880     | vm |
2881     if vm.vmType == :jsc
2882       $allDRT = false
2883     end
2884   }
2885
2886   SUNSPIDER = BenchmarkSuite.new("SunSpider", :arithmeticMean, 0)
2887   WARMUP = BenchmarkSuite.new("WARMUP", :arithmeticMean, 0)
2888   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
2889    "access-fannkuch", "access-nbody", "access-nsieve",
2890    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
2891    "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
2892    "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
2893    "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
2894    "string-base64", "string-fasta", "string-tagcloud",
2895    "string-unpack-code", "string-validate-input"].each {
2896     | name |
2897     SUNSPIDER.add SunSpiderBenchmark.new(name)
2898     WARMUP.addIgnoringPattern SunSpiderBenchmark.new(name)
2899   }
2900
2901   LONGSPIDER = BenchmarkSuite.new("LongSpider", :geometricMean, 0)
2902   ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
2903    "access-fannkuch", "access-nbody", "access-nsieve",
2904    "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-nsieve-bits",
2905    "controlflow-recursive", "crypto-aes", "crypto-md5", "crypto-sha1",
2906    "date-format-tofte", "date-format-xparb", "math-cordic",
2907    "math-partial-sums", "math-spectral-norm", "string-base64",
2908    "string-fasta", "string-tagcloud"].each {
2909     | name |
2910     LONGSPIDER.add LongSpiderBenchmark.new(name)
2911   }
2912
2913   V8 = BenchmarkSuite.new("V8Spider", :geometricMean, 0)
2914   ["crypto", "deltablue", "earley-boyer", "raytrace",
2915    "regexp", "richards", "splay"].each {
2916     | name |
2917     V8.add V8Benchmark.new(name)
2918   }
2919
2920   OCTANE = BenchmarkSuite.new("Octane", :geometricMean, 1)
2921   [[["crypto"], "encrypt", 1, true, false, 32],
2922    [["crypto"], "decrypt", 1, true, false, 32],
2923    [["deltablue"], "deltablue", 2, true, false, 32],
2924    [["earley-boyer"], "earley", 1, true, false, 32],
2925    [["earley-boyer"], "boyer", 1, true, false, 32],
2926    [["navier-stokes"], "navier-stokes", 2, true, false, 16],
2927    [["raytrace"], "raytrace", 2, true, false, 32],
2928    [["richards"], "richards", 2, true, false, 32],
2929    [["splay"], "splay", 2, true, false, 32],
2930    [["regexp"], "regexp", 2, false, false, 16],
2931    [["pdfjs"], "pdfjs", 2, false, false, 4],
2932    [["mandreel"], "mandreel", 2, false, false, 4],
2933    [["gbemu-part1", "gbemu-part2"], "gbemu", 2, false, false, 4],
2934    [["code-load"], "closure", 1, false, false, 16],
2935    [["code-load"], "jquery", 1, false, false, 16],
2936    [["box2d"], "box2d", 2, false, false, 8],
2937    [["zlib", "zlib-data"], "zlib", 2, false, true, 3],
2938    [["typescript", "typescript-input", "typescript-compiler"], "typescript", 2, false, true, 1]].each {
2939     | args |
2940     OCTANE.add OctaneBenchmark.new(*args)
2941   }
2942
2943   KRAKEN = BenchmarkSuite.new("Kraken", :arithmeticMean, -1)
2944   ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
2945    "audio-oscillator", "imaging-darkroom", "imaging-desaturate",
2946    "imaging-gaussian-blur", "json-parse-financial",
2947    "json-stringify-tinderbox", "stanford-crypto-aes",
2948    "stanford-crypto-ccm", "stanford-crypto-pbkdf2",
2949    "stanford-crypto-sha256-iterative"].each {
2950     | name |
2951     KRAKEN.add KrakenBenchmark.new(name)
2952   }
2953
2954   JSBENCH = BenchmarkSuite.new("JSBench", :arithmeticMean, 0)
2955   [["amazon", "urm"], ["facebook", "urem"], ["google", "urem"], ["twitter", "urem"],
2956    ["yahoo", "urem"]].each {
2957     | nameAndMode |
2958     JSBENCH.add JSBenchBenchmark.new(*nameAndMode)
2959   }
2960
2961   JSREGRESS = BenchmarkSuite.new("JSRegress", :geometricMean, 0)
2962   Dir.foreach(JSREGRESS_PATH) {
2963     | filename |
2964     if filename =~ /\.js$/
2965       name = $~.pre_match
2966       JSREGRESS.add JSRegressBenchmark.new(name)
2967     end
2968   }
2969
2970   ASMBENCH = BenchmarkSuite.new("AsmBench", :geometricMean, 0)
2971   if JSBENCH_PATH
2972     Dir.foreach(ASMBENCH_PATH) {
2973       | filename |
2974       if filename =~ /\.js$/
2975         name = $~.pre_match
2976         ASMBENCH.add AsmBenchBenchmark.new(name)
2977       end
2978     }
2979   end
2980
2981   COMPRESSIONBENCH = BenchmarkSuite.new("CompressionBench", :geometricMean, 0)
2982   [[["huffman", "compression-data"], "huffman", ""],
2983    [["arithmetic", "compression-data"], "arithmetic", "Simple"],
2984    [["arithmetic", "compression-data"], "arithmetic", "Precise"],
2985    [["arithmetic", "compression-data"], "arithmetic", "Complex Precise"],
2986    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 0"],
2987    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 1"],
2988    [["arithmetic", "compression-data"], "arithmetic", "Precise Order 2"],
2989    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 1"],
2990    [["arithmetic", "compression-data"], "arithmetic", "Simple Order 2"],
2991    [["lz-string", "compression-data"], "lz-string", ""]
2992   ].each {
2993     | args |
2994     COMPRESSIONBENCH.add CompressionBenchBenchmark.new(*args)
2995   }
2996
2997   DSPJS = BenchmarkSuite.new("DSP", :geometricMean, 0)
2998   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-posterize-tint", "e2")
2999   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-contrast-sat-bright", "e5")
3000   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-tint-sat-adj-contr-mult", "e7")
3001   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-blur-overlay-sat-contr", "e8")
3002   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sat-blur-mult-sharpen-contr", "e9")
3003   DSPJS.add DSPJSFiltrrBenchmark.new("filtrr-sepia-bias", "e10")
3004   DSPJS.add DSPJSVP8Benchmark.new
3005   DSPJS.add DSPStarfieldBenchmark.new
3006   DSPJS.add DSPJSJSLinuxBenchmark.new
3007   DSPJS.add DSPJSQuake3Benchmark.new
3008   DSPJS.add DSPJSMandelbrotBenchmark.new
3009   DSPJS.add DSPJSAmmoJSASMBenchmark.new
3010   DSPJS.add DSPJSAmmoJSRegularBenchmark.new
3011
3012   BROWSERMARK_JS = BenchmarkSuite.new("BrowsermarkJS", :geometricMean, 1)
3013   ["array_blur", "array_weighted", "string_chat", "string_filter", "string_weighted"].each {
3014     | name |
3015       BROWSERMARK_JS.add BrowsermarkJSBenchmark.new(name)
3016   }
3017
3018   BROWSERMARK_DOM = BenchmarkSuite.new("BrowsermarkDOM", :geometricMean, 1)
3019   ["advanced_search", "create_source", "dynamic_create", "search"].each {
3020     | name |
3021       BROWSERMARK_DOM.add BrowsermarkDOMBenchmark.new(name)
3022   }
3023
3024   $suites = []
3025
3026   if $includeSunSpider and not SUNSPIDER.empty?
3027     $suites << SUNSPIDER
3028   end
3029
3030   if $includeLongSpider and not LONGSPIDER.empty?
3031     $suites << LONGSPIDER
3032   end
3033
3034   if $includeV8 and not V8.empty?
3035     $suites << V8
3036   end
3037
3038   if $includeOctane and not OCTANE.empty?
3039     if OCTANE_PATH
3040       $suites << OCTANE
3041     else
3042       $stderr.puts "Warning: refusing to run Octane because \"OctanePath\" isn't set in #{$configPath}."
3043     end
3044   end
3045
3046   if $includeKraken and not KRAKEN.empty?
3047     if KRAKEN_PATH
3048       $suites << KRAKEN
3049     else
3050       $stderr.puts "Warning: refusing to run Kraken because \"KrakenPath\" isn't set in #{$configPath}."
3051     end
3052   end
3053
3054   if $includeJSBench and not JSBENCH.empty?
3055     if $allDRT
3056       if JSBENCH_PATH
3057         $suites << JSBENCH
3058       else
3059         $stderr.puts "Warning: refusing to run JSBench because \"JSBenchPath\" isn't set in #{$configPath}"
3060       end
3061     else
3062       $stderr.puts "Warning: refusing to run JSBench because not all VMs are DumpRenderTree or WebKitTestRunner."
3063     end
3064   end
3065
3066   if $includeJSRegress and not JSREGRESS.empty?
3067     $suites << JSREGRESS
3068   end
3069
3070   if $includeAsmBench and not ASMBENCH.empty?
3071     if ASMBENCH_PATH
3072       $suites << ASMBENCH
3073     else
3074       $stderr.puts "Warning: refusing to run AsmBench because \"AsmBenchPath\" isn't set in #{$configPath}."
3075     end
3076   end
3077
3078   if $includeDSPJS and not DSPJS.empty?
3079     if $allDRT
3080       if DSPJS_FILTRR_PATH and DSPJS_ROUTE9_PATH and DSPJS_STARFIELD_PATH and DSPJS_QUAKE3_PATH and DSPJS_MANDELBROT_PATH and DSPJS_JSLINUX_PATH and DSPJS_AMMOJS_ASMJS_PATH and DSPJS_AMMOJS_REGULAR_PATH
3081         $suites << DSPJS
3082       else
3083         $stderr.puts "Warning: refusing to run DSPJS because one of the following isn't set in #{$configPath}: \"DSPJSFiltrrPath\", \"DSPJSRoute9Path\", \"DSPJSStarfieldPath\", \"DSPJSQuake3Path\", \"DSPJSMandelbrotPath\", \"DSPJSLinuxPath\", \"DSPJSAmmoJSAsmJSPath\", \"DSPJSAmmoJSRegularPath\"."
3084       end
3085     else
3086       $stderr.puts "Warning: refusing to run DSPJS because not all VMs are DumpRenderTree or WebKitTestRunner."
3087     end
3088   end
3089
3090   if $includeBrowsermarkJS and not BROWSERMARK_JS.empty?
3091     if BROWSERMARK_PATH and BROWSERMARK_JS_PATH
3092       $suites << BROWSERMARK_JS
3093     else
3094       $stderr.puts "Warning: refusing to run Browsermark-JS because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\" or \"BrowserMarkJSPath\"."
3095     end
3096   end
3097
3098   if $includeBrowsermarkDOM and not BROWSERMARK_DOM.empty?
3099     if $allDRT
3100       if BROWSERMARK_PATH and BROWSERMARK_JS_PATH and BROWSERMARK_DOM_PATH
3101         $suites << BROWSERMARK_DOM
3102       else
3103         $stderr.puts "Warning: refusing to run Browsermark-DOM because one of the following isn't set in #{$configPath}: \"BrowserMarkPath\", \"BrowserMarkJSPath\", or \"BrowserMarkDOMPath\"."
3104       end
3105     else
3106       $stderr.puts "Warning: refusing to run Browsermark-DOM because not all VMs are DumpRenderTree or WebKitTestRunner."
3107     end
3108   end
3109
3110   if $includeCompressionBench and not COMPRESSIONBENCH.empty?
3111     if COMPRESSIONBENCH_PATH
3112       $suites << COMPRESSIONBENCH
3113     else
3114       $stderr.puts "Warning: refusing to run CompressionBench because \"CompressionBenchPath\" isn't set in #{$configPath}"
3115     end
3116   end
3117
3118   $allSuites = $suites.map{|v| v.suites}.flatten(1)
3119
3120   $benchmarks = []
3121   $suites.each {
3122     | suite |
3123     $benchmarks += suite.benchmarks
3124   }
3125
3126   if $suites.empty? or $benchmarks.empty?
3127     $stderr.puts "No benchmarks found.  Bailing out."
3128     exit 1
3129   end
3130
3131   if $outer*$inner == 1
3132     $stderr.puts "Warning: will only collect one sample per benchmark/VM.  Confidence interval calculation will fail."
3133   end
3134
3135   $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
3136
3137   $runPlans = []
3138   $vms.each {
3139     | vm |
3140     $benchmarks.each {
3141       | benchmark |
3142       $outer.times {
3143         | iteration |
3144         $runPlans << BenchRunPlan.new(benchmark, vm, iteration)
3145       }
3146     }
3147   }
3148
3149   $runPlans.shuffle!
3150
3151   if $sunSpiderWarmup
3152     warmupPlans = []
3153     $vms.each {
3154       | vm |
3155       WARMUP.benchmarks.each {
3156         | benchmark |
3157         warmupPlans << BenchRunPlan.new(benchmark, vm, 0)
3158       }
3159     }
3160
3161     $runPlans = warmupPlans.shuffle + $runPlans
3162   end
3163
3164   $suitepad = $suites.collect {
3165     | suite |
3166     suite.to_s.size
3167   }.max + 1
3168
3169   $planpad = $runPlans.collect {
3170     | plan |
3171     plan.to_s.size
3172   }.max + 1
3173
3174   maxBenchNameLength =
3175     ($benchmarks + ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
3176     | benchmark |
3177     if benchmark.respond_to? :name
3178       benchmark.name.size
3179     else
3180       benchmark.size
3181     end
3182   }.max
3183   $benchNameClip = 40
3184   $benchpad = [maxBenchNameLength, $benchNameClip].min + 1
3185
3186   $weightpad = $benchmarks.collect {
3187     | benchmark |
3188     benchmark.weightString.size
3189   }.max
3190
3191   $vmpad = $vms.collect {
3192     | vm |
3193     vm.to_s.size
3194   }.max + 1
3195
3196   $analyze.each_with_index {
3197     | filename, index |
3198     if index >= 1
3199       puts
3200     end
3201     parseAndDisplayResults(IO::read(filename))
3202   }
3203
3204   if not $prepare and not $run
3205     exit 0
3206   end
3207
3208   if FileTest.exist? BENCH_DATA_PATH
3209     cmd = "rm -rf #{BENCH_DATA_PATH}"
3210     $stderr.puts ">> #{cmd}" if $verbosity >= 2
3211     raise unless system cmd
3212   end
3213
3214   Dir.mkdir BENCH_DATA_PATH
3215
3216   if $needToCopyVMs
3217     canCopyIntoBenchPath = true
3218     $vms.each {
3219       | vm |
3220       canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath
3221     }
3222
3223     if canCopyIntoBenchPath
3224       $vms.each {
3225         | vm |
3226         $stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..."
3227         vm.copyIntoBenchPath
3228       }
3229       $stderr.puts "All VMs are in place."
3230     else
3231       $stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it."
3232     end
3233   end
3234
3235   if $measureGC and $measureGC != true
3236     found = false
3237     $vms.each {
3238       | vm |
3239       if vm.name == $measureGC
3240         found = true
3241       end
3242     }
3243     unless found
3244       $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
3245     end
3246   end
3247
3248   if $prepare
3249     File.open("#{BENCH_DATA_PATH}/runscript", "w") {
3250       | file |
3251       file.puts "echo \"HOSTNAME:\\c\""
3252       file.puts "hostname"
3253       file.puts "echo"
3254       file.puts "echo \"HARDWARE:\\c\""
3255       file.puts "/usr/sbin/sysctl hw.model"
3256       file.puts "echo"
3257       file.puts "set -e"
3258       $script = file
3259       $runPlans.each_with_index {
3260         | plan, idx |
3261         if $verbosity == 0 and not $silent
3262           text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s
3263           text2 = plan.to_s
3264           file.puts("echo " + Shellwords.shellescape("\r#{text1} #{rpad(text2,$planpad)}") + "\"\\c\" 1>&2")
3265           file.puts("echo " + Shellwords.shellescape("\r#{text1} #{text2}") + "\"\\c\" 1>&2")
3266         end
3267         plan.emitRunCode
3268       }
3269       if $verbosity == 0 and not $silent
3270         file.puts("echo " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}") + "\"\\c\" 1>&2")
3271         file.puts("echo " + Shellwords.shellescape("\r#{$runPlans.size}/#{$runPlans.size}") + " 1>&2")
3272       end
3273     }
3274   end
3275
3276   if $run
3277     unless $remoteHosts.empty?
3278       $stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0
3279       Dir.chdir(TEMP_PATH) {
3280         cmd = "tar -czf payload.tar.gz benchdata"
3281         $stderr.puts ">> #{cmd}" if $verbosity>=2
3282         raise unless system(cmd)
3283       }
3284
3285       def grokHost(host)
3286         if host =~ /:([0-9]+)$/
3287           "-p " + $1 + " " + Shellwords.shellescape($~.pre_match)
3288         else
3289           Shellwords.shellescape(host)
3290         end
3291       end
3292
3293       def sshRead(host, command)
3294         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3295         $stderr.puts ">> #{cmd}" if $verbosity>=2
3296         result = ""
3297         IO.popen(cmd, "r") {
3298           | inp |
3299           inp.each_line {
3300             | line |
3301             $stderr.puts "#{host}: #{line}" if $verbosity>=2
3302             result += line
3303           }
3304         }
3305         raise "#{$?}" unless $?.success?
3306         result
3307       end
3308
3309       def sshWrite(host, command, data)
3310         cmd = "ssh #{$sshOptions.collect{|x| Shellwords.shellescape(x)}.join(' ')} #{grokHost(host)} #{Shellwords.shellescape(command)}"
3311         $stderr.puts ">> #{cmd}" if $verbosity>=2
3312         IO.popen(cmd, "w") {
3313           | outp |
3314           outp.write(data)
3315         }
3316         raise "#{$?}" unless $?.success?
3317       end
3318
3319       $remoteHosts.each {
3320         | host |
3321         $stderr.puts "Sending benchmark payload to #{host}..." if $verbosity==0
3322
3323         remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"]
3324         raise unless remoteTempPath
3325
3326         sshWrite(host, "cd #{Shellwords.shellescape(remoteTempPath)} && rm -rf benchdata && tar -xz", IO::read("#{TEMP_PATH}/payload.tar.gz"))
3327
3328         $stderr.puts "Running on #{host}..." if $verbosity==0
3329
3330         parseAndDisplayResults(sshRead(host, "cd #{Shellwords.shellescape(remoteTempPath + '/benchdata')} && sh runscript"))
3331       }
3332     end
3333
3334     if not $remoteHosts.empty? and $alsoLocal
3335       $stderr.puts "Running locally..."
3336     end
3337
3338     if $remoteHosts.empty? or $alsoLocal
3339       parseAndDisplayResults(runAndGetResults)
3340     end
3341   end
3342
3343   if $prepare and not $run and $analyze.empty?
3344     puts wrap("Benchmarking script and data are in #{BENCH_DATA_PATH}. You can run "+
3345               "the benchmarks and get the results by doing:", 78)
3346     puts
3347     puts "cd #{BENCH_DATA_PATH}"
3348     puts "sh runscript > results.txt"
3349     puts
3350     puts wrap("Then you can analyze the results by running bencher with the same arguments "+
3351               "as now, but replacing --prepare-only with --analyze results.txt.", 78)
3352   end
3353 rescue => e
3354   fail(e)
3355 end
3356