Butterflies should be allocated in Auxiliary MarkedSpace instead of CopiedSpace and...
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2016 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
47 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
48 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
49 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
50 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
51 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
52
53 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
54
55 begin
56     require 'shellwords'
57 rescue Exception => e
58     $stderr.puts "Warning: did not find shellwords, not running any tests."
59     exit 0
60 end
61
62 $canRunDisplayProfilerOutput = false
63
64 begin
65     require 'rubygems'
66     require 'json'
67     require 'highline'
68     $canRunDisplayProfilerOutput = true
69 rescue Exception => e
70     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
71     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
72     $stderr.puts "Error: #{e.inspect}"
73 end
74
75 def printCommandArray(*cmd)
76     begin
77         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
78     rescue
79         commandArray = cmd.join(' ')
80     end
81     $stderr.puts ">> #{commandArray}"
82 end
83
84 def mysys(*cmd)
85     printCommandArray(*cmd) if $verbosity >= 1
86     raise "Command failed: #{$?.inspect}" unless system(*cmd)
87 end
88
89 def escapeAll(array)
90     array.map {
91         | v |
92         raise "Detected a non-string in #{inspect}" unless v.is_a? String
93         Shellwords.shellescape(v)
94     }.join(' ')
95 end
96
97
98 $jscPath = nil
99 $doNotMessWithVMPath = false
100 $jitTests = true
101 $memoryLimited = false
102 $outputDir = Pathname.new("results")
103 $verbosity = 0
104 $bundle = nil
105 $tarball = false
106 $tarFileName = "payload.tar.gz"
107 $copyVM = false
108 $testRunnerType = nil
109 $remoteUser = nil
110 $remoteHost = nil
111 $remotePort = nil
112 $remoteDirectory = nil
113 $architecture = nil
114 $hostOS = nil
115 $filter = nil
116 $envVars = []
117 $quickMode = false
118
119
120 def usage
121     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
122     puts
123     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
124     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
125     puts "                            --jsc specifies an already present JavaScriptCore to test."
126     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
127     puts "                            Skip tests tagged with //@large-heap"
128     puts "--no-jit                    Do not run JIT specific tests."
129     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
130     puts "--verbose            (-v)   Print more things while running."
131     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
132     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
133     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
134     puts "                            e.g. x86, x86_64, arm."
135     puts "--os                        Specify os instead of determining from JavaScriptCore build."
136     puts "                            e.g. darwin, linux & windows."
137     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
138     puts "                            In general the shell runner is slower than the make runner."
139     puts "--make-runner               Uses the faster make-based runner."
140     puts "--remote                    Specify a remote host on which to run tests from command line argument."
141     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
142     puts "--child-processes    (-c)   Specify the number of child processes."
143     puts "--filter                    Only run tests whose name matches the given regular expression."
144     puts "--help               (-h)   Print this message."
145     puts "--env-vars                  Add a list of environment variables to set before running jsc."
146     puts "                            Each environment variable should be separated by a space."
147     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
148     puts "                            it will override the default value."
149     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
150     exit 1
151 end
152
153 jscArg = nil
154 ifJSCArgIsntProvidedAreWeReleaseBuild = true
155
156 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
157                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
158                ['--no-copy', GetoptLong::NO_ARGUMENT],
159                ['--memory-limited', GetoptLong::NO_ARGUMENT],
160                ['--no-jit', GetoptLong::NO_ARGUMENT],
161                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
162                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
163                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
164                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
165                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
166                ['--os', GetoptLong::REQUIRED_ARGUMENT],
167                ['--shell-runner', GetoptLong::NO_ARGUMENT],
168                ['--make-runner', GetoptLong::NO_ARGUMENT],
169                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
170                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
171                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
172                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
173                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
174                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
175                ['--debug', GetoptLong::NO_ARGUMENT],
176                ['--release', GetoptLong::NO_ARGUMENT],
177                ['--quick', '-q', GetoptLong::NO_ARGUMENT]).each {
178     | opt, arg |
179     case opt
180     when '--help'
181         usage
182     when '--jsc'
183         jscArg = arg
184     when '--no-copy'
185         $doNotMessWithVMPath = true
186     when '--output-dir'
187         $outputDir = Pathname.new(arg)
188     when '--memory-limited'
189         $memoryLimited = true
190     when '--no-jit'
191         $jitTests = false
192     when '--verbose'
193         $verbosity += 1
194     when '--run-bundle'
195         $bundle = Pathname.new(arg)
196     when '--tarball'
197         $tarball = true
198         $copyVM = true
199         $tarFileName = arg unless arg == ''
200     when '--force-vm-copy'
201         $copyVM = true
202     when '--shell-runner'
203         $testRunnerType = :shell
204     when '--make-runner'
205         $testRunnerType = :make
206     when '--remote'
207         $copyVM = true
208         $tarball = true
209         $remote = true
210         uri = URI("ssh://" + arg)
211         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
212     when '--remote-config-file'
213         $remoteConfigFile = arg
214     when '--child-processes'
215         $numChildProcesses = arg.to_i
216     when '--filter'
217         $filter = Regexp.new(arg)
218     when '--arch'
219         $architecture = arg
220     when '--os'
221         $hostOS = arg
222     when '--env-vars'
223         $envVars = arg.gsub(/\s+/, ' ').split(' ')
224     when '--quick'
225         $quickMode = true
226     when '--debug'
227         ifJSCArgIsntProvidedAreWeReleaseBuild = false
228     when '--release'
229         ifJSCArgIsntProvidedAreWeReleaseBuild = true
230     end
231 }
232
233 if $remoteConfigFile
234     file = File.read($remoteConfigFile)
235     config = JSON.parse(file)
236
237     if !$remote and config['remote']
238         $copyVM = true
239         $tarball = true
240         $remote = true
241         uri = URI("ssh://" + config['remote'])
242         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
243     end
244
245     if config['remoteDirectory']
246         $remoteDirectory = config['remoteDirectory']
247     end
248 end
249
250 unless jscArg
251     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
252     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
253     command += ifJSCArgIsntProvidedAreWeReleaseBuild ? " --release" : " --debug"
254     command += " --executablePath"
255
256     output = `#{command}`.split("\n")
257     if !output.length
258         $stderr.puts "Error: must specify --jsc <path>"
259         exit 1
260     end
261
262     output = output[0]
263     jscArg = Pathname.new(output).join("jsc")
264     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
265     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
266     if !File.file?(jscArg)
267         $stderr.puts "Error: must specify --jsc <path>"
268         exit 1
269     end
270
271     puts "Using the following jsc path: #{jscArg}"
272 end
273
274 if $doNotMessWithVMPath
275     $jscPath = Pathname.new(jscArg)
276 else
277     $jscPath = Pathname.new(jscArg).realpath
278 end
279
280 $progressMeter = ($verbosity == 0 and $stdout.tty?)
281
282 if $bundle
283     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
284     $outputDir = $bundle
285 end
286
287 # Try to determine architecture. Return nil on failure.
288 def machOArchitectureCode
289     begin 
290         otoolLines = `otool -aSfh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
291         otoolLines.each_with_index {
292             | value, index |
293             if value =~ /magic/ and value =~ /cputype/
294                 return otoolLines[index + 1].split[1].to_i
295             end
296         }
297     rescue
298         $stderr.puts "Warning: unable to execute otool."
299     end
300     $stderr.puts "Warning: unable to determine architecture."
301     nil
302 end
303
304 def determineArchitectureFromMachOBinary
305     code = machOArchitectureCode
306     return nil unless code
307     is64BitFlag = 0x01000000
308     case code
309     when 7
310         "x86"
311     when 7 | is64BitFlag
312         "x86-64"
313     when 12
314         "arm"
315     when 12 | is64BitFlag
316         "arm64"
317     else
318         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
319         nil
320     end
321 end
322
323 def determineArchitectureFromELFBinary
324     f = File.open($jscPath.to_s)
325     data = f.read(19)
326
327     if !(data[0,4] == "\x7F\x45\x4C\x46")
328         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
329         return nil
330     end
331
332     code = data[18].ord
333     case code
334     when 3
335         "x86"
336     when 62
337         "x86-64"
338     when 40
339         "arm"
340     when 183
341         "arm64"
342     else
343         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
344         nil
345     end
346 end
347
348 def determineArchitectureFromPEBinary
349     f = File.open($jscPath.to_s)
350     data = f.read(1024)
351
352     if !(data[0, 2] == "MZ")
353         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
354         return nil
355     end
356
357     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
358
359     if !(data[peHeaderAddr, 4] == "PE\0\0")
360         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
361         return nil
362     end
363
364     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
365
366     case machine
367     when 0x014c
368         "x86"
369     when 0x8664
370         "x86-64"
371     else
372         $stderr.puts "Warning: unsupported machine type: #{machine}"
373         nil
374     end
375 end
376
377 def determineArchitecture
378     case $hostOS
379     when "darwin"
380         determineArchitectureFromMachOBinary
381     when "linux"
382         determineArchitectureFromELFBinary
383     when "windows"
384         determineArchitectureFromPEBinary
385     else
386         $stderr.puts "Warning: unable to determine architecture on this platform."
387         nil
388     end
389 end
390
391 def determineOS
392     case RbConfig::CONFIG["host_os"]
393     when /darwin/i
394         "darwin"
395     when /linux/i
396         "linux"
397     when /mswin|mingw|cygwin/
398         "windows"
399     else
400         $stderr.puts "Warning: unable to determine host operating system"
401         nil
402     end
403 end
404
405 $hostOS = determineOS unless $hostOS
406 $architecture = determineArchitecture unless $architecture
407 $isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $hostOS == "windows" || $hostOS == "linux" && $architecture == "arm64")
408
409 if !$testRunnerType
410     if $remote and $hostOS == "darwin"
411         $testRunnerType = :shell
412     else
413         $testRunnerType = :make
414     end
415 end
416
417 $numFailures = 0
418 $numPasses = 0
419
420 # We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
421 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--maxPerThreadStackUsage=1572864"]
422 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
423 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
424 FTL_OPTIONS = ["--useFTLJIT=true"]
425
426 $runlist = []
427
428 def frameworkFromJSCPath(jscPath)
429     parentDirectory = jscPath.dirname
430     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
431         parentDirectory.dirname
432     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
433         jscPath.dirname + "JavaScriptCore.framework"
434     else
435         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
436         nil
437     end
438 end
439
440 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
441     dir = Pathname.new(".")
442     $benchmarkDirectory.each_filename {
443         | pathComponent |
444         dir += ".."
445     }
446     dir + resourcePath
447 end
448
449 def pathToVM
450     pathToBundleResourceFromBenchmarkDirectory($jscPath)
451 end
452
453 def pathToHelpers
454     pathToBundleResourceFromBenchmarkDirectory(".helpers")
455 end
456
457 def prefixCommand(prefix)
458     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
459 end
460
461 def redirectAndPrefixCommand(prefix)
462     prefixCommand(prefix) + " 2>&1"
463 end
464
465 def pipeAndPrefixCommand(outputFilename, prefix)
466     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
467 end
468
469 # Output handler for tests that are expected to be silent.
470 def silentOutputHandler
471     Proc.new {
472         | name |
473         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
474     }
475 end
476
477 # Output handler for tests that are expected to produce meaningful output.
478 def noisyOutputHandler
479     Proc.new {
480         | name |
481         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
482     }
483 end
484
485 # Error handler for tests that fail exactly when they return non-zero exit status.
486 # This is useful when a test is expected to fail.
487 def simpleErrorHandler
488     Proc.new {
489         | outp, plan |
490         outp.puts "if test -e #{plan.failFile}"
491         outp.puts "then"
492         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
493         outp.puts "    " + plan.failCommand
494         outp.puts "else"
495         outp.puts "    " + plan.successCommand
496         outp.puts "fi"
497     }
498 end
499
500 # Error handler for tests that fail exactly when they return zero exit status.
501 def expectedFailErrorHandler
502     Proc.new {
503         | outp, plan |
504         outp.puts "if test -e #{plan.failFile}"
505         outp.puts "then"
506         outp.puts "    " + plan.successCommand
507         outp.puts "else"
508         outp.puts "    (echo ERROR: Unexpected exit code: 0) | " + redirectAndPrefixCommand(plan.name)
509         outp.puts "    " + plan.failCommand
510         outp.puts "fi"
511     }
512 end
513
514 # Error handler for tests that fail exactly when they return non-zero exit status and produce
515 # lots of spew. This will echo that spew when the test fails.
516 def noisyErrorHandler
517     Proc.new {
518         | outp, plan |
519         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
520     
521         outp.puts "if test -e #{plan.failFile}"
522         outp.puts "then"
523         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
524         outp.puts "    " + plan.failCommand
525         outp.puts "else"
526         outp.puts "    " + plan.successCommand
527         outp.puts "fi"
528     }
529 end
530
531 # Error handler for tests that diff their output with some expectation.
532 def diffErrorHandler(expectedFilename)
533     Proc.new {
534         | outp, plan |
535         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
536         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
537         
538         outp.puts "if test -e #{plan.failFile}"
539         outp.puts "then"
540         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
541         outp.puts "    " + plan.failCommand
542         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
543         outp.puts "then"
544         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
545         outp.puts "    if [ $? -eq 0 ]"
546         outp.puts "    then"
547         outp.puts "    " + plan.successCommand
548         outp.puts "    else"
549         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
550         outp.puts "        " + plan.failCommand
551         outp.puts "    fi"
552         outp.puts "else"
553         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
554         outp.puts "    " + plan.failCommand
555         outp.puts "fi"
556     }
557 end
558
559 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
560 # tests.
561 def mozillaErrorHandler
562     Proc.new {
563         | outp, plan |
564         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
565
566         outp.puts "if test -e #{plan.failFile}"
567         outp.puts "then"
568         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
569         outp.puts "    " + plan.failCommand
570         outp.puts "elif grep -i -q failed! #{outputFilename}"
571         outp.puts "then"
572         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
573         outp.puts "    " + plan.failCommand
574         outp.puts "else"
575         outp.puts "    " + plan.successCommand
576         outp.puts "fi"
577     }
578 end
579
580 # Error handler for tests that report error by saying "failed!", and are expected to
581 # fail. This is used by Mozilla tests.
582 def mozillaFailErrorHandler
583     Proc.new {
584         | outp, plan |
585         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
586
587         outp.puts "if test -e #{plan.failFile}"
588         outp.puts "then"
589         outp.puts "    " + plan.successCommand
590         outp.puts "elif grep -i -q failed! #{outputFilename}"
591         outp.puts "then"
592         outp.puts "    " + plan.successCommand
593         outp.puts "else"
594         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
595         outp.puts "    " + plan.failCommand
596         outp.puts "fi"
597     }
598 end
599
600 # Error handler for tests that report error by saying "failed!", and are expected to have
601 # an exit code of 3.
602 def mozillaExit3ErrorHandler
603     Proc.new {
604         | outp, plan |
605         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
606
607         outp.puts "if test -e #{plan.failFile}"
608         outp.puts "then"
609         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
610         outp.puts "    then"
611         outp.puts "        if grep -i -q failed! #{outputFilename}"
612         outp.puts "        then"
613         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
614         outp.puts "            " + plan.failCommand
615         outp.puts "        else"
616         outp.puts "            " + plan.successCommand
617         outp.puts "        fi"
618         outp.puts "    else"
619         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
620         outp.puts "        " + plan.failCommand
621         outp.puts "    fi"
622         outp.puts "else"
623         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
624         outp.puts "    " + plan.failCommand
625         outp.puts "fi"
626     }
627 end
628
629 $runCommandOptions = {}
630
631 class Plan
632     attr_reader :directory, :arguments, :family, :name, :outputHandler, :errorHandler
633     attr_accessor :index
634     
635     def initialize(directory, arguments, family, name, outputHandler, errorHandler)
636         @directory = directory
637         @arguments = arguments
638         @family = family
639         @name = name
640         @outputHandler = outputHandler
641         @errorHandler = errorHandler
642         @isSlow = !!$runCommandOptions[:isSlow]
643     end
644     
645     def shellCommand
646         # It's important to remember that the test is actually run in a subshell, so if we change directory
647         # in the subshell when we return we will be in our original directory. This is nice because we don't
648         # have to bend over backwards to do things relative to the root.
649         script = "(cd ../#{Shellwords.shellescape(@directory.to_s)} && ("
650         $envVars.each { |var| script += "export " << var << "; " }
651         script += "\"$@\" " + escapeAll(@arguments) + "))"
652         return script
653     end
654     
655     def reproScriptCommand
656         # We have to find our way back to the .runner directory since that's where all of the relative
657         # paths assume they start out from.
658         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
659         script += "cd $CURRENT_DIR\n"
660         Pathname.new(@name).dirname.each_filename {
661             | pathComponent |
662             script += "cd ..\n"
663         }
664         script += "cd .runner\n"
665
666         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
667         script += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])}\n"
668         $envVars.each { |var| script += "export " << var << "\n" }
669         script += "#{shellCommand} || exit 1"
670         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
671     end
672     
673     def failCommand
674         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
675     end
676     
677     def successCommand
678         if $progressMeter or $verbosity >= 2
679             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
680         else
681             "rm -f #{failFile}"
682         end
683     end
684     
685     def failFile
686         "test_fail_#{@index}"
687     end
688     
689     def writeRunScript(filename)
690         File.open(filename, "w") {
691             | outp |
692             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
693             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
694             cmd += @outputHandler.call(@name)
695             if $verbosity >= 3
696                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
697             end
698             outp.puts cmd
699             @errorHandler.call(outp, self)
700         }
701     end
702 end
703
704 $uniqueFilenameCounter = 0
705 def uniqueFilename(extension)
706     payloadDir = $outputDir + "_payload"
707     Dir.mkdir payloadDir unless payloadDir.directory?
708     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
709     $uniqueFilenameCounter += 1
710     result
711 end
712
713 def baseOutputName(kind)
714     "#{$collectionName}/#{$benchmark}.#{kind}"
715 end
716
717 def addRunCommand(kind, command, outputHandler, errorHandler)
718     $didAddRunCommand = true
719     name = baseOutputName(kind)
720     if $filter and name !~ $filter
721         return
722     end
723     plan = Plan.new(
724         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
725         errorHandler)
726     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
727         $runlist.unshift plan
728     else
729         $runlist << plan
730     end
731 end
732
733 # Returns true if there were run commands found in the file ($benchmarkDirectory +
734 # $benchmark), in which case those run commands have already been executed. Otherwise
735 # returns false, in which case you're supposed to add your own run commands.
736 def parseRunCommands
737     oldDidAddRunCommand = $didAddRunCommand
738     $didAddRunCommand = false
739
740     Dir.chdir($outputDir) {
741         File.open($benchmarkDirectory + $benchmark) {
742             | inp |
743             inp.each_line {
744                 | line |
745                 begin
746                     doesMatch = line =~ /^\/\/@/
747                 rescue Exception => e
748                     # Apparently this happens in the case of some UTF8 stuff in some files, where
749                     # Ruby tries to be strict and throw exceptions.
750                     next
751                 end
752                 next unless doesMatch
753                 eval $~.post_match
754             }
755         }
756     }
757
758     result = $didAddRunCommand
759     $didAddRunCommand = result or oldDidAddRunCommand
760     result
761 end
762
763 def slow!
764     $runCommandOptions[:isSlow] = true
765 end
766
767 def run(kind, *options)
768     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
769 end
770
771 def runNoFTL
772     run("no-ftl")
773 end
774
775 def runWithRAMSize(size)
776     run("ram-size-#{size}", "--forceRAMSize=#{size}")
777 end
778
779 def runOneLargeHeap
780     if $memoryLimited
781         $didAddRunCommand = true
782         puts "Skipping #{$collectionName}/#{$benchmark}"
783     else
784         run("default")
785     end
786 end
787
788 def runNoJIT
789     run("no-jit", "--useJIT=false")
790 end
791
792 def runNoLLInt
793     if $jitTests
794         run("no-llint", "--useLLInt=false")
795     end
796 end
797
798 def runNoCJITValidate
799     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *NO_CJIT_OPTIONS)
800 end
801
802 def runNoCJITValidatePhases
803     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *NO_CJIT_OPTIONS)
804 end
805
806 def runDefault
807     run("default", *FTL_OPTIONS)
808 end
809
810 def runFTLNoCJIT
811     run("ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
812 end
813
814 def runFTLNoCJITValidate
815     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
816 end
817
818 def runFTLNoCJITNoPutStackValidate
819     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
820 end
821
822 def runFTLNoCJITNoInlineValidate
823     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
824 end
825
826 def runFTLNoCJITOSRValidation
827     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
828 end
829
830 def runDFGEager
831     run("dfg-eager", *EAGER_OPTIONS)
832 end
833
834 def runDFGEagerNoCJITValidate
835     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
836 end
837
838 def runFTLEager
839     run("ftl-eager", *(FTL_OPTIONS + EAGER_OPTIONS))
840 end
841
842 def runFTLEagerNoCJITValidate
843     run("ftl-eager-no-cjit", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
844 end
845
846 def runFTLEagerNoCJITOSRValidation
847     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
848 end
849
850 def runAlwaysTriggerCopyPhase
851     run("always-trigger-copy-phase", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
852 end
853
854 def runNoCJITNoASO
855     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *NO_CJIT_OPTIONS)
856 end
857
858 def runNoCJITNoAccessInlining
859     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *NO_CJIT_OPTIONS)
860 end
861
862 def runFTLNoCJITNoAccessInlining
863     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
864 end
865
866 def runFTLNoCJITSmallPool
867     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
868 end
869
870 def runMiscNoCJITTest(*options)
871     run("misc-no-cjit", *(NO_CJIT_OPTIONS + options))
872 end
873
874 def runMiscFTLNoCJITTest(*options)
875     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + options))
876 end
877
878 def runDFGMaximalFlushPhase
879     run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *NO_CJIT_OPTIONS)
880 end
881
882 def runShadowChicken
883     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true")
884 end
885
886 def defaultRun
887     if $quickMode
888         defaultQuickRun
889     else
890         runDefault
891         runAlwaysTriggerCopyPhase
892         if $jitTests
893             runNoLLInt
894             runNoCJITValidatePhases
895             runDFGEager
896             runDFGEagerNoCJITValidate
897             runDFGMaximalFlushPhase
898
899             return if !$isFTLPlatform
900
901             runNoFTL
902             runFTLNoCJITValidate
903             runFTLNoCJITNoPutStackValidate
904             runFTLNoCJITNoInlineValidate
905             runFTLEager
906             runFTLEagerNoCJITValidate
907             runFTLNoCJITSmallPool
908         end
909     end
910 end
911
912 def defaultNoNoLLIntRun
913     if $quickMode
914         defaultQuickRun
915     else
916         runDefault
917         runAlwaysTriggerCopyPhase
918         if $jitTests
919             runNoCJITValidatePhases
920             runDFGEager
921             runDFGEagerNoCJITValidate
922             runDFGMaximalFlushPhase
923
924             return if !$isFTLPlatform
925
926             runNoFTL
927             runFTLNoCJITValidate
928             runFTLNoCJITNoPutStackValidate
929             runFTLNoCJITNoInlineValidate
930             runFTLEager
931             runFTLEagerNoCJITValidate
932             runFTLNoCJITSmallPool
933         end
934     end
935 end
936
937 def defaultQuickRun
938     runDefault
939     if $jitTests
940         runNoCJITValidate
941
942         return if $isFTLPlatform
943
944         runNoFTL
945         runFTLNoCJITValidate
946     end
947 end
948
949 def defaultSpotCheckNoMaximalFlush
950     defaultQuickRun
951     runNoCJITNoAccessInlining
952
953     return if !$isFTLPlatform
954
955     runFTLNoCJITOSRValidation
956     runFTLNoCJITNoAccessInlining
957 end
958
959 def defaultSpotCheck
960     defaultSpotCheckNoMaximalFlush
961     runDFGMaximalFlushPhase
962 end
963
964 # This is expected to not do eager runs because eager runs can have a lot of recompilations
965 # for reasons that don't arise in the real world. It's used for tests that assert convergence
966 # by counting recompilations.
967 def defaultNoEagerRun
968     runDefault
969     runAlwaysTriggerCopyPhase
970     if $jitTests
971         runNoLLInt
972         runNoCJITValidatePhases
973
974         return if !$isFTLPlatform
975
976         runNoFTL
977         runFTLNoCJITValidate
978         runFTLNoCJITNoInlineValidate
979     end
980 end
981
982 def defaultNoSamplingProfilerRun
983     runDefault
984     runAlwaysTriggerCopyPhase
985     if $jitTests
986         runNoLLInt
987         runNoCJITValidatePhases
988         runDFGEager
989         runDFGEagerNoCJITValidate
990         runDFGMaximalFlushPhase
991
992         return if !$isFTLPlatform
993
994         runNoFTL
995         runFTLNoCJITNoPutStackValidate
996         runFTLNoCJITNoInlineValidate
997         runFTLEager
998         runFTLEagerNoCJITValidate
999         runFTLNoCJITSmallPool
1000     end
1001 end
1002
1003 def runProfiler
1004     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
1005         skip
1006         return
1007     end
1008
1009     profilerOutput = uniqueFilename(".json")
1010     if $canRunDisplayProfilerOutput
1011         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
1012     else
1013         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
1014         run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
1015     end
1016 end
1017
1018 def runExceptionFuzz
1019     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
1020     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
1021 end
1022
1023 def runExecutableAllocationFuzz(name, *options)
1024     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
1025     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
1026 end
1027
1028 def runTypeProfiler
1029     if !$jitTests
1030         return
1031     end
1032
1033     return if !$isFTLPlatform
1034
1035     run("ftl-no-cjit-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1036     run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
1037 end
1038
1039 def runControlFlowProfiler
1040     if !$jitTests
1041         return
1042     end
1043
1044     return if !$isFTLPlatform
1045
1046     run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1047 end
1048
1049 def runTest262(mode, exception, includeFiles, flags)
1050     failsWithException = exception != "NoException"
1051     isStrict = false
1052     isModule = false
1053     isAsync = false
1054
1055     flags.each {
1056         | flag |
1057         case flag
1058         when :strict
1059             isStrict = true
1060         when :module
1061             isModule = true
1062         when :async
1063             isAsync = true
1064         else
1065             raise "Invalid flag for runTest262, #{flag}"
1066         end
1067     }
1068
1069     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
1070
1071     args = [pathToVM.to_s] + BASE_OPTIONS
1072     args << "--exception=" + exception if failsWithException
1073     args << "--test262-async" if isAsync
1074     args += includeFiles
1075
1076     case mode
1077     when :normal
1078         errorHandler = simpleErrorHandler
1079         outputHandler = silentOutputHandler
1080     when :fail
1081         errorHandler = expectedFailErrorHandler
1082         outputHandler = noisyOutputHandler
1083     else
1084         raise "Invalid mode: #{mode}"
1085     end
1086
1087     if isStrict
1088         kind = "default-strict"
1089         args << "--strict-file=#{$benchmark}"
1090     else
1091         kind = "default"
1092         if isModule
1093             args << "--module-file=#{$benchmark}"
1094         else
1095             args << $benchmark.to_s
1096         end
1097     end
1098
1099     addRunCommand(kind, args, outputHandler, errorHandler)
1100 end
1101
1102 def prepareTest262Fixture
1103     # This function is used to add the files used by Test262 modules tests.
1104     prepareExtraRelativeFiles([""], $collection)
1105 end
1106
1107 def runES6(mode)
1108     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
1109     case mode
1110     when :normal
1111         errorHandler = simpleErrorHandler
1112     when :fail
1113         errorHandler = expectedFailErrorHandler
1114     else
1115         raise "Invalid mode: #{mode}"
1116     end
1117     addRunCommand("default", args, noisyOutputHandler, errorHandler)
1118 end
1119
1120 def runModules
1121     run("default-modules", "-m")
1122     run("always-trigger-copy-phase-modules", "-m", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
1123
1124     if !$jitTests
1125         return
1126     end
1127
1128     run("no-llint-modules", "-m", "--useLLInt=false")
1129     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
1130     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
1131     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1132
1133     return if !$isFTLPlatform
1134
1135     run("default-ftl-modules", "-m", *FTL_OPTIONS)
1136     run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1137     run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1138     run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
1139     run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1140     run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1141 end
1142
1143 def runLayoutTest(kind, *options)
1144     raise unless $benchmark.to_s =~ /\.js$/
1145     testName = $~.pre_match
1146     if kind
1147         kind = "layout-" + kind
1148     else
1149         kind = "layout"
1150     end
1151
1152     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1153     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1154
1155     args = [pathToVM.to_s] + BASE_OPTIONS + options +
1156         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1157          $benchmark.to_s,
1158          (Pathname.new("resources") + "standalone-post.js").to_s]
1159     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1160 end
1161
1162 def runLayoutTestNoFTL
1163     runLayoutTest("no-ftl")
1164 end
1165
1166 def runLayoutTestNoLLInt
1167     runLayoutTest("no-llint", "--useLLInt=false")
1168 end
1169
1170 def runLayoutTestNoCJIT
1171     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1172 end
1173
1174 def runLayoutTestDFGEagerNoCJIT
1175     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1176 end
1177
1178 def runLayoutTestDefault
1179     runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
1180 end
1181
1182 def runLayoutTestFTLNoCJIT
1183     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1184 end
1185
1186 def runLayoutTestFTLEagerNoCJIT
1187     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1188 end
1189
1190 def noFTLRunLayoutTest
1191     if !$jitTests
1192         return
1193     end
1194
1195     runLayoutTestNoLLInt
1196     runLayoutTestNoCJIT
1197     runLayoutTestDFGEagerNoCJIT
1198 end
1199
1200 def defaultQuickRunLayoutTest
1201     runLayoutTestDefault
1202     if $jitTests
1203         if $isFTLPlatform
1204             runLayoutTestNoFTL
1205             runLayoutTestFTLNoCJIT
1206             runLayoutTestFTLEagerNoCJIT
1207         else
1208             noFTLRunLayoutTest
1209         end
1210     end
1211 end
1212
1213 def defaultRunLayoutTest
1214     if $quickMode
1215         defaultQuickRunLayoutTest
1216     else
1217         runLayoutTestDefault
1218         if $jitTests
1219             noFTLRunLayoutTest
1220
1221             return if !$isFTLPlatform
1222
1223             runLayoutTestNoFTL
1224             runLayoutTestFTLNoCJIT
1225             runLayoutTestFTLEagerNoCJIT
1226         end
1227     end
1228 end
1229
1230 def noEagerNoNoLLIntTestsRunLayoutTest
1231     runLayoutTestDefault
1232     if $jitTests
1233         runLayoutTestNoCJIT
1234
1235         return if !$isFTLPlatform
1236
1237         runLayoutTestNoFTL
1238         runLayoutTestFTLNoCJIT
1239     end
1240 end
1241
1242 def noNoLLIntRunLayoutTest
1243     runLayoutTestDefault
1244     if $jitTests
1245         runLayoutTestNoCJIT
1246         runLayoutTestDFGEagerNoCJIT
1247
1248         return if !$isFTLPlatform
1249
1250         runLayoutTestNoFTL
1251         runLayoutTestFTLNoCJIT
1252         runLayoutTestFTLEagerNoCJIT
1253     end
1254 end
1255
1256 def prepareExtraRelativeFiles(extraFiles, destination)
1257     Dir.chdir($outputDir) {
1258         extraFiles.each {
1259             | file |
1260             dest = destination + file
1261             FileUtils.mkdir_p(dest.dirname)
1262             FileUtils.cp $extraFilesBaseDir + file, dest
1263         }
1264     }
1265 end
1266
1267 def baseDirForCollection(collectionName)
1268     Pathname(".tests") + collectionName
1269 end
1270
1271 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1272     raise unless absoluteBase.absolute?
1273     Dir.chdir($outputDir) {
1274         collectionBaseDir = baseDirForCollection($collectionName)
1275         extraFiles.each {
1276             | file |
1277             destination = collectionBaseDir + file
1278             FileUtils.mkdir_p destination.dirname unless destination.directory?
1279             FileUtils.cp absoluteBase + file, destination
1280         }
1281     }
1282 end
1283
1284 def runMozillaTest(kind, mode, extraFiles, *options)
1285     if kind
1286         kind = "mozilla-" + kind
1287     else
1288         kind = "mozilla"
1289     end
1290     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1291     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1292     case mode
1293     when :normal
1294         errorHandler = mozillaErrorHandler
1295     when :negative
1296         errorHandler = mozillaExit3ErrorHandler
1297     when :fail
1298         errorHandler = mozillaFailErrorHandler
1299     when :skip
1300         return
1301     else
1302         raise "Invalid mode: #{mode}"
1303     end
1304     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1305 end
1306
1307 def runMozillaTestDefault(mode, *extraFiles)
1308     runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
1309 end
1310
1311 def runMozillaTestNoFTL(mode, *extraFiles)
1312     runMozillaTest("no-ftl", mode, extraFiles)
1313 end
1314
1315 def runMozillaTestLLInt(mode, *extraFiles)
1316     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1317 end
1318
1319 def runMozillaTestBaselineJIT(mode, *extraFiles)
1320     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1321 end
1322
1323 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1324     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1325 end
1326
1327 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1328     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1329 end
1330
1331 def defaultQuickRunMozillaTest(mode, *extraFiles)
1332     if $jitTests
1333         runMozillaTestDefault(mode, *extraFiles)
1334         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1335     else
1336         runMozillaTestNoFTL(mode, *extraFiles)
1337         if $jitTests
1338             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1339         end
1340     end
1341 end
1342
1343 def defaultRunMozillaTest(mode, *extraFiles)
1344     if $quickMode
1345         defaultQuickRunMozillaTest(mode, *extraFiles)
1346     else
1347         runMozillaTestNoFTL(mode, *extraFiles)
1348         if $jitTests
1349             runMozillaTestLLInt(mode, *extraFiles)
1350             runMozillaTestBaselineJIT(mode, *extraFiles)
1351             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1352             runMozillaTestDefault(mode, *extraFiles)
1353             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
1354         end
1355     end
1356 end
1357
1358 def runNoisyTest(kind, *options)
1359     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
1360 end
1361
1362 def runNoisyTestDefault
1363     runNoisyTest("default", *FTL_OPTIONS)
1364 end
1365
1366 def runNoisyTestNoFTL
1367     runNoisyTest("no-ftl")
1368 end
1369
1370 def runNoisyTestNoCJIT
1371     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1372 end
1373
1374 def runNoisyTestEagerNoCJIT
1375     runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1376 end
1377
1378 def defaultRunNoisyTest
1379     runNoisyTestDefault
1380     if $jitTests and $isFTLPlatform
1381         runNoisyTestNoFTL
1382         runNoisyTestNoCJIT
1383         runNoisyTestEagerNoCJIT
1384     end
1385 end
1386
1387 def skip
1388     $didAddRunCommand = true
1389     puts "Skipping #{$collectionName}/#{$benchmark}"
1390 end
1391
1392 def largeHeap
1393     if $memoryLimited
1394         $didAddRunCommand = true
1395         puts "Skipping #{$collectionName}/#{$benchmark}"
1396     end
1397 end
1398
1399 def allJSFiles(path)
1400     if path.file?
1401         [path]
1402     else
1403         result = []
1404         Dir.foreach(path) {
1405             | filename |
1406             next unless filename =~ /\.js$/
1407             next unless (path + filename).file?
1408             result << path + filename
1409         }
1410         result
1411     end
1412 end
1413
1414 def uniqueifyName(names, name)
1415     result = name.to_s
1416     toAdd = 1
1417     while names[result]
1418         result = "#{name}-#{toAdd}"
1419         toAdd += 1
1420     end
1421     names[result] = true
1422     result
1423 end
1424
1425 def simplifyCollectionName(collectionPath)
1426     outerDir = collectionPath.dirname
1427     name = collectionPath.basename
1428     lastName = name
1429     if collectionPath.directory?
1430         while lastName.to_s =~ /test/
1431             lastName = outerDir.basename
1432             name = lastName + name
1433             outerDir = outerDir.dirname
1434         end
1435     end
1436     uniqueifyName($collectionNames, name)
1437 end
1438
1439 def prepareCollection(name)
1440     FileUtils.mkdir_p $outputDir + name
1441
1442     absoluteCollection = $collection.realpath
1443
1444     Dir.chdir($outputDir) {
1445         bundleDir = baseDirForCollection(name)
1446
1447         # Create the proper directory structures.
1448         FileUtils.mkdir_p bundleDir
1449         if bundleDir.basename == $collection.basename
1450             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1451             $collection = bundleDir
1452         else
1453             FileUtils.cp_r absoluteCollection, bundleDir
1454             $collection = bundleDir + $collection.basename
1455         end
1456
1457         $extraFilesBaseDir = absoluteCollection
1458     }
1459 end
1460
1461 $collectionNames = {}
1462
1463 def handleCollectionFile(collection)
1464     collectionName = simplifyCollectionName(collection)
1465    
1466     paths = {}
1467     subCollections = []
1468     YAML::load(IO::read(collection)).each {
1469         | entry |
1470         if entry["collection"]
1471             subCollections << entry["collection"]
1472             next
1473         end
1474         
1475         if Pathname.new(entry["path"]).absolute?
1476             raise "Absolute path: " + entry["path"] + " in #{collection}"
1477         end
1478         
1479         if paths[entry["path"]]
1480             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1481         end
1482         
1483         subCollection = collection.dirname + entry["path"]
1484         
1485         if subCollection.file?
1486             subCollectionName = Pathname.new(entry["path"]).dirname
1487         else
1488             subCollectionName = entry["path"]
1489         end
1490         
1491         $collection = subCollection
1492         $collectionName = Pathname.new(collectionName)
1493         Pathname.new(subCollectionName).each_filename {
1494             | filename |
1495             next if filename =~ /^\./
1496             $collectionName += filename
1497         }
1498         $collectionName = $collectionName.to_s
1499         
1500         prepareCollection($collectionName)
1501       
1502         Dir.chdir($outputDir) {
1503             pathsToSearch = [$collection]
1504             if entry["tests"]
1505                 if entry["tests"].is_a? Array
1506                     pathsToSearch = entry["tests"].map {
1507                         | testName |
1508                         pathsToSearch[0] + testName
1509                     }
1510                 else
1511                     pathsToSearch[0] += entry["tests"]
1512                 end
1513             end
1514             pathsToSearch.each {
1515                 | pathToSearch |
1516                 allJSFiles(pathToSearch).each {
1517                     | path |
1518                     
1519                     $benchmark = path.basename
1520                     $benchmarkDirectory = path.dirname
1521                     
1522                     $runCommandOptions = {}
1523                     eval entry["cmd"]
1524                 }
1525             }
1526         }
1527     }
1528     
1529     subCollections.each {
1530         | subCollection |
1531         handleCollection(collection.dirname + subCollection)
1532     }
1533 end
1534
1535 def handleCollectionDirectory(collection)
1536     collectionName = simplifyCollectionName(collection)
1537     
1538     $collection = collection
1539     $collectionName = collectionName
1540     prepareCollection(collectionName)
1541    
1542     Dir.chdir($outputDir) {
1543         $benchmarkDirectory = $collection
1544         allJSFiles($collection).each {
1545             | path |
1546             
1547             $benchmark = path.basename
1548             
1549             $runCommandOptions = {}
1550             defaultRun unless parseRunCommands
1551         }
1552     }
1553 end
1554
1555 def handleCollection(collection)
1556     collection = Pathname.new(collection)
1557     
1558     if collection.file?
1559         handleCollectionFile(collection)
1560     else
1561         handleCollectionDirectory(collection)
1562     end
1563 end
1564
1565 def appendFailure(plan)
1566     File.open($outputDir + "failed", "a") {
1567         | outp |
1568         outp.puts plan.name
1569     }
1570     $numFailures += 1
1571 end
1572
1573 def appendPass(plan)
1574     File.open($outputDir + "passed", "a") {
1575         | outp |
1576         outp.puts plan.name
1577     }
1578     $numPasses += 1
1579 end
1580
1581 def appendResult(plan, didPass)
1582     File.open($outputDir + "results", "a") {
1583         | outp |
1584         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1585     }
1586 end
1587
1588 def prepareBundle
1589     raise if $bundle
1590
1591     if $doNotMessWithVMPath
1592         if !$remote and !$tarball
1593             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1594             $jscPath = Pathname.new($jscPath).realpath
1595         else
1596             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1597         end
1598     else
1599         originalJSCPath = $jscPath
1600         vmDir = $outputDir + ".vm"
1601         FileUtils.mkdir_p vmDir
1602         
1603         frameworkPath = frameworkFromJSCPath($jscPath)
1604         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1605         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1606         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1607
1608         if frameworkPath
1609             source = frameworkPath
1610             destination = Pathname.new(".vm")
1611         else
1612             source = originalJSCPath
1613             destination = $jscPath
1614
1615             Dir.chdir($outputDir) {
1616                 FileUtils.mkdir_p $jscPath.dirname
1617             }
1618         end
1619
1620         Dir.chdir($outputDir) {
1621             if $copyVM
1622                 FileUtils.cp_r source, destination
1623             else
1624                 begin 
1625                     FileUtils.ln_s source, destination
1626                 rescue Exception
1627                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1628                     FileUtils.cp_r source, destination
1629                 end
1630             end
1631
1632             if $remote and $hostOS == "linux"
1633                 begin
1634                     dependencies = `ldd #{source}`
1635                     dependencies.split(/\n/).each {
1636                         | dependency |
1637                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1638                     }
1639                 rescue
1640                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1641                 end
1642             end
1643         }
1644     end
1645     
1646     Dir.chdir($outputDir) {
1647         FileUtils.cp_r HELPERS_PATH, ".helpers"
1648     }
1649
1650     ARGV.each {
1651         | collection |
1652         handleCollection(collection)
1653     }
1654
1655     puts
1656 end
1657
1658 def cleanOldResults
1659     raise unless $bundle
1660
1661     eachResultFile($outputDir) {
1662         | path |
1663         FileUtils.rm_f path
1664     }
1665 end
1666
1667 def cleanEmptyResultFiles
1668     eachResultFile($outputDir) {
1669         | path |
1670         next unless path.basename.to_s =~ /\.out$/
1671         next unless FileTest.size(path) == 0
1672         FileUtils.rm_f path
1673     }
1674 end
1675
1676 def eachResultFile(startingDir, &block)
1677     dirsToClean = [startingDir]
1678     until dirsToClean.empty?
1679         nextDir = dirsToClean.pop
1680         Dir.foreach(nextDir) {
1681             | entry |
1682             next if entry =~ /^\./
1683             path = nextDir + entry
1684             if path.directory?
1685                 dirsToClean.push(path)
1686             else
1687                 block.call(path)
1688             end
1689         }
1690     end
1691 end
1692
1693 def prepareTestRunner
1694     raise if $bundle
1695
1696     $runlist.each_with_index {
1697         | plan, index |
1698         plan.index = index
1699     }
1700
1701     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1702     toDelete = []
1703     Dir.foreach($runnerDir) {
1704         | filename |
1705         if filename =~ /^test_/
1706             toDelete << filename
1707         end
1708     }
1709     
1710     toDelete.each {
1711         | filename |
1712         File.unlink($runnerDir + filename)
1713     }
1714
1715     $runlist.each {
1716         | plan |
1717         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1718     }
1719
1720     case $testRunnerType
1721     when :make
1722         prepareMakeTestRunner
1723     when :shell
1724         prepareShellTestRunner
1725     else
1726         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1727     end
1728 end
1729
1730 def prepareShellTestRunner
1731     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
1732 end
1733
1734 def prepareMakeTestRunner
1735     # The goals of our parallel test runner are scalability and simplicity. The
1736     # simplicity part is particularly important. We don't want to have to have
1737     # a full-time contributor just philosophising about parallel testing.
1738     #
1739     # As such, we just pass off all of the hard work to 'make'. This creates a
1740     # dummy directory ("$outputDir/.runner") in which we create a dummy
1741     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
1742     # That is, for each test we know we will run, there is a rule in the
1743     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
1744     # Makefile results in 'make' doing all of the hard work:
1745     #
1746     # - Load balancing just works. Most systems have a great load balancer in
1747     #   'make'. If your system doesn't then just install a real 'make'.
1748     #
1749     # - Interruptions just work. For example Ctrl-C handling in 'make' is
1750     #   exactly right. You don't have to worry about zombie processes.
1751     #
1752     # We then do some tricks to make failure detection work and to make this
1753     # totally sound. If a test fails, we don't want the whole 'make' job to
1754     # stop. We also don't have any facility for makefile-escaping of path names.
1755     # We do have such a thing for shell-escaping, though. We fix both problems
1756     # by having the actual work for each of the test rules be done in a shell
1757     # script on the side. There is one such script per test. The script responds
1758     # to failure by printing something on the console and then touching a
1759     # failure file for that test, but then still returns 0. This makes 'make'
1760     # continue past that failure and complete all the tests anyway.
1761     #
1762     # In the end, this script collects all of the failures by searching for
1763     # files in the .runner directory whose name matches /^test_fail_/, where
1764     # the thing after the 'fail_' is the test index. Those are the files that
1765     # would be created by the test scripts if they detect failure. We're
1766     # basically using the filesystem as a concurrent database of test failures.
1767     # Even if two tests fail at the same time, since they're touching different
1768     # files we won't miss any failures.
1769     runIndices = []
1770     $runlist.each {
1771         | plan |
1772         runIndices << plan.index
1773     }
1774     
1775     File.open($runnerDir + "Makefile", "w") {
1776         | outp |
1777         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1778         runIndices.each {
1779             | index |
1780             plan = $runlist[index]
1781             outp.puts "test_done_#{index}:"
1782             outp.puts "\tsh test_script_#{plan.index}"
1783         }
1784     }
1785 end
1786
1787 def cleanRunnerDirectory
1788     raise unless $bundle
1789     Dir.foreach($runnerDir) {
1790         | filename |
1791         next unless filename =~ /^test_fail/
1792         FileUtils.rm_f $runnerDir + filename
1793     }
1794 end
1795
1796 def sshRead(cmd)
1797     raise unless $remote
1798
1799     result = ""
1800     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1801       | inp |
1802       inp.each_line {
1803         | line |
1804         result += line
1805       }
1806     }
1807     raise "#{$?}" unless $?.success?
1808     result
1809 end
1810
1811 def runCommandOnTester(cmd)
1812     if $remote
1813         result = sshRead(cmd)
1814     else
1815         result = `#{cmd}`
1816     end
1817 end
1818
1819 def numberOfProcessors
1820     if $hostOS == "windows"
1821         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1822     else
1823         begin
1824             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1825         rescue
1826             numProcessors = 0
1827         end
1828
1829         if numProcessors == 0
1830             begin
1831                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1832             rescue
1833                 numProcessors == 0
1834             end
1835         end
1836     end
1837
1838     if numProcessors == 0
1839         numProcessors = 1
1840     end
1841     return numProcessors
1842 end
1843
1844 def runAndMonitorTestRunnerCommand(*cmd)
1845     numberOfTests = 0
1846     Dir.chdir($runnerDir) {
1847         # -1 for the runscript, and -2 for '..' and '.'
1848         numberOfTests = Dir.entries(".").count - 3
1849     }
1850     unless $progressMeter
1851         mysys(cmd.join(' '))
1852     else
1853        running = {}
1854        didRun = {}
1855        didFail = {}
1856        blankLine = true
1857        prevStringLength = 0
1858        IO.popen(cmd.join(' '), mode="r") {
1859            | inp |
1860            inp.each_line {
1861                | line |
1862                line = line.scrub.chomp
1863                if line =~ /^Running /
1864                    running[$~.post_match] = true
1865                elsif line =~ /^PASS: /
1866                    didRun[$~.post_match] = true
1867                elsif line =~ /^FAIL: /
1868                    didRun[$~.post_match] = true
1869                    didFail[$~.post_match] = true
1870                else
1871                    unless blankLine
1872                        print("\r" + " " * prevStringLength + "\r")
1873                    end
1874                    puts line
1875                    blankLine = true
1876                end
1877
1878                def lpad(str, chars)
1879                    str = str.to_s
1880                    if str.length > chars
1881                        str
1882                    else
1883                       "%#{chars}s"%(str)
1884                    end
1885                end
1886
1887                string  = ""
1888                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1889                unless didFail.empty?
1890                    string += " (failed #{didFail.size})"
1891                end
1892                string += " "
1893                (running.size - didRun.size).times {
1894                    string += "."
1895                }
1896                if string.length < prevStringLength
1897                    print string
1898                    print(" " * (prevStringLength - string.length))
1899                end
1900                print string
1901                prevStringLength = string.length
1902                blankLine = false
1903                $stdout.flush
1904            }
1905        }
1906        puts
1907        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1908     end
1909 end
1910
1911 def runTestRunner
1912     case $testRunnerType
1913     when :shell
1914         testRunnerCommand = "sh runscript"
1915     when :make
1916         testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
1917     else
1918         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1919     end
1920
1921     if $remote
1922         if !$remoteDirectory
1923             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1924         end
1925         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1926         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1927         remoteScript = "\""
1928         remoteScript += "cd #{$remoteDirectory} && "
1929         remoteScript += "rm -rf #{$outputDir.basename} && "
1930         remoteScript += "tar xzf #{$tarFileName} && "
1931         remoteScript += "cd #{$outputDir.basename}/.runner && "
1932         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1933         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1934         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
1935         $envVars.each { |var| remoteScript += "export " << var << "\n" }
1936         remoteScript += "#{testRunnerCommand}\""
1937         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1938     else
1939         Dir.chdir($runnerDir) {
1940             runAndMonitorTestRunnerCommand(testRunnerCommand)
1941         }
1942     end
1943 end
1944
1945 def detectFailures
1946     raise if $bundle
1947
1948     failures = []
1949     
1950     if $remote
1951         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1952         output.split(/\n/).each {
1953             | line |
1954             next unless line =~ /test_fail_/
1955             failures << $~.post_match.to_i
1956         }
1957     else
1958         Dir.foreach($runnerDir) {
1959             | filename |
1960             next unless filename =~ /test_fail_/
1961             failures << $~.post_match.to_i
1962         }
1963     end
1964
1965     failureSet = {}
1966
1967     failures.each {
1968         | failure | 
1969         appendFailure($runlist[failure])
1970         failureSet[failure] = true
1971     }
1972
1973     familyMap = {}
1974     $runlist.each_with_index {
1975         | plan, index |
1976         unless familyMap[plan.family]
1977             familyMap[plan.family] = []
1978         end
1979         if failureSet[index]
1980             appendResult(plan, false)
1981             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
1982             next
1983         else
1984             appendResult(plan, true)
1985             familyMap[plan.family] << {:result => "PASS", :plan => plan};
1986         end
1987         appendPass(plan)
1988     }
1989
1990     File.open($outputDir + "resultsByFamily", "w") {
1991         | outp |
1992         first = true
1993         familyMap.keys.sort.each {
1994             | familyName |
1995             if first
1996                 first = false
1997             else
1998                 outp.puts
1999             end
2000             
2001             outp.print "#{familyName}:"
2002
2003             numPassed = 0
2004             familyMap[familyName].each {
2005                 | entry |
2006                 if entry[:result] == "PASS"
2007                     numPassed += 1
2008                 end
2009             }
2010
2011             if numPassed == familyMap[familyName].size
2012                 outp.puts " PASSED"
2013             elsif numPassed == 0
2014                 outp.puts " FAILED"
2015             else
2016                 outp.puts
2017                 familyMap[familyName].each {
2018                     | entry |
2019                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
2020                 }
2021             end
2022         }
2023     }
2024 end
2025
2026 def compressBundle
2027     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
2028     $stderr.puts ">> #{cmd}" if $verbosity >= 2
2029     raise unless system(cmd)
2030 end
2031
2032 def clean(file)
2033     FileUtils.rm_rf file unless $bundle
2034 end
2035
2036 clean($outputDir + "failed")
2037 clean($outputDir + "passed")
2038 clean($outputDir + "results")
2039 clean($outputDir + "resultsByFamily")
2040 clean($outputDir + ".vm")
2041 clean($outputDir + ".helpers")
2042 clean($outputDir + ".runner")
2043 clean($outputDir + ".tests")
2044 clean($outputDir + "_payload")
2045
2046 Dir.mkdir($outputDir) unless $outputDir.directory?
2047
2048 $outputDir = $outputDir.realpath
2049 $runnerDir = $outputDir + ".runner"
2050
2051 if !$numChildProcesses
2052     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
2053         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
2054     else
2055         $numChildProcesses = numberOfProcessors
2056     end
2057 end
2058
2059 if ENV["JSCTEST_timeout"]
2060     # In the worst case, the processors just interfere with each other.
2061     # Increase the timeout proportionally to the number of processors.
2062     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
2063 end
2064
2065 def runBundle
2066     raise unless $bundle
2067
2068     cleanRunnerDirectory
2069     cleanOldResults
2070     runTestRunner
2071     cleanEmptyResultFiles
2072 end
2073
2074 def runNormal
2075     raise if $bundle or $tarball
2076
2077     prepareBundle
2078     prepareTestRunner
2079     runTestRunner
2080     cleanEmptyResultFiles
2081     detectFailures
2082 end
2083
2084 def runTarball
2085     raise unless $tarball
2086
2087     prepareBundle 
2088     prepareTestRunner
2089     compressBundle
2090 end
2091
2092 def runRemote
2093     raise unless $remote
2094
2095     prepareBundle
2096     prepareTestRunner
2097     compressBundle
2098     runTestRunner
2099     detectFailures
2100 end
2101
2102 puts
2103 if $bundle
2104     runBundle
2105 elsif $remote
2106     runRemote
2107 elsif $tarball
2108     runTarball
2109 else
2110     runNormal
2111 end