949e3f5b552011b51ad22df3ea0b5abe36b30ad9
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2016 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 RemoteHost = Struct.new(:name, :user, :host, :port, :remoteDirectory)
47
48 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
49 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
50 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
51 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
52 WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
53 CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
54 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
55 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
56
57 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
58
59 begin
60     require 'shellwords'
61 rescue Exception => e
62     $stderr.puts "Warning: did not find shellwords, not running any tests."
63     exit 0
64 end
65
66 $canRunDisplayProfilerOutput = false
67
68 begin
69     require 'rubygems'
70     require 'json'
71     require 'highline'
72     $canRunDisplayProfilerOutput = true
73 rescue Exception => e
74     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
75     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
76     $stderr.puts "Error: #{e.inspect}"
77 end
78
79 def printCommandArray(*cmd)
80     begin
81         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
82     rescue
83         commandArray = cmd.join(' ')
84     end
85     $stderr.puts ">> #{commandArray}"
86 end
87
88 def mysys(*cmd)
89     printCommandArray(*cmd) if $verbosity >= 1
90     raise "Command failed: #{$?.inspect}" unless system(*cmd)
91 end
92
93 def escapeAll(array)
94     array.map {
95         | v |
96         raise "Detected a non-string in #{inspect}" unless v.is_a? String
97         Shellwords.shellescape(v)
98     }.join(' ')
99 end
100
101
102 $jscPath = nil
103 $doNotMessWithVMPath = false
104 $jitTests = true
105 $memoryLimited = false
106 $outputDir = Pathname.new("results")
107 $verbosity = 0
108 $bundle = nil
109 $tarball = false
110 $tarFileName = "payload.tar.gz"
111 $copyVM = false
112 $testRunnerType = nil
113 $testWriter = "default"
114 $remoteHosts = []
115 $architecture = nil
116 $hostOS = nil
117 $filter = nil
118 $envVars = []
119 $mode = "full"
120 $buildType = "release"
121 $forceCollectContinuously = false
122
123 def usage
124     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
125     puts
126     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
127     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
128     puts "                            --jsc specifies an already present JavaScriptCore to test."
129     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
130     puts "                            Skip tests tagged with //@skip if $memoryLimited"
131     puts "--no-jit                    Do not run JIT specific tests."
132     puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
133     puts "                            platform."
134     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
135     puts "--verbose            (-v)   Print more things while running."
136     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
137     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
138     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
139     puts "                            e.g. x86, x86_64, arm."
140     puts "--os                        Specify os instead of determining from JavaScriptCore build."
141     puts "                            e.g. darwin, linux & windows."
142     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
143     puts "                            In general the shell runner is slower than the make runner."
144     puts "--make-runner               Uses the faster make-based runner."
145     puts "--ruby-runner               Uses the ruby runner for machines without unix shell or make."
146     puts "--test-writer [writer]      Specifies the test script format."
147     puts "                            default is to use shell scripts to run the tests"
148     puts "                            \"ruby\" to use ruby scripts for systems without a unix shell."
149     puts "--remote                    Specify a remote host on which to run tests from command line argument."
150     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
151     puts "--child-processes    (-c)   Specify the number of child processes."
152     puts "--filter                    Only run tests whose name matches the given regular expression."
153     puts "--help               (-h)   Print this message."
154     puts "--env-vars                  Add a list of environment variables to set before running jsc."
155     puts "                            Each environment variable should be separated by a space."
156     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
157     puts "                            it will override the default value."
158     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
159     puts "--basic                     Run with default and these additional modes: no-llint,"
160     puts "                            no-cjit-validate-phases, no-cjit-collect-continuously, dfg-eager"
161     puts "                            and for FTL platforms: no-ftl, ftl-eager-no-cjit and"
162     puts "                            ftl-no-cjit-small-pool."
163     exit 1
164 end
165
166 jscArg = nil
167
168 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
169                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
170                ['--no-copy', GetoptLong::NO_ARGUMENT],
171                ['--memory-limited', GetoptLong::NO_ARGUMENT],
172                ['--no-jit', GetoptLong::NO_ARGUMENT],
173                ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
174                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
175                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
176                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
177                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
178                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
179                ['--os', GetoptLong::REQUIRED_ARGUMENT],
180                ['--shell-runner', GetoptLong::NO_ARGUMENT],
181                ['--make-runner', GetoptLong::NO_ARGUMENT],
182                ['--ruby-runner', GetoptLong::NO_ARGUMENT],
183                ['--test-writer', GetoptLong::REQUIRED_ARGUMENT],
184                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
185                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
186                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
187                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
188                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
189                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
190                ['--debug', GetoptLong::NO_ARGUMENT],
191                ['--release', GetoptLong::NO_ARGUMENT],
192                ['--quick', '-q', GetoptLong::NO_ARGUMENT],
193                ['--basic', GetoptLong::NO_ARGUMENT]).each {
194     | opt, arg |
195     case opt
196     when '--help'
197         usage
198     when '--jsc'
199         jscArg = arg
200     when '--no-copy'
201         $doNotMessWithVMPath = true
202     when '--output-dir'
203         $outputDir = Pathname.new(arg)
204     when '--memory-limited'
205         $memoryLimited = true
206     when '--no-jit'
207         $jitTests = false
208     when '--force-collectContinuously'
209         $forceCollectContinuously = true;
210     when '--verbose'
211         $verbosity += 1
212     when '--run-bundle'
213         $bundle = Pathname.new(arg)
214     when '--tarball'
215         $tarball = true
216         $copyVM = true
217         $tarFileName = arg unless arg == ''
218     when '--force-vm-copy'
219         $copyVM = true
220     when '--shell-runner'
221         $testRunnerType = :shell
222     when '--make-runner'
223         $testRunnerType = :make
224     when '--ruby-runner'
225         $testRunnerType = :ruby
226     when '--test-writer'
227         $testWriter = arg
228     when '--remote'
229         $copyVM = true
230         $tarball = true
231         $remote = true
232         uri = URI("ssh://" + arg)
233         $remoteHosts << RemoteHost.new("default-#{$remoteHosts.length}", uri.user, uri.host, uri.port)
234     when '--remote-config-file'
235         $remoteConfigFile = arg
236     when '--child-processes'
237         $numChildProcesses = arg.to_i
238     when '--filter'
239         $filter = Regexp.new(arg)
240     when '--arch'
241         $architecture = arg
242     when '--os'
243         $hostOS = arg
244     when '--env-vars'
245         $envVars = arg.gsub(/\s+/, ' ').split(' ')
246     when '--quick'
247         $mode = "quick"
248     when '--basic'
249         $mode = "basic"
250     when '--debug'
251         $buildType = "debug"
252     when '--release'
253         $buildType = "release"
254     end
255 }
256
257 if $remoteConfigFile
258     file = File.read($remoteConfigFile)
259     config = JSON.parse(file)
260
261     # old style config allowing for only one remote
262     if !$remote and config['remote']
263         $copyVM = true
264         $tarball = true
265         $remote = true
266         uri = URI("ssh://" + config['remote'])
267         $remoteHosts = [ RemoteHost.new("default", uri.user, uri.host, uri.port) ]
268         if config['remoteDirectory']
269             $remoteHosts[0].remoteDirectory = config['remoteDirectory']
270         end
271     end
272
273     # we can combine --remote and a new style config
274     if config['remotes']
275         $copyVM = true
276         $tarball = true
277         $remote = true
278         $remoteHosts += config['remotes'].map {
279             | remote |
280             uri = URI("ssh://" + remote['address'])
281
282             host = RemoteHost.new(remote['name'], uri.user, uri.host, uri.port)
283             if remote['remoteDirectory']
284                 host.remoteDirectory = remote['remoteDirectory']
285             end
286             host
287         }
288     end
289 end
290
291 unless jscArg
292     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
293     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
294     command += ($buildType == "release") ? " --release" : " --debug"
295     command += " --executablePath"
296
297     output = `#{command}`.split("\n")
298     if !output.length
299         $stderr.puts "Error: must specify --jsc <path>"
300         exit 1
301     end
302
303     output = output[0]
304     jscArg = Pathname.new(output).join("jsc")
305     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
306     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
307     if !File.file?(jscArg)
308         $stderr.puts "Error: must specify --jsc <path>"
309         exit 1
310     end
311
312     puts "Using the following jsc path: #{jscArg}"
313 end
314
315 if $doNotMessWithVMPath
316     $jscPath = Pathname.new(jscArg)
317 else
318     $jscPath = Pathname.new(jscArg).realpath
319 end
320
321 $progressMeter = ($verbosity == 0 and $stdout.tty? and $remoteHosts.length <= 1)
322
323 if $bundle
324     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
325     $outputDir = $bundle
326 end
327
328 # Try to determine architecture. Return nil on failure.
329 def machOArchitectureCode
330     begin 
331         otoolLines = `otool -afh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
332         otoolLines.each_with_index {
333             | value, index |
334             if value =~ /magic/ and value =~ /cputype/
335                 return otoolLines[index + 1].split[1].to_i
336             end
337         }
338     rescue
339         $stderr.puts "Warning: unable to execute otool."
340     end
341     $stderr.puts "Warning: unable to determine architecture."
342     nil
343 end
344
345 def determineArchitectureFromMachOBinary
346     code = machOArchitectureCode
347     return nil unless code
348     is64BitFlag = 0x01000000
349     case code
350     when 7
351         "x86"
352     when 7 | is64BitFlag
353         "x86-64"
354     when 12
355         "arm"
356     when 12 | is64BitFlag
357         "arm64"
358     else
359         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
360         nil
361     end
362 end
363
364 def determineArchitectureFromELFBinary
365     f = File.open($jscPath.to_s)
366     data = f.read(19)
367
368     if !(data[0,4] == "\x7F\x45\x4C\x46")
369         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
370         return nil
371     end
372
373     code = data[18].ord
374     case code
375     when 3
376         "x86"
377     when 8
378         "mips"
379     when 62
380         "x86-64"
381     when 40
382         "arm"
383     when 183
384         "arm64"
385     else
386         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
387         nil
388     end
389 end
390
391 def determineArchitectureFromPEBinary
392     f = File.open($jscPath.to_s)
393     data = f.read(1024)
394
395     if !(data[0, 2] == "MZ")
396         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
397         return nil
398     end
399
400     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
401
402     if !(data[peHeaderAddr, 4] == "PE\0\0")
403         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
404         return nil
405     end
406
407     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
408
409     case machine
410     when 0x014c
411         "x86"
412     when 0x8664
413         "x86-64"
414     else
415         $stderr.puts "Warning: unsupported machine type: #{machine}"
416         nil
417     end
418 end
419
420 def determineArchitecture
421     case $hostOS
422     when "darwin"
423         determineArchitectureFromMachOBinary
424     when "linux"
425         determineArchitectureFromELFBinary
426     when "windows"
427         determineArchitectureFromPEBinary
428     else
429         $stderr.puts "Warning: unable to determine architecture on this platform."
430         nil
431     end
432 end
433
434 def determineOS
435     case RbConfig::CONFIG["host_os"]
436     when /darwin/i
437         "darwin"
438     when /linux/i
439         "linux"
440     when /mswin|mingw|cygwin/
441         "windows"
442     else
443         $stderr.puts "Warning: unable to determine host operating system"
444         nil
445     end
446 end
447
448 $hostOS = determineOS unless $hostOS
449 $architecture = determineArchitecture unless $architecture
450 $isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $architecture == "mips" || $hostOS == "windows")
451
452 if ["mips", "x86"].include?($architecture)
453     # The JIT is temporarily disabled on these platforms since
454     # https://trac.webkit.org/changeset/237547
455     $jitTests = false
456 end
457
458 def isFTLEnabled
459     $jitTests && $isFTLPlatform
460 end
461
462 if !$testRunnerType
463     if $remote and $hostOS == "darwin"
464         $testRunnerType = :shell
465     else
466         $testRunnerType = :make
467     end
468 end
469
470 if $remoteHosts.length > 1 and $testRunnerType != :make
471     raise "Multiple remote hosts only supported with make runner"
472 end
473
474 if $testWriter
475     if /[^-a-zA-Z0-9_]/.match($testWriter)
476         raise "Invalid test writer #{$testWriter} given"
477     end
478 end
479
480 $numFailures = 0
481 $numPasses = 0
482
483 # We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
484 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--validateExceptionChecks=true", "--useDollarVM=true", "--maxPerThreadStackUsage=1572864"]
485 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
486 # NOTE: Tests rely on this using scribbleFreeCells.
487 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
488 B3O1_OPTIONS = ["--defaultB3OptLevel=1"]
489 FTL_OPTIONS = ["--useFTLJIT=true"]
490 PROBE_OSR_EXIT_OPTION = ["--useProbeOSRExit=true"]
491
492 require_relative "webkitruby/jsc-stress-test-writer-#{$testWriter}"
493
494 def shouldCollectContinuously?
495     $buildType == "release" or $forceCollectContinuously
496 end
497
498 COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false"] : []
499
500 $runlist = []
501
502 def frameworkFromJSCPath(jscPath)
503     parentDirectory = jscPath.dirname
504     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
505         parentDirectory.dirname
506     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
507         jscPath.dirname + "JavaScriptCore.framework"
508     else
509         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
510         nil
511     end
512 end
513
514 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
515     dir = Pathname.new(".")
516     $benchmarkDirectory.each_filename {
517         | pathComponent |
518         dir += ".."
519     }
520     dir + resourcePath
521 end
522
523 def pathToVM
524     pathToBundleResourceFromBenchmarkDirectory($jscPath)
525 end
526
527 def pathToHelpers
528     pathToBundleResourceFromBenchmarkDirectory(".helpers")
529 end
530
531 $runCommandOptions = {}
532 $testSpecificRequiredOptions = []
533
534 $uniqueFilenameCounter = 0
535 def uniqueFilename(extension)
536     payloadDir = $outputDir + "_payload"
537     Dir.mkdir payloadDir unless payloadDir.directory?
538     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
539     $uniqueFilenameCounter += 1
540     result
541 end
542
543 def baseOutputName(kind)
544     "#{$collectionName}/#{$benchmark}.#{kind}"
545 end
546
547 def addRunCommand(kind, command, outputHandler, errorHandler, *additionalEnv)
548     $didAddRunCommand = true
549     name = baseOutputName(kind)
550     if $filter and name !~ $filter
551         return
552     end
553     plan = Plan.new(
554         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
555         errorHandler)
556     plan.additionalEnv.push(*additionalEnv)
557     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
558         $runlist.unshift plan
559     else
560         $runlist << plan
561     end
562 end
563
564 # Returns true if there were run commands found in the file ($benchmarkDirectory +
565 # $benchmark), in which case those run commands have already been executed. Otherwise
566 # returns false, in which case you're supposed to add your own run commands.
567 def parseRunCommands
568     oldDidAddRunCommand = $didAddRunCommand
569     $didAddRunCommand = false
570     $skipped = false
571
572     Dir.chdir($outputDir) {
573         File.open($benchmarkDirectory + $benchmark) {
574             | inp |
575             inp.each_line {
576                 | line |
577                 begin
578                     doesMatch = line =~ /^\/\/@/
579                 rescue Exception => e
580                     # Apparently this happens in the case of some UTF8 stuff in some files, where
581                     # Ruby tries to be strict and throw exceptions.
582                     next
583                 end
584                 next unless doesMatch
585                 eval $~.post_match
586                 if $skipped
587                     break
588                 end
589             }
590         }
591     }
592
593     result = $didAddRunCommand
594     $didAddRunCommand = result or oldDidAddRunCommand
595     result
596 end
597
598 def slow!
599     $runCommandOptions[:isSlow] = true
600 end
601
602 def requireOptions(*options)
603     $testSpecificRequiredOptions += options
604 end
605
606 def runWithOutputHandler(kind, outputHandler, *options)
607     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + $testSpecificRequiredOptions + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
608 end
609
610 def run(kind, *options)
611     runWithOutputHandler(kind, silentOutputHandler, *options)
612 end
613
614 def runNoFTL(*optionalTestSpecificOptions)
615     run("no-ftl", *optionalTestSpecificOptions)
616 end
617
618 def runWithRAMSize(size, *optionalTestSpecificOptions)
619     run("ram-size-#{size}", "--forceRAMSize=#{size}", *optionalTestSpecificOptions)
620 end
621
622 def runOneLargeHeap(*optionalTestSpecificOptions)
623     if $memoryLimited
624         $didAddRunCommand = true
625         puts "Skipping #{$collectionName}/#{$benchmark}"
626     else
627         run("default", *optionalTestSpecificOptions)
628     end
629 end
630
631 def runNoJIT(*optionalTestSpecificOptions)
632     run("no-jit", "--useJIT=false", *optionalTestSpecificOptions)
633 end
634
635 def runNoLLInt(*optionalTestSpecificOptions)
636     if $jitTests
637         run("no-llint", "--useLLInt=false", *optionalTestSpecificOptions)
638     end
639 end
640
641 # NOTE: Tests rely on this using scribbleFreeCells.
642 def runNoCJITValidate(*optionalTestSpecificOptions)
643     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
644 end
645
646 def runNoCJITValidatePhases(*optionalTestSpecificOptions)
647     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
648 end
649
650 def runNoCJITCollectContinuously(*optionalTestSpecificOptions)
651     run("no-cjit-collect-continuously", *(NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
652 end
653
654 def runDefault(*optionalTestSpecificOptions)
655     run("default", *(FTL_OPTIONS + optionalTestSpecificOptions))
656 end
657
658 def runBigIntEnabled(*optionalTestSpecificOptions)
659     # FIXME: <rdar://problem/40331121>
660     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin")
661         skip
662         return
663     end
664
665     run("big-int-enabled", "--useBigInt=true" , *(FTL_OPTIONS + optionalTestSpecificOptions))
666 end
667
668 def runFTLNoCJIT(*optionalTestSpecificOptions)
669     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
670 end
671
672 def runFTLNoCJITB3O1(*optionalTestSpecificOptions)
673     run("ftl-no-cjit-b3o1", "--useArrayAllocationProfiling=false", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
674 end
675
676 def runFTLNoCJITValidate(*optionalTestSpecificOptions)
677     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + PROBE_OSR_EXIT_OPTION + optionalTestSpecificOptions))
678 end
679
680 def runFTLNoCJITNoPutStackValidate(*optionalTestSpecificOptions)
681     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
682 end
683
684 def runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
685     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
686 end
687
688 def runFTLNoCJITOSRValidation(*optionalTestSpecificOptions)
689     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
690 end
691
692 def runDFGEager(*optionalTestSpecificOptions)
693     run("dfg-eager", *(EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + PROBE_OSR_EXIT_OPTION + optionalTestSpecificOptions))
694 end
695
696 def runDFGEagerNoCJITValidate(*optionalTestSpecificOptions)
697     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
698 end
699
700 def runFTLEager(*optionalTestSpecificOptions)
701     run("ftl-eager", "--airForceBriggsAllocator=true", "--forcePolyProto=true", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
702 end
703
704 def runFTLEagerWatchdog(*optionalTestSpecificOptions)
705     timeout = rand(100)
706     run("ftl-eager-watchdog-#{timeout}", "--watchdog=#{timeout}", "--watchdog-exception-ok", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
707 end
708
709 def runFTLEagerNoCJITValidate(*optionalTestSpecificOptions)
710     run("ftl-eager-no-cjit", "--validateGraph=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
711 end
712
713 def runFTLEagerNoCJITB3O1(*optionalTestSpecificOptions)
714     run("ftl-eager-no-cjit-b3o1", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
715 end
716
717 def runFTLEagerNoCJITOSRValidation(*optionalTestSpecificOptions)
718     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
719 end
720
721 def runNoCJITNoASO(*optionalTestSpecificOptions)
722     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
723 end
724
725 def runNoCJITNoAccessInlining(*optionalTestSpecificOptions)
726     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
727 end
728
729 def runFTLNoCJITNoAccessInlining(*optionalTestSpecificOptions)
730     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
731 end
732
733 def runFTLNoCJITSmallPool(*optionalTestSpecificOptions)
734     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
735 end
736
737 def runNoCJIT(*optionalTestSpecificOptions)
738     run("no-cjit", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
739 end
740
741 def runDFGMaximalFlushPhase(*optionalTestSpecificOptions)
742     run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
743 end
744
745 def runShadowChicken(*optionalTestSpecificOptions)
746     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true", *optionalTestSpecificOptions)
747 end
748
749 def defaultRun
750     if $mode == "quick"
751         defaultQuickRun
752     else
753         runDefault
754         if $jitTests
755             runNoLLInt
756             runNoCJITValidatePhases
757             runNoCJITCollectContinuously if shouldCollectContinuously?
758             runDFGEager
759             if $mode != "basic"
760                 runDFGEagerNoCJITValidate
761                 runDFGMaximalFlushPhase
762             end
763
764             return if !$isFTLPlatform
765
766             runNoFTL
767             runFTLEager
768             runFTLEagerNoCJITValidate
769             runFTLNoCJITSmallPool
770
771             return if $mode == "basic"
772
773             runFTLNoCJITValidate
774             runFTLNoCJITB3O1
775             runFTLNoCJITNoPutStackValidate
776             runFTLNoCJITNoInlineValidate
777             runFTLEagerNoCJITB3O1
778         end
779     end
780 end
781
782 def defaultNoNoLLIntRun
783     if $mode == "quick"
784         defaultQuickRun
785     else
786         runDefault
787         if $jitTests
788             runNoCJITValidatePhases
789             runNoCJITCollectContinuously if shouldCollectContinuously?
790             runDFGEager
791             if $mode != "basic"
792                 runDFGEagerNoCJITValidate
793                 runDFGMaximalFlushPhase
794             end
795
796             return if !$isFTLPlatform
797
798             runNoFTL
799             runFTLNoCJITValidate
800             runFTLNoCJITSmallPool
801
802             return if $mode == "basic"
803
804             runFTLNoCJITB3O1
805             runFTLNoCJITNoPutStackValidate
806             runFTLNoCJITNoInlineValidate
807             runFTLEager
808             runFTLEagerNoCJITValidate
809         end
810     end
811 end
812
813 def defaultQuickRun
814     runDefault
815     if $jitTests
816         runNoCJITValidate
817
818         return if !$isFTLPlatform
819
820         runNoFTL
821         runFTLNoCJITValidate
822     end
823 end
824
825 def defaultSpotCheckNoMaximalFlush
826     defaultQuickRun
827     runNoCJITNoAccessInlining
828
829     return if !$isFTLPlatform
830
831     runFTLNoCJITOSRValidation
832     runFTLNoCJITNoAccessInlining
833     runFTLNoCJITB3O1
834 end
835
836 def defaultSpotCheck
837     defaultSpotCheckNoMaximalFlush
838     runDFGMaximalFlushPhase
839 end
840
841 # This is expected to not do eager runs because eager runs can have a lot of recompilations
842 # for reasons that don't arise in the real world. It's used for tests that assert convergence
843 # by counting recompilations.
844 def defaultNoEagerRun
845     runDefault
846     if $jitTests
847         runNoLLInt
848         runNoCJITValidatePhases
849         runNoCJITCollectContinuously if shouldCollectContinuously?
850
851         return if !$isFTLPlatform
852
853         runNoFTL
854         runFTLNoCJITValidate
855
856         return if $mode == "basic"
857
858         runFTLNoCJITNoInlineValidate
859         runFTLNoCJITB3O1
860     end
861 end
862
863 def defaultNoSamplingProfilerRun
864     runDefault
865     if $jitTests
866         runNoLLInt
867         runNoCJITValidatePhases
868         runNoCJITCollectContinuously if shouldCollectContinuously?
869         runDFGEager
870         runDFGEagerNoCJITValidate
871         runDFGMaximalFlushPhase
872
873         return if !$isFTLPlatform
874
875         runNoFTL
876         runFTLNoCJITNoPutStackValidate
877         runFTLNoCJITNoInlineValidate
878         runFTLEager
879         runFTLEagerNoCJITValidate
880         runFTLNoCJITSmallPool
881     end
882 end
883
884 def runProfiler
885     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
886         skip
887         return
888     end
889
890     profilerOutput = uniqueFilename(".json")
891     if $canRunDisplayProfilerOutput
892         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
893     else
894         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
895         run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
896     end
897 end
898
899 def runExceptionFuzz
900     subCommand = escapeAll([pathToVM.to_s, "--useDollarVM=true", $benchmark.to_s])
901     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
902 end
903
904 def runExecutableAllocationFuzz(name, *options)
905     subCommand = escapeAll([pathToVM.to_s, "--useDollarVM=true", $benchmark.to_s] + options)
906     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
907 end
908
909 def runTypeProfiler
910     if !$jitTests
911         return
912     end
913
914     run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
915     run("ftl-no-cjit-type-profiler-force-poly-proto", "--useTypeProfiler=true", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
916
917     return if !$isFTLPlatform
918
919     run("ftl-type-profiler-ftl-eager", "--useTypeProfiler=true", *(FTL_OPTIONS + EAGER_OPTIONS))
920 end
921
922 def runControlFlowProfiler
923     if !$jitTests
924         return
925     end
926
927     return if !$isFTLPlatform
928
929     run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
930 end
931
932 def runTest262(mode, exception, includeFiles, flags)
933     failsWithException = exception != "NoException"
934     isStrict = false
935     isModule = false
936     isAsync = false
937
938     flags.each {
939         | flag |
940         case flag
941         when :strict
942             isStrict = true
943         when :module
944             isModule = true
945         when :async
946             isAsync = true
947         else
948             raise "Invalid flag for runTest262, #{flag}"
949         end
950     }
951
952     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
953
954     args = [pathToVM.to_s] + BASE_OPTIONS + $testSpecificRequiredOptions
955     args << "--exception=" + exception if failsWithException
956     args << "--test262-async" if isAsync
957     args += includeFiles
958
959     case mode
960     when :normal
961         errorHandler = simpleErrorHandler
962         outputHandler = silentOutputHandler
963     when :fail
964         errorHandler = expectedFailErrorHandler
965         outputHandler = noisyOutputHandler
966     when :failDueToOutdatedOrBadTest
967         errorHandler = expectedFailErrorHandler
968         outputHandler = noisyOutputHandler
969     when :skip
970         return
971     else
972         raise "Invalid mode: #{mode}"
973     end
974
975     if isStrict
976         kind = "default-strict"
977         args << "--strict-file=#{$benchmark}"
978     else
979         kind = "default"
980         if isModule
981             args << "--module-file=#{$benchmark}"
982         else
983             args << $benchmark.to_s
984         end
985     end
986
987     addRunCommand(kind, args, outputHandler, errorHandler)
988 end
989
990 def prepareTest262Fixture
991     # This function is used to add the files used by Test262 modules tests.
992     prepareExtraRelativeFiles([""], $collection)
993 end
994
995 def runES6(mode)
996     args = [pathToVM.to_s] + BASE_OPTIONS + $testSpecificRequiredOptions + [$benchmark.to_s]
997     case mode
998     when :normal
999         errorHandler = simpleErrorHandler
1000     when :fail
1001         errorHandler = expectedFailErrorHandler
1002     when :failDueToOutdatedOrBadTest
1003         errorHandler = expectedFailErrorHandler
1004     when :skip
1005         return
1006     else
1007         raise "Invalid mode: #{mode}"
1008     end
1009     addRunCommand("default", args, noisyOutputHandler, errorHandler)
1010 end
1011
1012 def defaultRunModules(noLLInt: true)
1013     run("default-modules", "-m")
1014
1015     if !$jitTests
1016         return
1017     end
1018
1019     run("no-llint-modules", "-m", "--useLLInt=false") if noLLInt
1020     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
1021     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
1022     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1023
1024     return if !$isFTLPlatform
1025
1026     run("default-ftl-modules", "-m", *FTL_OPTIONS)
1027     run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1028     run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1029     run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
1030     run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1031     run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1032 end
1033
1034 def noNoLLIntRunModules
1035     defaultRunModules(noLLInt: false)
1036 end
1037
1038 def runWebAssembly
1039     return if !$jitTests
1040     return if !$isFTLPlatform
1041     run("default-wasm", "-m", *FTL_OPTIONS)
1042     if $mode != "quick"
1043         run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1044         run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1045         run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1046         run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1047         run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
1048     end
1049 end
1050
1051 def runWebAssemblySuite
1052     return if !$jitTests
1053     return if !$isFTLPlatform
1054     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1055     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1056     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1057     run("default-wasm", "-m", *FTL_OPTIONS)
1058     if $mode != "quick"
1059         run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1060         run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1061         run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1062         run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1063         run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
1064     end
1065 end
1066
1067 def runWebAssemblyEmscripten(mode)
1068     case mode
1069     when :skip
1070         return
1071     end
1072     return if !$jitTests
1073     return if !$isFTLPlatform
1074     wasm = $benchmark.to_s.sub! '.js', '.wasm'
1075     prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
1076     run("default-wasm", *FTL_OPTIONS)
1077     if $mode != "quick"
1078         run("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1079         run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1080         run("wasm-no-call-ic", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1081         run("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1082     end
1083 end
1084
1085 def runWebAssemblySpecTest(mode)
1086     case mode
1087     when :skip
1088         return
1089     end
1090     return if !$jitTests
1091     return if !$isFTLPlatform
1092     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1093
1094     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1095     prepareExtraRelativeFiles(modules.map { |f| "../../" + f }, $collection)
1096
1097     harness = Dir[WASMTESTS_PATH + "spec-harness/" + "*.js"].map { |f| File.basename(f) }
1098     prepareExtraRelativeFiles(harness.map { |f| "../../spec-harness/" + f }, $collection)
1099
1100     runWithOutputHandler("default-wasm", noisyOutputHandler, "../spec-harness.js", *FTL_OPTIONS)
1101     if $mode != "quick"
1102       runWithOutputHandler("wasm-no-cjit-yes-tls-context", noisyOutputHandler, "../spec-harness.js",  "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1103       runWithOutputHandler("wasm-eager-jettison", noisyOutputHandler, "../spec-harness.js", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1104       runWithOutputHandler("wasm-no-call-ic", noisyOutputHandler, "../spec-harness.js", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1105       runWithOutputHandler("wasm-no-tls-context", noisyOutputHandler, "../spec-harness.js", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1106     end
1107 end
1108
1109 def runWebAssemblyLowExecutableMemory(*optionalTestSpecificOptions)
1110     return if !$jitTests
1111     return if !$isFTLPlatform
1112     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1113     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1114     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1115     # Only let WebAssembly get executable memory.
1116     run("default-wasm", "--useConcurrentGC=0" , "--useConcurrentJIT=0", "--jitMemoryReservationSize=15000", "--useBaselineJIT=0", "--useDFGJIT=0", "--useFTLJIT=0", "-m")
1117 end
1118
1119 def runChakra(mode, exception, baselineFile, extraFiles)
1120     raise unless $benchmark.to_s =~ /\.js$/
1121     failsWithException = exception != "NoException"
1122     testName = $~.pre_match
1123
1124     prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
1125     prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)
1126
1127     args = [pathToVM.to_s] + BASE_OPTIONS + $testSpecificRequiredOptions
1128     args += FTL_OPTIONS if $isFTLPlatform
1129     args += EAGER_OPTIONS
1130     args << "--exception=" + exception if failsWithException
1131     args << "--dumpException" if failsWithException
1132     args += ["jsc-lib.js"]
1133
1134     case mode
1135     when :baseline
1136         prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
1137         errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
1138         outputHandler = noisyOutputHandler
1139     when :pass
1140         errorHandler = chakraPassFailErrorHandler
1141         outputHandler = noisyOutputHandler
1142     when :skipDueToOutdatedOrBadTest
1143         return
1144     when :skip
1145         return
1146     else
1147         raise "Invalid mode: #{mode}"
1148     end
1149
1150     kind = "default"
1151     args << $benchmark.to_s
1152
1153     addRunCommand(kind, args, outputHandler, errorHandler)
1154 end
1155
1156 def runLayoutTest(kind, *options)
1157     raise unless $benchmark.to_s =~ /\.js$/
1158     testName = $~.pre_match
1159     if kind
1160         kind = "layout-" + kind
1161     else
1162         kind = "layout"
1163     end
1164
1165     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1166     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1167
1168     args = [pathToVM.to_s] + BASE_OPTIONS + $testSpecificRequiredOptions + options +
1169         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1170          $benchmark.to_s,
1171          (Pathname.new("resources") + "standalone-post.js").to_s]
1172     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1173 end
1174
1175 def runLayoutTestNoFTL
1176     runLayoutTest("no-ftl")
1177 end
1178
1179 def runLayoutTestNoLLInt
1180     runLayoutTest("no-llint", "--useLLInt=false")
1181 end
1182
1183 def runLayoutTestNoCJIT
1184     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1185 end
1186
1187 def runLayoutTestDFGEagerNoCJIT
1188     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1189 end
1190
1191 def runLayoutTestDefault
1192     runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
1193 end
1194
1195 def runLayoutTestFTLNoCJIT
1196     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1197 end
1198
1199 def runLayoutTestFTLEagerNoCJIT
1200     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1201 end
1202
1203 def runLayoutTestFTLEagerNoCJITB3O1
1204     runLayoutTest("ftl-eager-no-cjit-b3o1", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS))
1205 end
1206
1207 def noFTLRunLayoutTest
1208     if !$jitTests
1209         return
1210     end
1211
1212     runLayoutTestNoLLInt
1213     runLayoutTestNoCJIT
1214     runLayoutTestDFGEagerNoCJIT
1215 end
1216
1217 def defaultQuickRunLayoutTest
1218     runLayoutTestDefault
1219     if $jitTests
1220         if $isFTLPlatform
1221             runLayoutTestNoFTL
1222             runLayoutTestFTLNoCJIT
1223             runLayoutTestFTLEagerNoCJIT
1224         else
1225             noFTLRunLayoutTest
1226         end
1227     end
1228 end
1229
1230 def defaultRunLayoutTest
1231     if $mode == "quick"
1232         defaultQuickRunLayoutTest
1233     else
1234         runLayoutTestDefault
1235         if $jitTests
1236             noFTLRunLayoutTest
1237
1238             return if !$isFTLPlatform
1239
1240             runLayoutTestNoFTL
1241             runLayoutTestFTLNoCJIT
1242             runLayoutTestFTLEagerNoCJIT
1243         end
1244     end
1245 end
1246
1247 def noEagerNoNoLLIntTestsRunLayoutTest
1248     runLayoutTestDefault
1249     if $jitTests
1250         runLayoutTestNoCJIT
1251
1252         return if !$isFTLPlatform
1253
1254         runLayoutTestNoFTL
1255         runLayoutTestFTLNoCJIT
1256     end
1257 end
1258
1259 def noNoLLIntRunLayoutTest
1260     runLayoutTestDefault
1261     if $jitTests
1262         runLayoutTestNoCJIT
1263         runLayoutTestDFGEagerNoCJIT
1264
1265         return if !$isFTLPlatform
1266
1267         runLayoutTestNoFTL
1268         runLayoutTestFTLNoCJIT
1269         runLayoutTestFTLEagerNoCJIT
1270     end
1271 end
1272
1273 def prepareExtraRelativeFiles(extraFiles, destination)
1274     Dir.chdir($outputDir) {
1275         extraFiles.each {
1276             | file |
1277             dest = destination + file
1278             FileUtils.mkdir_p(dest.dirname)
1279             FileUtils.cp $extraFilesBaseDir + file, dest
1280         }
1281     }
1282 end
1283
1284 def baseDirForCollection(collectionName)
1285     Pathname(".tests") + collectionName
1286 end
1287
1288 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1289     raise unless absoluteBase.absolute?
1290     Dir.chdir($outputDir) {
1291         collectionBaseDir = baseDirForCollection($collectionName)
1292         extraFiles.each {
1293             | file |
1294             destination = collectionBaseDir + file
1295             FileUtils.mkdir_p destination.dirname unless destination.directory?
1296             FileUtils.cp absoluteBase + file, destination
1297         }
1298     }
1299 end
1300
1301 def runMozillaTest(kind, mode, extraFiles, *options)
1302     if kind
1303         kind = "mozilla-" + kind
1304     else
1305         kind = "mozilla"
1306     end
1307     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1308     args = [pathToVM.to_s] + BASE_OPTIONS + $testSpecificRequiredOptions + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1309     case mode
1310     when :normal
1311         errorHandler = mozillaErrorHandler
1312     when :negative
1313         errorHandler = mozillaExit3ErrorHandler
1314     when :fail
1315         errorHandler = mozillaFailErrorHandler
1316     when :failDueToOutdatedOrBadTest
1317         errorHandler = mozillaFailErrorHandler
1318     when :skip
1319         return
1320     else
1321         raise "Invalid mode: #{mode}"
1322     end
1323     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1324 end
1325
1326 def runMozillaTestDefault(mode, *extraFiles)
1327     runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
1328 end
1329
1330 def runMozillaTestNoFTL(mode, *extraFiles)
1331     runMozillaTest("no-ftl", mode, extraFiles)
1332 end
1333
1334 def runMozillaTestLLInt(mode, *extraFiles)
1335     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1336 end
1337
1338 def runMozillaTestBaselineJIT(mode, *extraFiles)
1339     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1340 end
1341
1342 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1343     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1344 end
1345
1346 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1347     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1348 end
1349
1350 def defaultQuickRunMozillaTest(mode, *extraFiles)
1351     if $jitTests
1352         runMozillaTestDefault(mode, *extraFiles)
1353         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1354     else
1355         runMozillaTestNoFTL(mode, *extraFiles)
1356         if $jitTests
1357             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1358         end
1359     end
1360 end
1361
1362 def defaultRunMozillaTest(mode, *extraFiles)
1363     if $mode == "quick"
1364         defaultQuickRunMozillaTest(mode, *extraFiles)
1365     else
1366         runMozillaTestNoFTL(mode, *extraFiles)
1367         if $jitTests
1368             runMozillaTestLLInt(mode, *extraFiles)
1369             runMozillaTestBaselineJIT(mode, *extraFiles)
1370             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1371             runMozillaTestDefault(mode, *extraFiles)
1372             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
1373         end
1374     end
1375 end
1376
1377 def runNoisyTestImpl(kind, options, additionalEnv)
1378     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + $testSpecificRequiredOptions + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler, *additionalEnv)
1379 end
1380
1381 def runNoisyTest(kind, *options)
1382     runNoisyTestImpl(kind, options, [])
1383 end
1384
1385 def runNoisyTestWithEnv(kind, *additionalEnv)
1386     runNoisyTestImpl(kind, [], additionalEnv)
1387 end
1388
1389 def runNoisyTestDefault
1390     runNoisyTest("default", *FTL_OPTIONS)
1391 end
1392
1393 def runNoisyTestNoFTL
1394     runNoisyTest("no-ftl")
1395 end
1396
1397 def runNoisyTestNoCJIT
1398     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1399 end
1400
1401 def runNoisyTestNoCJITB3O1
1402     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS))
1403 end
1404
1405 def runNoisyTestEagerNoCJIT
1406     runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1407 end
1408
1409 def defaultRunNoisyTest
1410     runNoisyTestDefault
1411     if $jitTests and $isFTLPlatform
1412         runNoisyTestNoFTL
1413         runNoisyTestNoCJIT
1414         runNoisyTestNoCJITB3O1
1415         runNoisyTestEagerNoCJIT
1416     end
1417 end
1418
1419 def skip
1420     $didAddRunCommand = true
1421     $skipped = true
1422     puts "Skipping #{$collectionName}/#{$benchmark}"
1423 end
1424
1425 def allJSFiles(path)
1426     if path.file?
1427         [path]
1428     else
1429         result = []
1430         Dir.foreach(path) {
1431             | filename |
1432             next unless filename =~ /\.m?js$/
1433             next unless (path + filename).file?
1434             result << path + filename
1435         }
1436         result
1437     end
1438 end
1439
1440 def uniqueifyName(names, name)
1441     result = name.to_s
1442     toAdd = 1
1443     while names[result]
1444         result = "#{name}-#{toAdd}"
1445         toAdd += 1
1446     end
1447     names[result] = true
1448     result
1449 end
1450
1451 def simplifyCollectionName(collectionPath)
1452     outerDir = collectionPath.dirname
1453     name = collectionPath.basename
1454     lastName = name
1455     if collectionPath.directory?
1456         while lastName.to_s =~ /test/
1457             lastName = outerDir.basename
1458             name = lastName + name
1459             outerDir = outerDir.dirname
1460         end
1461     end
1462     uniqueifyName($collectionNames, name)
1463 end
1464
1465 def prepareCollection(name)
1466     FileUtils.mkdir_p $outputDir + name
1467
1468     absoluteCollection = $collection.realpath
1469
1470     Dir.chdir($outputDir) {
1471         bundleDir = baseDirForCollection(name)
1472
1473         # Create the proper directory structures.
1474         FileUtils.mkdir_p bundleDir
1475         if bundleDir.basename == $collection.basename
1476             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1477             $collection = bundleDir
1478         else
1479             FileUtils.cp_r absoluteCollection, bundleDir
1480             $collection = bundleDir + $collection.basename
1481         end
1482
1483         $extraFilesBaseDir = absoluteCollection
1484     }
1485 end
1486
1487 $collectionNames = {}
1488
1489 def handleCollectionFile(collection)
1490     collectionName = simplifyCollectionName(collection)
1491    
1492     paths = {}
1493     subCollections = []
1494     YAML::load(IO::read(collection)).each {
1495         | entry |
1496         if entry["collection"]
1497             subCollections << entry["collection"]
1498             next
1499         end
1500         
1501         if Pathname.new(entry["path"]).absolute?
1502             raise "Absolute path: " + entry["path"] + " in #{collection}"
1503         end
1504         
1505         if paths[entry["path"]]
1506             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1507         end
1508         
1509         subCollection = collection.dirname + entry["path"]
1510         
1511         if subCollection.file?
1512             subCollectionName = Pathname.new(entry["path"]).dirname
1513         else
1514             subCollectionName = entry["path"]
1515         end
1516         
1517         $collection = subCollection
1518         $collectionName = Pathname.new(collectionName)
1519         Pathname.new(subCollectionName).each_filename {
1520             | filename |
1521             next if filename =~ /^\./
1522             $collectionName += filename
1523         }
1524         $collectionName = $collectionName.to_s
1525         
1526         prepareCollection($collectionName)
1527       
1528         Dir.chdir($outputDir) {
1529             pathsToSearch = [$collection]
1530             if entry["tests"]
1531                 if entry["tests"].is_a? Array
1532                     pathsToSearch = entry["tests"].map {
1533                         | testName |
1534                         pathsToSearch[0] + testName
1535                     }
1536                 else
1537                     pathsToSearch[0] += entry["tests"]
1538                 end
1539             end
1540             pathsToSearch.each {
1541                 | pathToSearch |
1542                 allJSFiles(pathToSearch).each {
1543                     | path |
1544                     
1545                     $benchmark = path.basename
1546                     $benchmarkDirectory = path.dirname
1547                     
1548                     $runCommandOptions = {}
1549                     eval entry["cmd"]
1550                 }
1551             }
1552         }
1553     }
1554     
1555     subCollections.each {
1556         | subCollection |
1557         handleCollection(collection.dirname + subCollection)
1558     }
1559 end
1560
1561 def handleCollectionDirectory(collection)
1562     collectionName = simplifyCollectionName(collection)
1563     
1564     $collection = collection
1565     $collectionName = collectionName
1566     prepareCollection(collectionName)
1567    
1568     Dir.chdir($outputDir) {
1569         $benchmarkDirectory = $collection
1570         allJSFiles($collection).each {
1571             | path |
1572             
1573             $benchmark = path.basename
1574             
1575             $runCommandOptions = {}
1576             $testSpecificRequiredOptions = []
1577             defaultRun unless parseRunCommands
1578         }
1579     }
1580 end
1581
1582 def handleCollection(collection)
1583     collection = Pathname.new(collection)
1584     
1585     if collection.file?
1586         handleCollectionFile(collection)
1587     else
1588         handleCollectionDirectory(collection)
1589     end
1590 end
1591
1592 def appendFailure(plan)
1593     File.open($outputDir + "failed", "a") {
1594         | outp |
1595         outp.puts plan.name
1596     }
1597     $numFailures += 1
1598 end
1599
1600 def appendPass(plan)
1601     File.open($outputDir + "passed", "a") {
1602         | outp |
1603         outp.puts plan.name
1604     }
1605     $numPasses += 1
1606 end
1607
1608 def appendResult(plan, didPass)
1609     File.open($outputDir + "results", "a") {
1610         | outp |
1611         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1612     }
1613 end
1614
1615 def prepareBundle
1616     raise if $bundle
1617
1618     if $doNotMessWithVMPath
1619         if !$remote and !$tarball
1620             $testingFrameworkPath = (frameworkFromJSCPath($jscPath) || $jscPath.dirname).realpath
1621             $jscPath = Pathname.new($jscPath).realpath
1622         else
1623             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1624         end
1625     else
1626         originalJSCPath = $jscPath
1627         vmDir = $outputDir + ".vm"
1628         FileUtils.mkdir_p vmDir
1629         
1630         frameworkPath = frameworkFromJSCPath($jscPath)
1631         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1632         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1633         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1634
1635         if frameworkPath
1636             source = frameworkPath
1637             destination = Pathname.new(".vm")
1638         elsif $hostOS == "windows"
1639             # Make sure to copy dll along with jsc on Windows
1640             originalJSCDir = File.dirname(originalJSCPath)
1641             source = [originalJSCPath] + [originalJSCDir + "/jscLib.dll"]
1642
1643             # Check for and copy JavaScriptCore.dll and WTF.dll for dynamic builds
1644             javaScriptCoreDLLPath = File.join(originalJSCDir, "JavaScriptCore.dll")
1645             wtfDLLPath = File.join(originalJSCDir, "WTF.dll")
1646             if (File.exists?(javaScriptCoreDLLPath))
1647                 source = source + [javaScriptCoreDLLPath]
1648             end
1649             if (File.exists?(wtfDLLPath))
1650                 source = source + [wtfDLLPath]
1651             end
1652
1653             destination = $jscPath.dirname
1654
1655             Dir.chdir($outputDir) {
1656                 FileUtils.mkdir_p destination
1657             }
1658         else
1659             source = originalJSCPath
1660             destination = $jscPath
1661
1662             Dir.chdir($outputDir) {
1663                 FileUtils.mkdir_p $jscPath.dirname
1664             }
1665         end
1666
1667         Dir.chdir($outputDir) {
1668             if $copyVM
1669                 FileUtils.cp_r source, destination
1670             else
1671                 begin 
1672                     FileUtils.ln_s source, destination
1673                 rescue Exception
1674                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1675                     FileUtils.cp_r source, destination
1676                 end
1677             end
1678
1679             if $remote and $hostOS == "linux"
1680                 begin
1681                     dependencies = `ldd #{source}`
1682                     dependencies.split(/\n/).each {
1683                         | dependency |
1684                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1685                     }
1686                 rescue
1687                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1688                 end
1689             end
1690         }
1691     end
1692     
1693     Dir.chdir($outputDir) {
1694         FileUtils.cp_r HELPERS_PATH, ".helpers"
1695     }
1696
1697     ARGV.each {
1698         | collection |
1699         handleCollection(collection)
1700     }
1701
1702     puts
1703 end
1704
1705 def cleanOldResults
1706     raise unless $bundle
1707
1708     eachResultFile($outputDir) {
1709         | path |
1710         FileUtils.rm_f path
1711     }
1712 end
1713
1714 def cleanEmptyResultFiles
1715     eachResultFile($outputDir) {
1716         | path |
1717         next unless path.basename.to_s =~ /\.out$/
1718         next unless FileTest.size(path) == 0
1719         FileUtils.rm_f path
1720     }
1721 end
1722
1723 def eachResultFile(startingDir, &block)
1724     dirsToClean = [startingDir]
1725     until dirsToClean.empty?
1726         nextDir = dirsToClean.pop
1727         Dir.foreach(nextDir) {
1728             | entry |
1729             next if entry =~ /^\./
1730             path = nextDir + entry
1731             if path.directory?
1732                 dirsToClean.push(path)
1733             else
1734                 block.call(path)
1735             end
1736         }
1737     end
1738 end
1739
1740 def prepareTestRunner(remoteIndex=0)
1741     raise if $bundle
1742
1743     $runlist.each_with_index {
1744         | plan, index |
1745         plan.index = index
1746     }
1747
1748     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1749     toDelete = []
1750     Dir.foreach($runnerDir) {
1751         | filename |
1752         if filename =~ /^test_/
1753             toDelete << filename
1754         end
1755     }
1756     
1757     toDelete.each {
1758         | filename |
1759         File.unlink($runnerDir + filename)
1760     }
1761
1762     $runlist.each {
1763         | plan |
1764         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1765     }
1766
1767     case $testRunnerType
1768     when :make
1769         prepareMakeTestRunner(remoteIndex)
1770     when :shell
1771         prepareShellTestRunner
1772     when :ruby
1773         prepareRubyTestRunner
1774     else
1775         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1776     end
1777 end
1778
1779 def cleanRunnerDirectory
1780     raise unless $bundle
1781     Dir.foreach($runnerDir) {
1782         | filename |
1783         next unless filename =~ /^test_fail/
1784         FileUtils.rm_f $runnerDir + filename
1785     }
1786 end
1787
1788 def sshRead(cmd, remoteIndex=0)
1789     raise unless $remote
1790
1791     remoteHost = $remoteHosts[remoteIndex]
1792
1793     result = ""
1794     IO.popen("ssh -o NoHostAuthenticationForLocalhost=yes -p #{remoteHost.port} #{remoteHost.user}@#{remoteHost.host} '#{cmd}'", "r") {
1795       | inp |
1796       inp.each_line {
1797         | line |
1798         result += line
1799       }
1800     }
1801     raise "#{$?}" unless $?.success?
1802     result
1803 end
1804
1805 def runCommandOnTester(cmd)
1806     if $remote
1807         result = sshRead(cmd)
1808     else
1809         result = `#{cmd}`
1810     end
1811 end
1812
1813 def numberOfProcessors
1814     if $hostOS == "windows"
1815         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1816     else
1817         begin
1818             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1819         rescue
1820             numProcessors = 0
1821         end
1822
1823         if numProcessors == 0
1824             begin
1825                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1826             rescue
1827                 numProcessors == 0
1828             end
1829         end
1830     end
1831
1832     if numProcessors == 0
1833         numProcessors = 1
1834     end
1835     return numProcessors
1836 end
1837
1838 def runAndMonitorTestRunnerCommand(*cmd)
1839     numberOfTests = 0
1840     Dir.chdir($runnerDir) {
1841         # -1 for the runscript, and -2 for '..' and '.'
1842         numberOfTests = Dir.entries(".").count - 3
1843     }
1844     unless $progressMeter
1845         mysys(cmd.join(' '))
1846     else
1847        running = {}
1848        didRun = {}
1849        didFail = {}
1850        blankLine = true
1851        prevStringLength = 0
1852        IO.popen(cmd.join(' '), mode="r") {
1853            | inp |
1854            inp.each_line {
1855                | line |
1856                line = line.scrub.chomp
1857                if line =~ /^Running /
1858                    running[$~.post_match] = true
1859                elsif line =~ /^PASS: /
1860                    didRun[$~.post_match] = true
1861                elsif line =~ /^FAIL: /
1862                    didRun[$~.post_match] = true
1863                    didFail[$~.post_match] = true
1864                else
1865                    unless blankLine
1866                        print("\r" + " " * prevStringLength + "\r")
1867                    end
1868                    puts line
1869                    blankLine = true
1870                end
1871
1872                def lpad(str, chars)
1873                    str = str.to_s
1874                    if str.length > chars
1875                        str
1876                    else
1877                       "%#{chars}s"%(str)
1878                    end
1879                end
1880
1881                string  = ""
1882                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1883                unless didFail.empty?
1884                    string += " (failed #{didFail.size})"
1885                end
1886                string += " "
1887                (running.size - didRun.size).times {
1888                    string += "."
1889                }
1890                if string.length < prevStringLength
1891                    print string
1892                    print(" " * (prevStringLength - string.length))
1893                end
1894                print string
1895                prevStringLength = string.length
1896                blankLine = false
1897                $stdout.flush
1898            }
1899        }
1900        puts
1901        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1902     end
1903 end
1904
1905 def runTestRunner(remoteIndex=0)
1906     if $remote
1907         remoteHost = $remoteHosts[remoteIndex]
1908         if !remoteHost.remoteDirectory
1909             remoteHost.remoteDirectory = JSON::parse(sshRead("cat ~/.bencher", remoteIndex))["tempPath"]
1910         end
1911         mysys("ssh", "-o", "NoHostAuthenticationForLocalhost=yes", "-p", remoteHost.port.to_s, "#{remoteHost.user}@#{remoteHost.host}", "mkdir -p #{remoteHost.remoteDirectory}")
1912         mysys("scp", "-o", "NoHostAuthenticationForLocalhost=yes", "-P", remoteHost.port.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{remoteHost.user}@#{remoteHost.host}:#{remoteHost.remoteDirectory}")
1913         remoteScript = "\""
1914         remoteScript += "cd #{remoteHost.remoteDirectory} && "
1915         remoteScript += "rm -rf #{$outputDir.basename} && "
1916         remoteScript += "tar xzf #{$tarFileName} && "
1917         remoteScript += "cd #{$outputDir.basename}/.runner && "
1918         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1919         remoteScript += "export LD_LIBRARY_PATH=#{remoteHost.remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1920         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
1921         $envVars.each { |var| remoteScript += "export " << var << "\n" }
1922         remoteScript += "#{testRunnerCommand(remoteIndex)}\""
1923         runAndMonitorTestRunnerCommand("ssh", "-o", "NoHostAuthenticationForLocalhost=yes", "-p", remoteHost.port.to_s, "#{remoteHost.user}@#{remoteHost.host}", remoteScript)
1924     else
1925         Dir.chdir($runnerDir) {
1926             runAndMonitorTestRunnerCommand(testRunnerCommand)
1927         }
1928     end
1929 end
1930
1931 def detectFailures
1932     raise if $bundle
1933
1934     failures = []
1935     
1936     if $remote
1937         $remoteHosts.each_with_index {
1938             | host, remoteIndex |
1939             output = sshRead("cd #{host.remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"", remoteIndex)
1940             output.split(/\n/).each {
1941                 | line |
1942                 next unless line =~ /test_fail_/
1943                 failures << $~.post_match.to_i
1944             }
1945         }
1946     else
1947         Dir.foreach($runnerDir) {
1948             | filename |
1949             next unless filename =~ /test_fail_/
1950             failures << $~.post_match.to_i
1951         }
1952     end
1953
1954     failureSet = {}
1955
1956     failures.each {
1957         | failure | 
1958         appendFailure($runlist[failure])
1959         failureSet[failure] = true
1960     }
1961
1962     familyMap = {}
1963     $runlist.each_with_index {
1964         | plan, index |
1965         unless familyMap[plan.family]
1966             familyMap[plan.family] = []
1967         end
1968         if failureSet[index]
1969             appendResult(plan, false)
1970             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
1971             next
1972         else
1973             appendResult(plan, true)
1974             familyMap[plan.family] << {:result => "PASS", :plan => plan};
1975         end
1976         appendPass(plan)
1977     }
1978
1979     File.open($outputDir + "resultsByFamily", "w") {
1980         | outp |
1981         first = true
1982         familyMap.keys.sort.each {
1983             | familyName |
1984             if first
1985                 first = false
1986             else
1987                 outp.puts
1988             end
1989             
1990             outp.print "#{familyName}:"
1991
1992             numPassed = 0
1993             familyMap[familyName].each {
1994                 | entry |
1995                 if entry[:result] == "PASS"
1996                     numPassed += 1
1997                 end
1998             }
1999
2000             if numPassed == familyMap[familyName].size
2001                 outp.puts " PASSED"
2002             elsif numPassed == 0
2003                 outp.puts " FAILED"
2004             else
2005                 outp.puts
2006                 familyMap[familyName].each {
2007                     | entry |
2008                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
2009                 }
2010             end
2011         }
2012     }
2013 end
2014
2015 def compressBundle
2016     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
2017     $stderr.puts ">> #{cmd}" if $verbosity >= 2
2018     raise unless system(cmd)
2019 end
2020
2021 def clean(file)
2022     FileUtils.rm_rf file unless $bundle
2023 end
2024
2025 clean($outputDir + "failed")
2026 clean($outputDir + "passed")
2027 clean($outputDir + "results")
2028 clean($outputDir + "resultsByFamily")
2029 clean($outputDir + ".vm")
2030 clean($outputDir + ".helpers")
2031 clean($outputDir + ".runner")
2032 clean($outputDir + ".tests")
2033 clean($outputDir + "_payload")
2034
2035 Dir.mkdir($outputDir) unless $outputDir.directory?
2036
2037 $outputDir = $outputDir.realpath
2038 $runnerDir = $outputDir + ".runner"
2039
2040 if !$numChildProcesses
2041     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
2042         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
2043     else
2044         $numChildProcesses = numberOfProcessors
2045     end
2046 end
2047
2048 if ENV["JSCTEST_timeout"]
2049     # In the worst case, the processors just interfere with each other.
2050     # Increase the timeout proportionally to the number of processors.
2051     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
2052 end
2053
2054 def runBundle
2055     raise unless $bundle
2056
2057     cleanRunnerDirectory
2058     cleanOldResults
2059     runTestRunner
2060     cleanEmptyResultFiles
2061 end
2062
2063 def runNormal
2064     raise if $bundle or $tarball
2065
2066     prepareBundle
2067     prepareTestRunner
2068     runTestRunner
2069     cleanEmptyResultFiles
2070     detectFailures
2071 end
2072
2073 def runTarball
2074     raise unless $tarball
2075
2076     prepareBundle 
2077     prepareTestRunner
2078     compressBundle
2079 end
2080
2081 def runRemote
2082     raise unless $remote
2083
2084     prepareBundle
2085     $remoteHosts.each_index {
2086         | index |
2087         prepareTestRunner(index)
2088     }
2089     compressBundle
2090     threads = []
2091     $remoteHosts.each_index {
2092         | index |
2093         threads << Thread.new {
2094             runTestRunner(index)
2095         }
2096     }
2097     threads.each { |thr| thr.join }
2098     detectFailures
2099 end
2100
2101 puts
2102 if $bundle
2103     runBundle
2104 elsif $remote
2105     runRemote
2106 elsif $tarball
2107     runTarball
2108 else
2109     runNormal
2110 end