Cache bytecode to disk
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2016 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 RemoteHost = Struct.new(:name, :user, :host, :port, :remoteDirectory)
47
48 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
49 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
50 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
51 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
52 WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
53 CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
54 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
55 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
56
57 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
58
59 begin
60     require 'shellwords'
61 rescue Exception => e
62     $stderr.puts "Warning: did not find shellwords, not running any tests."
63     exit 0
64 end
65
66 $canRunDisplayProfilerOutput = false
67
68 begin
69     require 'rubygems'
70     require 'json'
71     require 'highline'
72     $canRunDisplayProfilerOutput = true
73 rescue Exception => e
74     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
75     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
76     $stderr.puts "Error: #{e.inspect}"
77 end
78
79 def printCommandArray(*cmd)
80     begin
81         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
82     rescue
83         commandArray = cmd.join(' ')
84     end
85     $stderr.puts ">> #{commandArray}"
86 end
87
88 def mysys(*cmd)
89     printCommandArray(*cmd) if $verbosity >= 1
90     raise "Command failed: #{$?.inspect}" unless system(*cmd)
91 end
92
93 def escapeAll(array)
94     array.map {
95         | v |
96         raise "Detected a non-string in #{inspect}" unless v.is_a? String
97         Shellwords.shellescape(v)
98     }.join(' ')
99 end
100
101
102 $jscPath = nil
103 $doNotMessWithVMPath = false
104 $jitTests = true
105 $memoryLimited = false
106 $outputDir = Pathname.new("results")
107 $verbosity = 0
108 $bundle = nil
109 $tarball = false
110 $tarFileName = "payload.tar.gz"
111 $copyVM = false
112 $testRunnerType = nil
113 $testWriter = "default"
114 $remoteHosts = []
115 $architecture = nil
116 $hostOS = nil
117 $filter = nil
118 $envVars = []
119 $mode = "full"
120 $buildType = "release"
121 $forceCollectContinuously = false
122
123 def usage
124     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
125     puts
126     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
127     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
128     puts "                            --jsc specifies an already present JavaScriptCore to test."
129     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
130     puts "                            Skip tests tagged with //@skip if $memoryLimited"
131     puts "--no-jit                    Do not run JIT specific tests."
132     puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
133     puts "                            platform."
134     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
135     puts "--verbose            (-v)   Print more things while running."
136     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
137     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
138     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
139     puts "                            e.g. x86, x86_64, arm."
140     puts "--os                        Specify os instead of determining from JavaScriptCore build."
141     puts "                            e.g. darwin, linux & windows."
142     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
143     puts "                            In general the shell runner is slower than the make runner."
144     puts "--make-runner               Uses the faster make-based runner."
145     puts "--ruby-runner               Uses the ruby runner for machines without unix shell or make."
146     puts "--test-writer [writer]      Specifies the test script format."
147     puts "                            default is to use shell scripts to run the tests"
148     puts "                            \"ruby\" to use ruby scripts for systems without a unix shell."
149     puts "--remote                    Specify a remote host on which to run tests from command line argument."
150     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
151     puts "--child-processes    (-c)   Specify the number of child processes."
152     puts "--filter                    Only run tests whose name matches the given regular expression."
153     puts "--help               (-h)   Print this message."
154     puts "--env-vars                  Add a list of environment variables to set before running jsc."
155     puts "                            Each environment variable should be separated by a space."
156     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
157     puts "                            it will override the default value."
158     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
159     puts "--basic                     Run with default and these additional modes: no-llint,"
160     puts "                            no-cjit-validate-phases, no-cjit-collect-continuously, dfg-eager"
161     puts "                            and for FTL platforms: no-ftl, ftl-eager-no-cjit and"
162     puts "                            ftl-no-cjit-small-pool."
163     exit 1
164 end
165
166 jscArg = nil
167
168 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
169                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
170                ['--no-copy', GetoptLong::NO_ARGUMENT],
171                ['--memory-limited', GetoptLong::NO_ARGUMENT],
172                ['--no-jit', GetoptLong::NO_ARGUMENT],
173                ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
174                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
175                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
176                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
177                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
178                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
179                ['--os', GetoptLong::REQUIRED_ARGUMENT],
180                ['--shell-runner', GetoptLong::NO_ARGUMENT],
181                ['--make-runner', GetoptLong::NO_ARGUMENT],
182                ['--ruby-runner', GetoptLong::NO_ARGUMENT],
183                ['--test-writer', GetoptLong::REQUIRED_ARGUMENT],
184                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
185                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
186                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
187                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
188                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
189                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
190                ['--debug', GetoptLong::NO_ARGUMENT],
191                ['--release', GetoptLong::NO_ARGUMENT],
192                ['--quick', '-q', GetoptLong::NO_ARGUMENT],
193                ['--basic', GetoptLong::NO_ARGUMENT]).each {
194     | opt, arg |
195     case opt
196     when '--help'
197         usage
198     when '--jsc'
199         jscArg = arg
200     when '--no-copy'
201         $doNotMessWithVMPath = true
202     when '--output-dir'
203         $outputDir = Pathname.new(arg)
204     when '--memory-limited'
205         $memoryLimited = true
206     when '--no-jit'
207         $jitTests = false
208     when '--force-collectContinuously'
209         $forceCollectContinuously = true;
210     when '--verbose'
211         $verbosity += 1
212     when '--run-bundle'
213         $bundle = Pathname.new(arg)
214     when '--tarball'
215         $tarball = true
216         $copyVM = true
217         $tarFileName = arg unless arg == ''
218     when '--force-vm-copy'
219         $copyVM = true
220     when '--shell-runner'
221         $testRunnerType = :shell
222     when '--make-runner'
223         $testRunnerType = :make
224     when '--ruby-runner'
225         $testRunnerType = :ruby
226     when '--test-writer'
227         $testWriter = arg
228     when '--remote'
229         $copyVM = true
230         $tarball = true
231         $remote = true
232         uri = URI("ssh://" + arg)
233         $remoteHosts << RemoteHost.new("default-#{$remoteHosts.length}", uri.user, uri.host, uri.port)
234     when '--remote-config-file'
235         $remoteConfigFile = arg
236     when '--child-processes'
237         $numChildProcesses = arg.to_i
238     when '--filter'
239         $filter = Regexp.new(arg)
240     when '--arch'
241         $architecture = arg
242     when '--os'
243         $hostOS = arg
244     when '--env-vars'
245         $envVars = arg.gsub(/\s+/, ' ').split(' ')
246     when '--quick'
247         $mode = "quick"
248     when '--basic'
249         $mode = "basic"
250     when '--debug'
251         $buildType = "debug"
252     when '--release'
253         $buildType = "release"
254     end
255 }
256
257 if $remoteConfigFile
258     file = File.read($remoteConfigFile)
259     config = JSON.parse(file)
260
261     # old style config allowing for only one remote
262     if !$remote and config['remote']
263         $copyVM = true
264         $tarball = true
265         $remote = true
266         uri = URI("ssh://" + config['remote'])
267         $remoteHosts = [ RemoteHost.new("default", uri.user, uri.host, uri.port) ]
268         if config['remoteDirectory']
269             $remoteHosts[0].remoteDirectory = config['remoteDirectory']
270         end
271     end
272
273     # we can combine --remote and a new style config
274     if config['remotes']
275         $copyVM = true
276         $tarball = true
277         $remote = true
278         $remoteHosts += config['remotes'].map {
279             | remote |
280             uri = URI("ssh://" + remote['address'])
281
282             host = RemoteHost.new(remote['name'], uri.user, uri.host, uri.port)
283             if remote['remoteDirectory']
284                 host.remoteDirectory = remote['remoteDirectory']
285             end
286             host
287         }
288     end
289 end
290
291 unless jscArg
292     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
293     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
294     command += ($buildType == "release") ? " --release" : " --debug"
295     command += " --executablePath"
296
297     output = `#{command}`.split("\n")
298     if !output.length
299         $stderr.puts "Error: must specify --jsc <path>"
300         exit 1
301     end
302
303     output = output[0]
304     jscArg = Pathname.new(output).join("jsc")
305     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
306     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
307     if !File.file?(jscArg)
308         $stderr.puts "Error: must specify --jsc <path>"
309         exit 1
310     end
311
312     puts "Using the following jsc path: #{jscArg}"
313 end
314
315 if $doNotMessWithVMPath
316     $jscPath = Pathname.new(jscArg)
317 else
318     $jscPath = Pathname.new(jscArg).realpath
319 end
320
321 $progressMeter = ($verbosity == 0 and $stdout.tty? and $remoteHosts.length <= 1)
322
323 if $bundle
324     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
325     $outputDir = $bundle
326 end
327
328 # Try to determine architecture. Return nil on failure.
329 def machOArchitectureCode
330     begin 
331         otoolLines = `otool -afh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
332         otoolLines.each_with_index {
333             | value, index |
334             if value =~ /magic/ and value =~ /cputype/
335                 return otoolLines[index + 1].split[1].to_i
336             end
337         }
338     rescue
339         $stderr.puts "Warning: unable to execute otool."
340     end
341     $stderr.puts "Warning: unable to determine architecture."
342     nil
343 end
344
345 def determineArchitectureFromMachOBinary
346     code = machOArchitectureCode
347     return nil unless code
348     is64BitFlag = 0x01000000
349     case code
350     when 7
351         "x86"
352     when 7 | is64BitFlag
353         "x86-64"
354     when 12
355         "arm"
356     when 12 | is64BitFlag
357         "arm64"
358     else
359         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
360         nil
361     end
362 end
363
364 def determineArchitectureFromELFBinary
365     f = File.open($jscPath.to_s)
366     data = f.read(19)
367
368     if !(data[0,4] == "\x7F\x45\x4C\x46")
369         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
370         return nil
371     end
372
373     code = data[18].ord
374     case code
375     when 3
376         "x86"
377     when 8
378         "mips"
379     when 62
380         "x86-64"
381     when 40
382         "arm"
383     when 183
384         "arm64"
385     else
386         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
387         nil
388     end
389 end
390
391 def determineArchitectureFromPEBinary
392     f = File.open($jscPath.to_s)
393     data = f.read(1024)
394
395     if !(data[0, 2] == "MZ")
396         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
397         return nil
398     end
399
400     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
401
402     if !(data[peHeaderAddr, 4] == "PE\0\0")
403         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
404         return nil
405     end
406
407     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
408
409     case machine
410     when 0x014c
411         "x86"
412     when 0x8664
413         "x86-64"
414     else
415         $stderr.puts "Warning: unsupported machine type: #{machine}"
416         nil
417     end
418 end
419
420 def determineArchitecture
421     case $hostOS
422     when "darwin"
423         determineArchitectureFromMachOBinary
424     when "linux"
425         determineArchitectureFromELFBinary
426     when "windows"
427         determineArchitectureFromPEBinary
428     else
429         $stderr.puts "Warning: unable to determine architecture on this platform."
430         nil
431     end
432 end
433
434 def determineOS
435     case RbConfig::CONFIG["host_os"]
436     when /darwin/i
437         "darwin"
438     when /linux/i
439         "linux"
440     when /mswin|mingw|cygwin/
441         "windows"
442     else
443         $stderr.puts "Warning: unable to determine host operating system"
444         nil
445     end
446 end
447
448 $hostOS = determineOS unless $hostOS
449 $architecture = determineArchitecture unless $architecture
450 $isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $architecture == "mips" || $hostOS == "windows")
451
452 if ["mips", "x86"].include?($architecture)
453     # The JIT is temporarily disabled on these platforms since
454     # https://trac.webkit.org/changeset/237547
455     $jitTests = false
456 end
457
458 def isFTLEnabled
459     $jitTests && $isFTLPlatform
460 end
461
462 if !$testRunnerType
463     if $remote and $hostOS == "darwin"
464         $testRunnerType = :shell
465     else
466         $testRunnerType = :make
467     end
468 end
469
470 if $remoteHosts.length > 1 and $testRunnerType != :make
471     raise "Multiple remote hosts only supported with make runner"
472 end
473
474 if $testWriter
475     if /[^-a-zA-Z0-9_]/.match($testWriter)
476         raise "Invalid test writer #{$testWriter} given"
477     end
478 end
479
480 $numFailures = 0
481 $numPasses = 0
482
483 # We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
484 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--validateExceptionChecks=true", "--useDollarVM=true", "--maxPerThreadStackUsage=1572864"]
485 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
486 # NOTE: Tests rely on this using scribbleFreeCells.
487 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
488 B3O1_OPTIONS = ["--defaultB3OptLevel=1"]
489 FTL_OPTIONS = ["--useFTLJIT=true"]
490 PROBE_OSR_EXIT_OPTION = ["--useProbeOSRExit=true"]
491
492 require_relative "webkitruby/jsc-stress-test-writer-#{$testWriter}"
493
494 def shouldCollectContinuously?
495     $buildType == "release" or $forceCollectContinuously
496 end
497
498 COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false"] : []
499
500 $runlist = []
501
502 def frameworkFromJSCPath(jscPath)
503     parentDirectory = jscPath.dirname
504     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
505         parentDirectory.dirname
506     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
507         jscPath.dirname + "JavaScriptCore.framework"
508     else
509         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
510         nil
511     end
512 end
513
514 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
515     dir = Pathname.new(".")
516     $benchmarkDirectory.each_filename {
517         | pathComponent |
518         dir += ".."
519     }
520     dir + resourcePath
521 end
522
523 def pathToVM
524     pathToBundleResourceFromBenchmarkDirectory($jscPath)
525 end
526
527 def pathToHelpers
528     pathToBundleResourceFromBenchmarkDirectory(".helpers")
529 end
530
531 $runCommandOptions = {}
532 $testSpecificRequiredOptions = []
533
534 $uniqueFilenameCounter = 0
535 def uniqueFilename(extension)
536     payloadDir = $outputDir + "_payload"
537     Dir.mkdir payloadDir unless payloadDir.directory?
538     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
539     $uniqueFilenameCounter += 1
540     result
541 end
542
543 def baseOutputName(kind)
544     "#{$collectionName}/#{$benchmark}.#{kind}"
545 end
546
547 def addRunCommand(kind, command, outputHandler, errorHandler, *additionalEnv)
548     $didAddRunCommand = true
549     name = baseOutputName(kind)
550     if $filter and name !~ $filter
551         return
552     end
553     plan = Plan.new(
554         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
555         errorHandler)
556     plan.additionalEnv.push(*additionalEnv)
557     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
558         $runlist.unshift plan
559     else
560         $runlist << plan
561     end
562 end
563
564 # Returns true if there were run commands found in the file ($benchmarkDirectory +
565 # $benchmark), in which case those run commands have already been executed. Otherwise
566 # returns false, in which case you're supposed to add your own run commands.
567 def parseRunCommands
568     oldDidAddRunCommand = $didAddRunCommand
569     $didAddRunCommand = false
570     $skipped = false
571
572     Dir.chdir($outputDir) {
573         File.open($benchmarkDirectory + $benchmark) {
574             | inp |
575             inp.each_line {
576                 | line |
577                 begin
578                     doesMatch = line =~ /^\/\/@/
579                 rescue Exception => e
580                     # Apparently this happens in the case of some UTF8 stuff in some files, where
581                     # Ruby tries to be strict and throw exceptions.
582                     next
583                 end
584                 next unless doesMatch
585                 eval $~.post_match
586                 if $skipped
587                     break
588                 end
589             }
590         }
591     }
592
593     result = $didAddRunCommand
594     $didAddRunCommand = result or oldDidAddRunCommand
595     result
596 end
597
598 def slow!
599     $runCommandOptions[:isSlow] = true
600 end
601
602 def requireOptions(*options)
603     $testSpecificRequiredOptions += options
604 end
605
606 def runWithOutputHandler(kind, outputHandler, *options)
607     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + $testSpecificRequiredOptions + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
608 end
609
610 def run(kind, *options)
611     runWithOutputHandler(kind, silentOutputHandler, *options)
612 end
613
614 def runNoFTL(*optionalTestSpecificOptions)
615     run("no-ftl", *optionalTestSpecificOptions)
616 end
617
618 def runWithRAMSize(size, *optionalTestSpecificOptions)
619     run("ram-size-#{size}", "--forceRAMSize=#{size}", *optionalTestSpecificOptions)
620 end
621
622 def runOneLargeHeap(*optionalTestSpecificOptions)
623     if $memoryLimited
624         $didAddRunCommand = true
625         puts "Skipping #{$collectionName}/#{$benchmark}"
626     else
627         run("default", *optionalTestSpecificOptions)
628     end
629 end
630
631 def runNoJIT(*optionalTestSpecificOptions)
632     run("no-jit", "--useJIT=false", *optionalTestSpecificOptions)
633 end
634
635 def runNoLLInt(*optionalTestSpecificOptions)
636     if $jitTests
637         run("no-llint", "--useLLInt=false", *optionalTestSpecificOptions)
638     end
639 end
640
641 # NOTE: Tests rely on this using scribbleFreeCells.
642 def runNoCJITValidate(*optionalTestSpecificOptions)
643     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
644 end
645
646 def runNoCJITValidatePhases(*optionalTestSpecificOptions)
647     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
648 end
649
650 def runNoCJITCollectContinuously(*optionalTestSpecificOptions)
651     run("no-cjit-collect-continuously", *(NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
652 end
653
654 def runDefault(*optionalTestSpecificOptions)
655     run("default", *(FTL_OPTIONS + optionalTestSpecificOptions))
656 end
657
658 def runBytecodeCache(*optionalTestSpecificOptions)
659     unless $hostOS == "darwin"
660         skip
661         return
662     end
663     options = BASE_OPTIONS + $testSpecificRequiredOptions + FTL_OPTIONS + optionalTestSpecificOptions
664     addRunCommand("bytecode-cache", ["ruby", (pathToHelpers + "bytecode-cache-test-helper").to_s, pathToVM.to_s, $benchmark.to_s] + options, silentOutputHandler, simpleErrorHandler)
665 end
666
667 def runBigIntEnabled(*optionalTestSpecificOptions)
668     # FIXME: <rdar://problem/40331121>
669     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin")
670         skip
671         return
672     end
673
674     run("big-int-enabled", "--useBigInt=true" , *(FTL_OPTIONS + optionalTestSpecificOptions))
675 end
676
677 def runFTLNoCJIT(*optionalTestSpecificOptions)
678     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
679 end
680
681 def runFTLNoCJITB3O1(*optionalTestSpecificOptions)
682     run("ftl-no-cjit-b3o1", "--useArrayAllocationProfiling=false", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
683 end
684
685 def runFTLNoCJITValidate(*optionalTestSpecificOptions)
686     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + PROBE_OSR_EXIT_OPTION + optionalTestSpecificOptions))
687 end
688
689 def runFTLNoCJITNoPutStackValidate(*optionalTestSpecificOptions)
690     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
691 end
692
693 def runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
694     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
695 end
696
697 def runFTLNoCJITOSRValidation(*optionalTestSpecificOptions)
698     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
699 end
700
701 def runDFGEager(*optionalTestSpecificOptions)
702     run("dfg-eager", *(EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + PROBE_OSR_EXIT_OPTION + optionalTestSpecificOptions))
703 end
704
705 def runDFGEagerNoCJITValidate(*optionalTestSpecificOptions)
706     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
707 end
708
709 def runFTLEager(*optionalTestSpecificOptions)
710     run("ftl-eager", "--airForceBriggsAllocator=true", "--forcePolyProto=true", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
711 end
712
713 def runFTLEagerWatchdog(*optionalTestSpecificOptions)
714     timeout = rand(100)
715     run("ftl-eager-watchdog-#{timeout}", "--watchdog=#{timeout}", "--watchdog-exception-ok", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
716 end
717
718 def runFTLEagerNoCJITValidate(*optionalTestSpecificOptions)
719     run("ftl-eager-no-cjit", "--validateGraph=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
720 end
721
722 def runFTLEagerNoCJITB3O1(*optionalTestSpecificOptions)
723     run("ftl-eager-no-cjit-b3o1", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
724 end
725
726 def runFTLEagerNoCJITOSRValidation(*optionalTestSpecificOptions)
727     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
728 end
729
730 def runNoCJITNoASO(*optionalTestSpecificOptions)
731     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
732 end
733
734 def runNoCJITNoAccessInlining(*optionalTestSpecificOptions)
735     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
736 end
737
738 def runFTLNoCJITNoAccessInlining(*optionalTestSpecificOptions)
739     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
740 end
741
742 def runFTLNoCJITSmallPool(*optionalTestSpecificOptions)
743     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
744 end
745
746 def runNoCJIT(*optionalTestSpecificOptions)
747     run("no-cjit", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
748 end
749
750 def runDFGMaximalFlushPhase(*optionalTestSpecificOptions)
751     run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
752 end
753
754 def runShadowChicken(*optionalTestSpecificOptions)
755     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true", *optionalTestSpecificOptions)
756 end
757
758 def defaultRun
759     if $mode == "quick"
760         defaultQuickRun
761     else
762         runDefault
763         runBytecodeCache
764         if $jitTests
765             runNoLLInt
766             runNoCJITValidatePhases
767             runNoCJITCollectContinuously if shouldCollectContinuously?
768             runDFGEager
769             if $mode != "basic"
770                 runDFGEagerNoCJITValidate
771                 runDFGMaximalFlushPhase
772             end
773
774             return if !$isFTLPlatform
775
776             runNoFTL
777             runFTLEager
778             runFTLEagerNoCJITValidate
779             runFTLNoCJITSmallPool
780
781             return if $mode == "basic"
782
783             runFTLNoCJITValidate
784             runFTLNoCJITB3O1
785             runFTLNoCJITNoPutStackValidate
786             runFTLNoCJITNoInlineValidate
787             runFTLEagerNoCJITB3O1
788         end
789     end
790 end
791
792 def defaultNoNoLLIntRun
793     if $mode == "quick"
794         defaultQuickRun
795     else
796         runDefault
797         if $jitTests
798             runNoCJITValidatePhases
799             runNoCJITCollectContinuously if shouldCollectContinuously?
800             runDFGEager
801             if $mode != "basic"
802                 runDFGEagerNoCJITValidate
803                 runDFGMaximalFlushPhase
804             end
805
806             return if !$isFTLPlatform
807
808             runNoFTL
809             runFTLNoCJITValidate
810             runFTLNoCJITSmallPool
811
812             return if $mode == "basic"
813
814             runFTLNoCJITB3O1
815             runFTLNoCJITNoPutStackValidate
816             runFTLNoCJITNoInlineValidate
817             runFTLEager
818             runFTLEagerNoCJITValidate
819         end
820     end
821 end
822
823 def defaultQuickRun
824     runDefault
825     if $jitTests
826         runNoCJITValidate
827
828         return if !$isFTLPlatform
829
830         runNoFTL
831         runFTLNoCJITValidate
832     end
833 end
834
835 def defaultSpotCheckNoMaximalFlush
836     defaultQuickRun
837     runNoCJITNoAccessInlining
838
839     return if !$isFTLPlatform
840
841     runFTLNoCJITOSRValidation
842     runFTLNoCJITNoAccessInlining
843     runFTLNoCJITB3O1
844 end
845
846 def defaultSpotCheck
847     defaultSpotCheckNoMaximalFlush
848     runDFGMaximalFlushPhase
849 end
850
851 # This is expected to not do eager runs because eager runs can have a lot of recompilations
852 # for reasons that don't arise in the real world. It's used for tests that assert convergence
853 # by counting recompilations.
854 def defaultNoEagerRun
855     runDefault
856     if $jitTests
857         runNoLLInt
858         runNoCJITValidatePhases
859         runNoCJITCollectContinuously if shouldCollectContinuously?
860
861         return if !$isFTLPlatform
862
863         runNoFTL
864         runFTLNoCJITValidate
865
866         return if $mode == "basic"
867
868         runFTLNoCJITNoInlineValidate
869         runFTLNoCJITB3O1
870     end
871 end
872
873 def defaultNoSamplingProfilerRun
874     runDefault
875     if $jitTests
876         runNoLLInt
877         runNoCJITValidatePhases
878         runNoCJITCollectContinuously if shouldCollectContinuously?
879         runDFGEager
880         runDFGEagerNoCJITValidate
881         runDFGMaximalFlushPhase
882
883         return if !$isFTLPlatform
884
885         runNoFTL
886         runFTLNoCJITNoPutStackValidate
887         runFTLNoCJITNoInlineValidate
888         runFTLEager
889         runFTLEagerNoCJITValidate
890         runFTLNoCJITSmallPool
891     end
892 end
893
894 def runProfiler
895     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
896         skip
897         return
898     end
899
900     profilerOutput = uniqueFilename(".json")
901     if $canRunDisplayProfilerOutput
902         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
903     else
904         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
905         run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
906     end
907 end
908
909 def runExceptionFuzz
910     subCommand = escapeAll([pathToVM.to_s, "--useDollarVM=true", $benchmark.to_s])
911     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
912 end
913
914 def runExecutableAllocationFuzz(name, *options)
915     subCommand = escapeAll([pathToVM.to_s, "--useDollarVM=true", $benchmark.to_s] + options)
916     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
917 end
918
919 def runTypeProfiler
920     if !$jitTests
921         return
922     end
923
924     run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
925     run("ftl-no-cjit-type-profiler-force-poly-proto", "--useTypeProfiler=true", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
926
927     return if !$isFTLPlatform
928
929     run("ftl-type-profiler-ftl-eager", "--useTypeProfiler=true", *(FTL_OPTIONS + EAGER_OPTIONS))
930 end
931
932 def runControlFlowProfiler
933     if !$jitTests
934         return
935     end
936
937     return if !$isFTLPlatform
938
939     run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
940 end
941
942 def runTest262(mode, exception, includeFiles, flags)
943     failsWithException = exception != "NoException"
944     isStrict = false
945     isModule = false
946     isAsync = false
947
948     flags.each {
949         | flag |
950         case flag
951         when :strict
952             isStrict = true
953         when :module
954             isModule = true
955         when :async
956             isAsync = true
957         else
958             raise "Invalid flag for runTest262, #{flag}"
959         end
960     }
961
962     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
963
964     args = [pathToVM.to_s] + BASE_OPTIONS + $testSpecificRequiredOptions
965     args << "--exception=" + exception if failsWithException
966     args << "--test262-async" if isAsync
967     args += includeFiles
968
969     case mode
970     when :normal
971         errorHandler = simpleErrorHandler
972         outputHandler = silentOutputHandler
973     when :fail
974         errorHandler = expectedFailErrorHandler
975         outputHandler = noisyOutputHandler
976     when :failDueToOutdatedOrBadTest
977         errorHandler = expectedFailErrorHandler
978         outputHandler = noisyOutputHandler
979     when :skip
980         return
981     else
982         raise "Invalid mode: #{mode}"
983     end
984
985     if isStrict
986         kind = "default-strict"
987         args << "--strict-file=#{$benchmark}"
988     else
989         kind = "default"
990         if isModule
991             args << "--module-file=#{$benchmark}"
992         else
993             args << $benchmark.to_s
994         end
995     end
996
997     addRunCommand(kind, args, outputHandler, errorHandler)
998 end
999
1000 def prepareTest262Fixture
1001     # This function is used to add the files used by Test262 modules tests.
1002     prepareExtraRelativeFiles([""], $collection)
1003 end
1004
1005 def runES6(mode)
1006     args = [pathToVM.to_s] + BASE_OPTIONS + $testSpecificRequiredOptions + [$benchmark.to_s]
1007     case mode
1008     when :normal
1009         errorHandler = simpleErrorHandler
1010     when :fail
1011         errorHandler = expectedFailErrorHandler
1012     when :failDueToOutdatedOrBadTest
1013         errorHandler = expectedFailErrorHandler
1014     when :skip
1015         return
1016     else
1017         raise "Invalid mode: #{mode}"
1018     end
1019     addRunCommand("default", args, noisyOutputHandler, errorHandler)
1020 end
1021
1022 def defaultRunModules(noLLInt: true)
1023     run("default-modules", "-m")
1024
1025     if !$jitTests
1026         return
1027     end
1028
1029     run("no-llint-modules", "-m", "--useLLInt=false") if noLLInt
1030     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
1031     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
1032     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1033
1034     return if !$isFTLPlatform
1035
1036     run("default-ftl-modules", "-m", *FTL_OPTIONS)
1037     run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1038     run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1039     run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
1040     run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1041     run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1042 end
1043
1044 def noNoLLIntRunModules
1045     defaultRunModules(noLLInt: false)
1046 end
1047
1048 def runWebAssembly
1049     return if !$jitTests
1050     return if !$isFTLPlatform
1051     run("default-wasm", "-m", *FTL_OPTIONS)
1052     if $mode != "quick"
1053         run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1054         run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1055         run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1056         run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1057         run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
1058     end
1059 end
1060
1061 def runWebAssemblySuite
1062     return if !$jitTests
1063     return if !$isFTLPlatform
1064     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1065     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1066     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1067     run("default-wasm", "-m", *FTL_OPTIONS)
1068     if $mode != "quick"
1069         run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1070         run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1071         run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1072         run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1073         run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
1074     end
1075 end
1076
1077 def runWebAssemblyEmscripten(mode)
1078     case mode
1079     when :skip
1080         return
1081     end
1082     return if !$jitTests
1083     return if !$isFTLPlatform
1084     wasm = $benchmark.to_s.sub! '.js', '.wasm'
1085     prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
1086     run("default-wasm", *FTL_OPTIONS)
1087     if $mode != "quick"
1088         run("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1089         run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1090         run("wasm-no-call-ic", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1091         run("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1092     end
1093 end
1094
1095 def runWebAssemblySpecTest(mode)
1096     case mode
1097     when :skip
1098         return
1099     end
1100     return if !$jitTests
1101     return if !$isFTLPlatform
1102     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1103
1104     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1105     prepareExtraRelativeFiles(modules.map { |f| "../../" + f }, $collection)
1106
1107     harness = Dir[WASMTESTS_PATH + "spec-harness/" + "*.js"].map { |f| File.basename(f) }
1108     prepareExtraRelativeFiles(harness.map { |f| "../../spec-harness/" + f }, $collection)
1109
1110     runWithOutputHandler("default-wasm", noisyOutputHandler, "../spec-harness.js", *FTL_OPTIONS)
1111     if $mode != "quick"
1112       runWithOutputHandler("wasm-no-cjit-yes-tls-context", noisyOutputHandler, "../spec-harness.js",  "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1113       runWithOutputHandler("wasm-eager-jettison", noisyOutputHandler, "../spec-harness.js", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1114       runWithOutputHandler("wasm-no-call-ic", noisyOutputHandler, "../spec-harness.js", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1115       runWithOutputHandler("wasm-no-tls-context", noisyOutputHandler, "../spec-harness.js", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1116     end
1117 end
1118
1119 def runWebAssemblyLowExecutableMemory(*optionalTestSpecificOptions)
1120     return if !$jitTests
1121     return if !$isFTLPlatform
1122     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1123     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1124     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1125     # Only let WebAssembly get executable memory.
1126     run("default-wasm", "--useConcurrentGC=0" , "--useConcurrentJIT=0", "--jitMemoryReservationSize=15000", "--useBaselineJIT=0", "--useDFGJIT=0", "--useFTLJIT=0", "-m")
1127 end
1128
1129 def runChakra(mode, exception, baselineFile, extraFiles)
1130     raise unless $benchmark.to_s =~ /\.js$/
1131     failsWithException = exception != "NoException"
1132     testName = $~.pre_match
1133
1134     prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
1135     prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)
1136
1137     args = [pathToVM.to_s] + BASE_OPTIONS + $testSpecificRequiredOptions
1138     args += FTL_OPTIONS if $isFTLPlatform
1139     args += EAGER_OPTIONS
1140     args << "--exception=" + exception if failsWithException
1141     args << "--dumpException" if failsWithException
1142     args += ["jsc-lib.js"]
1143
1144     case mode
1145     when :baseline
1146         prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
1147         errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
1148         outputHandler = noisyOutputHandler
1149     when :pass
1150         errorHandler = chakraPassFailErrorHandler
1151         outputHandler = noisyOutputHandler
1152     when :skipDueToOutdatedOrBadTest
1153         return
1154     when :skip
1155         return
1156     else
1157         raise "Invalid mode: #{mode}"
1158     end
1159
1160     kind = "default"
1161     args << $benchmark.to_s
1162
1163     addRunCommand(kind, args, outputHandler, errorHandler)
1164 end
1165
1166 def runLayoutTest(kind, *options)
1167     raise unless $benchmark.to_s =~ /\.js$/
1168     testName = $~.pre_match
1169     if kind
1170         kind = "layout-" + kind
1171     else
1172         kind = "layout"
1173     end
1174
1175     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1176     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1177
1178     args = [pathToVM.to_s] + BASE_OPTIONS + $testSpecificRequiredOptions + options +
1179         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1180          $benchmark.to_s,
1181          (Pathname.new("resources") + "standalone-post.js").to_s]
1182     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1183 end
1184
1185 def runLayoutTestNoFTL
1186     runLayoutTest("no-ftl")
1187 end
1188
1189 def runLayoutTestNoLLInt
1190     runLayoutTest("no-llint", "--useLLInt=false")
1191 end
1192
1193 def runLayoutTestNoCJIT
1194     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1195 end
1196
1197 def runLayoutTestDFGEagerNoCJIT
1198     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1199 end
1200
1201 def runLayoutTestDefault
1202     runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
1203 end
1204
1205 def runLayoutTestFTLNoCJIT
1206     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1207 end
1208
1209 def runLayoutTestFTLEagerNoCJIT
1210     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1211 end
1212
1213 def runLayoutTestFTLEagerNoCJITB3O1
1214     runLayoutTest("ftl-eager-no-cjit-b3o1", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS))
1215 end
1216
1217 def noFTLRunLayoutTest
1218     if !$jitTests
1219         return
1220     end
1221
1222     runLayoutTestNoLLInt
1223     runLayoutTestNoCJIT
1224     runLayoutTestDFGEagerNoCJIT
1225 end
1226
1227 def defaultQuickRunLayoutTest
1228     runLayoutTestDefault
1229     if $jitTests
1230         if $isFTLPlatform
1231             runLayoutTestNoFTL
1232             runLayoutTestFTLNoCJIT
1233             runLayoutTestFTLEagerNoCJIT
1234         else
1235             noFTLRunLayoutTest
1236         end
1237     end
1238 end
1239
1240 def defaultRunLayoutTest
1241     if $mode == "quick"
1242         defaultQuickRunLayoutTest
1243     else
1244         runLayoutTestDefault
1245         if $jitTests
1246             noFTLRunLayoutTest
1247
1248             return if !$isFTLPlatform
1249
1250             runLayoutTestNoFTL
1251             runLayoutTestFTLNoCJIT
1252             runLayoutTestFTLEagerNoCJIT
1253         end
1254     end
1255 end
1256
1257 def noEagerNoNoLLIntTestsRunLayoutTest
1258     runLayoutTestDefault
1259     if $jitTests
1260         runLayoutTestNoCJIT
1261
1262         return if !$isFTLPlatform
1263
1264         runLayoutTestNoFTL
1265         runLayoutTestFTLNoCJIT
1266     end
1267 end
1268
1269 def noNoLLIntRunLayoutTest
1270     runLayoutTestDefault
1271     if $jitTests
1272         runLayoutTestNoCJIT
1273         runLayoutTestDFGEagerNoCJIT
1274
1275         return if !$isFTLPlatform
1276
1277         runLayoutTestNoFTL
1278         runLayoutTestFTLNoCJIT
1279         runLayoutTestFTLEagerNoCJIT
1280     end
1281 end
1282
1283 def prepareExtraRelativeFiles(extraFiles, destination)
1284     Dir.chdir($outputDir) {
1285         extraFiles.each {
1286             | file |
1287             dest = destination + file
1288             FileUtils.mkdir_p(dest.dirname)
1289             FileUtils.cp $extraFilesBaseDir + file, dest
1290         }
1291     }
1292 end
1293
1294 def baseDirForCollection(collectionName)
1295     Pathname(".tests") + collectionName
1296 end
1297
1298 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1299     raise unless absoluteBase.absolute?
1300     Dir.chdir($outputDir) {
1301         collectionBaseDir = baseDirForCollection($collectionName)
1302         extraFiles.each {
1303             | file |
1304             destination = collectionBaseDir + file
1305             FileUtils.mkdir_p destination.dirname unless destination.directory?
1306             FileUtils.cp absoluteBase + file, destination
1307         }
1308     }
1309 end
1310
1311 def runMozillaTest(kind, mode, extraFiles, *options)
1312     if kind
1313         kind = "mozilla-" + kind
1314     else
1315         kind = "mozilla"
1316     end
1317     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1318     args = [pathToVM.to_s] + BASE_OPTIONS + $testSpecificRequiredOptions + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1319     case mode
1320     when :normal
1321         errorHandler = mozillaErrorHandler
1322     when :negative
1323         errorHandler = mozillaExit3ErrorHandler
1324     when :fail
1325         errorHandler = mozillaFailErrorHandler
1326     when :failDueToOutdatedOrBadTest
1327         errorHandler = mozillaFailErrorHandler
1328     when :skip
1329         return
1330     else
1331         raise "Invalid mode: #{mode}"
1332     end
1333     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1334 end
1335
1336 def runMozillaTestDefault(mode, *extraFiles)
1337     runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
1338 end
1339
1340 def runMozillaTestNoFTL(mode, *extraFiles)
1341     runMozillaTest("no-ftl", mode, extraFiles)
1342 end
1343
1344 def runMozillaTestLLInt(mode, *extraFiles)
1345     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1346 end
1347
1348 def runMozillaTestBaselineJIT(mode, *extraFiles)
1349     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1350 end
1351
1352 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1353     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1354 end
1355
1356 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1357     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1358 end
1359
1360 def defaultQuickRunMozillaTest(mode, *extraFiles)
1361     if $jitTests
1362         runMozillaTestDefault(mode, *extraFiles)
1363         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1364     else
1365         runMozillaTestNoFTL(mode, *extraFiles)
1366         if $jitTests
1367             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1368         end
1369     end
1370 end
1371
1372 def defaultRunMozillaTest(mode, *extraFiles)
1373     if $mode == "quick"
1374         defaultQuickRunMozillaTest(mode, *extraFiles)
1375     else
1376         runMozillaTestNoFTL(mode, *extraFiles)
1377         if $jitTests
1378             runMozillaTestLLInt(mode, *extraFiles)
1379             runMozillaTestBaselineJIT(mode, *extraFiles)
1380             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1381             runMozillaTestDefault(mode, *extraFiles)
1382             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
1383         end
1384     end
1385 end
1386
1387 def runNoisyTestImpl(kind, options, additionalEnv)
1388     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + $testSpecificRequiredOptions + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler, *additionalEnv)
1389 end
1390
1391 def runNoisyTest(kind, *options)
1392     runNoisyTestImpl(kind, options, [])
1393 end
1394
1395 def runNoisyTestWithEnv(kind, *additionalEnv)
1396     runNoisyTestImpl(kind, [], additionalEnv)
1397 end
1398
1399 def runNoisyTestDefault
1400     runNoisyTest("default", *FTL_OPTIONS)
1401 end
1402
1403 def runNoisyTestNoFTL
1404     runNoisyTest("no-ftl")
1405 end
1406
1407 def runNoisyTestNoCJIT
1408     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1409 end
1410
1411 def runNoisyTestNoCJITB3O1
1412     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS))
1413 end
1414
1415 def runNoisyTestEagerNoCJIT
1416     runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1417 end
1418
1419 def defaultRunNoisyTest
1420     runNoisyTestDefault
1421     if $jitTests and $isFTLPlatform
1422         runNoisyTestNoFTL
1423         runNoisyTestNoCJIT
1424         runNoisyTestNoCJITB3O1
1425         runNoisyTestEagerNoCJIT
1426     end
1427 end
1428
1429 def skip
1430     $didAddRunCommand = true
1431     $skipped = true
1432     puts "Skipping #{$collectionName}/#{$benchmark}"
1433 end
1434
1435 def allJSFiles(path)
1436     if path.file?
1437         [path]
1438     else
1439         result = []
1440         Dir.foreach(path) {
1441             | filename |
1442             next unless filename =~ /\.m?js$/
1443             next unless (path + filename).file?
1444             result << path + filename
1445         }
1446         result
1447     end
1448 end
1449
1450 def uniqueifyName(names, name)
1451     result = name.to_s
1452     toAdd = 1
1453     while names[result]
1454         result = "#{name}-#{toAdd}"
1455         toAdd += 1
1456     end
1457     names[result] = true
1458     result
1459 end
1460
1461 def simplifyCollectionName(collectionPath)
1462     outerDir = collectionPath.dirname
1463     name = collectionPath.basename
1464     lastName = name
1465     if collectionPath.directory?
1466         while lastName.to_s =~ /test/
1467             lastName = outerDir.basename
1468             name = lastName + name
1469             outerDir = outerDir.dirname
1470         end
1471     end
1472     uniqueifyName($collectionNames, name)
1473 end
1474
1475 def prepareCollection(name)
1476     FileUtils.mkdir_p $outputDir + name
1477
1478     absoluteCollection = $collection.realpath
1479
1480     Dir.chdir($outputDir) {
1481         bundleDir = baseDirForCollection(name)
1482
1483         # Create the proper directory structures.
1484         FileUtils.mkdir_p bundleDir
1485         if bundleDir.basename == $collection.basename
1486             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1487             $collection = bundleDir
1488         else
1489             FileUtils.cp_r absoluteCollection, bundleDir
1490             $collection = bundleDir + $collection.basename
1491         end
1492
1493         $extraFilesBaseDir = absoluteCollection
1494     }
1495 end
1496
1497 $collectionNames = {}
1498
1499 def handleCollectionFile(collection)
1500     collectionName = simplifyCollectionName(collection)
1501    
1502     paths = {}
1503     subCollections = []
1504     YAML::load(IO::read(collection)).each {
1505         | entry |
1506         if entry["collection"]
1507             subCollections << entry["collection"]
1508             next
1509         end
1510         
1511         if Pathname.new(entry["path"]).absolute?
1512             raise "Absolute path: " + entry["path"] + " in #{collection}"
1513         end
1514         
1515         if paths[entry["path"]]
1516             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1517         end
1518         
1519         subCollection = collection.dirname + entry["path"]
1520         
1521         if subCollection.file?
1522             subCollectionName = Pathname.new(entry["path"]).dirname
1523         else
1524             subCollectionName = entry["path"]
1525         end
1526         
1527         $collection = subCollection
1528         $collectionName = Pathname.new(collectionName)
1529         Pathname.new(subCollectionName).each_filename {
1530             | filename |
1531             next if filename =~ /^\./
1532             $collectionName += filename
1533         }
1534         $collectionName = $collectionName.to_s
1535         
1536         prepareCollection($collectionName)
1537       
1538         Dir.chdir($outputDir) {
1539             pathsToSearch = [$collection]
1540             if entry["tests"]
1541                 if entry["tests"].is_a? Array
1542                     pathsToSearch = entry["tests"].map {
1543                         | testName |
1544                         pathsToSearch[0] + testName
1545                     }
1546                 else
1547                     pathsToSearch[0] += entry["tests"]
1548                 end
1549             end
1550             pathsToSearch.each {
1551                 | pathToSearch |
1552                 allJSFiles(pathToSearch).each {
1553                     | path |
1554                     
1555                     $benchmark = path.basename
1556                     $benchmarkDirectory = path.dirname
1557                     
1558                     $runCommandOptions = {}
1559                     eval entry["cmd"]
1560                 }
1561             }
1562         }
1563     }
1564     
1565     subCollections.each {
1566         | subCollection |
1567         handleCollection(collection.dirname + subCollection)
1568     }
1569 end
1570
1571 def handleCollectionDirectory(collection)
1572     collectionName = simplifyCollectionName(collection)
1573     
1574     $collection = collection
1575     $collectionName = collectionName
1576     prepareCollection(collectionName)
1577    
1578     Dir.chdir($outputDir) {
1579         $benchmarkDirectory = $collection
1580         allJSFiles($collection).each {
1581             | path |
1582             
1583             $benchmark = path.basename
1584             
1585             $runCommandOptions = {}
1586             $testSpecificRequiredOptions = []
1587             defaultRun unless parseRunCommands
1588         }
1589     }
1590 end
1591
1592 def handleCollection(collection)
1593     collection = Pathname.new(collection)
1594     
1595     if collection.file?
1596         handleCollectionFile(collection)
1597     else
1598         handleCollectionDirectory(collection)
1599     end
1600 end
1601
1602 def appendFailure(plan)
1603     File.open($outputDir + "failed", "a") {
1604         | outp |
1605         outp.puts plan.name
1606     }
1607     $numFailures += 1
1608 end
1609
1610 def appendPass(plan)
1611     File.open($outputDir + "passed", "a") {
1612         | outp |
1613         outp.puts plan.name
1614     }
1615     $numPasses += 1
1616 end
1617
1618 def appendResult(plan, didPass)
1619     File.open($outputDir + "results", "a") {
1620         | outp |
1621         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1622     }
1623 end
1624
1625 def prepareBundle
1626     raise if $bundle
1627
1628     if $doNotMessWithVMPath
1629         if !$remote and !$tarball
1630             $testingFrameworkPath = (frameworkFromJSCPath($jscPath) || $jscPath.dirname).realpath
1631             $jscPath = Pathname.new($jscPath).realpath
1632         else
1633             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1634         end
1635     else
1636         originalJSCPath = $jscPath
1637         vmDir = $outputDir + ".vm"
1638         FileUtils.mkdir_p vmDir
1639         
1640         frameworkPath = frameworkFromJSCPath($jscPath)
1641         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1642         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1643         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1644
1645         if frameworkPath
1646             source = frameworkPath
1647             destination = Pathname.new(".vm")
1648         elsif $hostOS == "windows"
1649             # Make sure to copy dll along with jsc on Windows
1650             originalJSCDir = File.dirname(originalJSCPath)
1651             source = [originalJSCPath] + [originalJSCDir + "/jscLib.dll"]
1652
1653             # Check for and copy JavaScriptCore.dll and WTF.dll for dynamic builds
1654             javaScriptCoreDLLPath = File.join(originalJSCDir, "JavaScriptCore.dll")
1655             wtfDLLPath = File.join(originalJSCDir, "WTF.dll")
1656             if (File.exists?(javaScriptCoreDLLPath))
1657                 source = source + [javaScriptCoreDLLPath]
1658             end
1659             if (File.exists?(wtfDLLPath))
1660                 source = source + [wtfDLLPath]
1661             end
1662
1663             destination = $jscPath.dirname
1664
1665             Dir.chdir($outputDir) {
1666                 FileUtils.mkdir_p destination
1667             }
1668         else
1669             source = originalJSCPath
1670             destination = $jscPath
1671
1672             Dir.chdir($outputDir) {
1673                 FileUtils.mkdir_p $jscPath.dirname
1674             }
1675         end
1676
1677         Dir.chdir($outputDir) {
1678             if $copyVM
1679                 FileUtils.cp_r source, destination
1680             else
1681                 begin 
1682                     FileUtils.ln_s source, destination
1683                 rescue Exception
1684                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1685                     FileUtils.cp_r source, destination
1686                 end
1687             end
1688
1689             if $remote and $hostOS == "linux"
1690                 begin
1691                     dependencies = `ldd #{source}`
1692                     dependencies.split(/\n/).each {
1693                         | dependency |
1694                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1695                     }
1696                 rescue
1697                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1698                 end
1699             end
1700         }
1701     end
1702     
1703     Dir.chdir($outputDir) {
1704         FileUtils.cp_r HELPERS_PATH, ".helpers"
1705     }
1706
1707     ARGV.each {
1708         | collection |
1709         handleCollection(collection)
1710     }
1711
1712     puts
1713 end
1714
1715 def cleanOldResults
1716     raise unless $bundle
1717
1718     eachResultFile($outputDir) {
1719         | path |
1720         FileUtils.rm_f path
1721     }
1722 end
1723
1724 def cleanEmptyResultFiles
1725     eachResultFile($outputDir) {
1726         | path |
1727         next unless path.basename.to_s =~ /\.out$/
1728         next unless FileTest.size(path) == 0
1729         FileUtils.rm_f path
1730     }
1731 end
1732
1733 def eachResultFile(startingDir, &block)
1734     dirsToClean = [startingDir]
1735     until dirsToClean.empty?
1736         nextDir = dirsToClean.pop
1737         Dir.foreach(nextDir) {
1738             | entry |
1739             next if entry =~ /^\./
1740             path = nextDir + entry
1741             if path.directory?
1742                 dirsToClean.push(path)
1743             else
1744                 block.call(path)
1745             end
1746         }
1747     end
1748 end
1749
1750 def prepareTestRunner(remoteIndex=0)
1751     raise if $bundle
1752
1753     $runlist.each_with_index {
1754         | plan, index |
1755         plan.index = index
1756     }
1757
1758     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1759     toDelete = []
1760     Dir.foreach($runnerDir) {
1761         | filename |
1762         if filename =~ /^test_/
1763             toDelete << filename
1764         end
1765     }
1766     
1767     toDelete.each {
1768         | filename |
1769         File.unlink($runnerDir + filename)
1770     }
1771
1772     $runlist.each {
1773         | plan |
1774         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1775     }
1776
1777     case $testRunnerType
1778     when :make
1779         prepareMakeTestRunner(remoteIndex)
1780     when :shell
1781         prepareShellTestRunner
1782     when :ruby
1783         prepareRubyTestRunner
1784     else
1785         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1786     end
1787 end
1788
1789 def cleanRunnerDirectory
1790     raise unless $bundle
1791     Dir.foreach($runnerDir) {
1792         | filename |
1793         next unless filename =~ /^test_fail/
1794         FileUtils.rm_f $runnerDir + filename
1795     }
1796 end
1797
1798 def sshRead(cmd, remoteIndex=0)
1799     raise unless $remote
1800
1801     remoteHost = $remoteHosts[remoteIndex]
1802
1803     result = ""
1804     IO.popen("ssh -o NoHostAuthenticationForLocalhost=yes -p #{remoteHost.port} #{remoteHost.user}@#{remoteHost.host} '#{cmd}'", "r") {
1805       | inp |
1806       inp.each_line {
1807         | line |
1808         result += line
1809       }
1810     }
1811     raise "#{$?}" unless $?.success?
1812     result
1813 end
1814
1815 def runCommandOnTester(cmd)
1816     if $remote
1817         result = sshRead(cmd)
1818     else
1819         result = `#{cmd}`
1820     end
1821 end
1822
1823 def numberOfProcessors
1824     if $hostOS == "windows"
1825         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1826     else
1827         begin
1828             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1829         rescue
1830             numProcessors = 0
1831         end
1832
1833         if numProcessors == 0
1834             begin
1835                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1836             rescue
1837                 numProcessors == 0
1838             end
1839         end
1840     end
1841
1842     if numProcessors == 0
1843         numProcessors = 1
1844     end
1845     return numProcessors
1846 end
1847
1848 def runAndMonitorTestRunnerCommand(*cmd)
1849     numberOfTests = 0
1850     Dir.chdir($runnerDir) {
1851         # -1 for the runscript, and -2 for '..' and '.'
1852         numberOfTests = Dir.entries(".").count - 3
1853     }
1854     unless $progressMeter
1855         mysys(cmd.join(' '))
1856     else
1857        running = {}
1858        didRun = {}
1859        didFail = {}
1860        blankLine = true
1861        prevStringLength = 0
1862        IO.popen(cmd.join(' '), mode="r") {
1863            | inp |
1864            inp.each_line {
1865                | line |
1866                line = line.scrub.chomp
1867                if line =~ /^Running /
1868                    running[$~.post_match] = true
1869                elsif line =~ /^PASS: /
1870                    didRun[$~.post_match] = true
1871                elsif line =~ /^FAIL: /
1872                    didRun[$~.post_match] = true
1873                    didFail[$~.post_match] = true
1874                else
1875                    unless blankLine
1876                        print("\r" + " " * prevStringLength + "\r")
1877                    end
1878                    puts line
1879                    blankLine = true
1880                end
1881
1882                def lpad(str, chars)
1883                    str = str.to_s
1884                    if str.length > chars
1885                        str
1886                    else
1887                       "%#{chars}s"%(str)
1888                    end
1889                end
1890
1891                string  = ""
1892                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1893                unless didFail.empty?
1894                    string += " (failed #{didFail.size})"
1895                end
1896                string += " "
1897                (running.size - didRun.size).times {
1898                    string += "."
1899                }
1900                if string.length < prevStringLength
1901                    print string
1902                    print(" " * (prevStringLength - string.length))
1903                end
1904                print string
1905                prevStringLength = string.length
1906                blankLine = false
1907                $stdout.flush
1908            }
1909        }
1910        puts
1911        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1912     end
1913 end
1914
1915 def runTestRunner(remoteIndex=0)
1916     if $remote
1917         remoteHost = $remoteHosts[remoteIndex]
1918         if !remoteHost.remoteDirectory
1919             remoteHost.remoteDirectory = JSON::parse(sshRead("cat ~/.bencher", remoteIndex))["tempPath"]
1920         end
1921         mysys("ssh", "-o", "NoHostAuthenticationForLocalhost=yes", "-p", remoteHost.port.to_s, "#{remoteHost.user}@#{remoteHost.host}", "mkdir -p #{remoteHost.remoteDirectory}")
1922         mysys("scp", "-o", "NoHostAuthenticationForLocalhost=yes", "-P", remoteHost.port.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{remoteHost.user}@#{remoteHost.host}:#{remoteHost.remoteDirectory}")
1923         remoteScript = "\""
1924         remoteScript += "cd #{remoteHost.remoteDirectory} && "
1925         remoteScript += "rm -rf #{$outputDir.basename} && "
1926         remoteScript += "tar xzf #{$tarFileName} && "
1927         remoteScript += "cd #{$outputDir.basename}/.runner && "
1928         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1929         remoteScript += "export LD_LIBRARY_PATH=#{remoteHost.remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1930         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
1931         $envVars.each { |var| remoteScript += "export " << var << "\n" }
1932         remoteScript += "#{testRunnerCommand(remoteIndex)}\""
1933         runAndMonitorTestRunnerCommand("ssh", "-o", "NoHostAuthenticationForLocalhost=yes", "-p", remoteHost.port.to_s, "#{remoteHost.user}@#{remoteHost.host}", remoteScript)
1934     else
1935         Dir.chdir($runnerDir) {
1936             runAndMonitorTestRunnerCommand(testRunnerCommand)
1937         }
1938     end
1939 end
1940
1941 def detectFailures
1942     raise if $bundle
1943
1944     failures = []
1945     
1946     if $remote
1947         $remoteHosts.each_with_index {
1948             | host, remoteIndex |
1949             output = sshRead("cd #{host.remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"", remoteIndex)
1950             output.split(/\n/).each {
1951                 | line |
1952                 next unless line =~ /test_fail_/
1953                 failures << $~.post_match.to_i
1954             }
1955         }
1956     else
1957         Dir.foreach($runnerDir) {
1958             | filename |
1959             next unless filename =~ /test_fail_/
1960             failures << $~.post_match.to_i
1961         }
1962     end
1963
1964     failureSet = {}
1965
1966     failures.each {
1967         | failure | 
1968         appendFailure($runlist[failure])
1969         failureSet[failure] = true
1970     }
1971
1972     familyMap = {}
1973     $runlist.each_with_index {
1974         | plan, index |
1975         unless familyMap[plan.family]
1976             familyMap[plan.family] = []
1977         end
1978         if failureSet[index]
1979             appendResult(plan, false)
1980             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
1981             next
1982         else
1983             appendResult(plan, true)
1984             familyMap[plan.family] << {:result => "PASS", :plan => plan};
1985         end
1986         appendPass(plan)
1987     }
1988
1989     File.open($outputDir + "resultsByFamily", "w") {
1990         | outp |
1991         first = true
1992         familyMap.keys.sort.each {
1993             | familyName |
1994             if first
1995                 first = false
1996             else
1997                 outp.puts
1998             end
1999             
2000             outp.print "#{familyName}:"
2001
2002             numPassed = 0
2003             familyMap[familyName].each {
2004                 | entry |
2005                 if entry[:result] == "PASS"
2006                     numPassed += 1
2007                 end
2008             }
2009
2010             if numPassed == familyMap[familyName].size
2011                 outp.puts " PASSED"
2012             elsif numPassed == 0
2013                 outp.puts " FAILED"
2014             else
2015                 outp.puts
2016                 familyMap[familyName].each {
2017                     | entry |
2018                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
2019                 }
2020             end
2021         }
2022     }
2023 end
2024
2025 def compressBundle
2026     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
2027     $stderr.puts ">> #{cmd}" if $verbosity >= 2
2028     raise unless system(cmd)
2029 end
2030
2031 def clean(file)
2032     FileUtils.rm_rf file unless $bundle
2033 end
2034
2035 clean($outputDir + "failed")
2036 clean($outputDir + "passed")
2037 clean($outputDir + "results")
2038 clean($outputDir + "resultsByFamily")
2039 clean($outputDir + ".vm")
2040 clean($outputDir + ".helpers")
2041 clean($outputDir + ".runner")
2042 clean($outputDir + ".tests")
2043 clean($outputDir + "_payload")
2044
2045 Dir.mkdir($outputDir) unless $outputDir.directory?
2046
2047 $outputDir = $outputDir.realpath
2048 $runnerDir = $outputDir + ".runner"
2049
2050 if !$numChildProcesses
2051     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
2052         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
2053     else
2054         $numChildProcesses = numberOfProcessors
2055     end
2056 end
2057
2058 if ENV["JSCTEST_timeout"]
2059     # In the worst case, the processors just interfere with each other.
2060     # Increase the timeout proportionally to the number of processors.
2061     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
2062 end
2063
2064 def runBundle
2065     raise unless $bundle
2066
2067     cleanRunnerDirectory
2068     cleanOldResults
2069     runTestRunner
2070     cleanEmptyResultFiles
2071 end
2072
2073 def runNormal
2074     raise if $bundle or $tarball
2075
2076     prepareBundle
2077     prepareTestRunner
2078     runTestRunner
2079     cleanEmptyResultFiles
2080     detectFailures
2081 end
2082
2083 def runTarball
2084     raise unless $tarball
2085
2086     prepareBundle 
2087     prepareTestRunner
2088     compressBundle
2089 end
2090
2091 def runRemote
2092     raise unless $remote
2093
2094     prepareBundle
2095     $remoteHosts.each_index {
2096         | index |
2097         prepareTestRunner(index)
2098     }
2099     compressBundle
2100     threads = []
2101     $remoteHosts.each_index {
2102         | index |
2103         threads << Thread.new {
2104             runTestRunner(index)
2105         }
2106     }
2107     threads.each { |thr| thr.join }
2108     detectFailures
2109 end
2110
2111 puts
2112 if $bundle
2113     runBundle
2114 elsif $remote
2115     runRemote
2116 elsif $tarball
2117     runTarball
2118 else
2119     runNormal
2120 end