Enable gigacage on iOS
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2016 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
47 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
48 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
49 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
50 WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
51 CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
52 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
53 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
54
55 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
56
57 begin
58     require 'shellwords'
59 rescue Exception => e
60     $stderr.puts "Warning: did not find shellwords, not running any tests."
61     exit 0
62 end
63
64 $canRunDisplayProfilerOutput = false
65
66 begin
67     require 'rubygems'
68     require 'json'
69     require 'highline'
70     $canRunDisplayProfilerOutput = true
71 rescue Exception => e
72     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
73     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
74     $stderr.puts "Error: #{e.inspect}"
75 end
76
77 def printCommandArray(*cmd)
78     begin
79         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
80     rescue
81         commandArray = cmd.join(' ')
82     end
83     $stderr.puts ">> #{commandArray}"
84 end
85
86 def mysys(*cmd)
87     printCommandArray(*cmd) if $verbosity >= 1
88     raise "Command failed: #{$?.inspect}" unless system(*cmd)
89 end
90
91 def escapeAll(array)
92     array.map {
93         | v |
94         raise "Detected a non-string in #{inspect}" unless v.is_a? String
95         Shellwords.shellescape(v)
96     }.join(' ')
97 end
98
99
100 $jscPath = nil
101 $doNotMessWithVMPath = false
102 $jitTests = true
103 $memoryLimited = false
104 $outputDir = Pathname.new("results")
105 $verbosity = 0
106 $bundle = nil
107 $tarball = false
108 $tarFileName = "payload.tar.gz"
109 $copyVM = false
110 $testRunnerType = nil
111 $testWriter = "default"
112 $remoteUser = nil
113 $remoteHost = nil
114 $remotePort = nil
115 $remoteDirectory = nil
116 $architecture = nil
117 $hostOS = nil
118 $filter = nil
119 $envVars = []
120 $mode = "full"
121 $buildType = "release"
122 $forceCollectContinuously = false
123
124 def usage
125     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
126     puts
127     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
128     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
129     puts "                            --jsc specifies an already present JavaScriptCore to test."
130     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
131     puts "                            Skip tests tagged with //@skip if $memoryLimited"
132     puts "--no-jit                    Do not run JIT specific tests."
133     puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
134     puts "                            platform."
135     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
136     puts "--verbose            (-v)   Print more things while running."
137     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
138     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
139     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
140     puts "                            e.g. x86, x86_64, arm."
141     puts "--os                        Specify os instead of determining from JavaScriptCore build."
142     puts "                            e.g. darwin, linux & windows."
143     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
144     puts "                            In general the shell runner is slower than the make runner."
145     puts "--make-runner               Uses the faster make-based runner."
146     puts "--ruby-runner               Uses the ruby runner for machines without unix shell or make."
147     puts "--test-writer [writer]      Specifies the test script format."
148     puts "                            default is to use shell scripts to run the tests"
149     puts "                            \"ruby\" to use ruby scripts for systems without a unix shell."
150     puts "--remote                    Specify a remote host on which to run tests from command line argument."
151     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
152     puts "--child-processes    (-c)   Specify the number of child processes."
153     puts "--filter                    Only run tests whose name matches the given regular expression."
154     puts "--help               (-h)   Print this message."
155     puts "--env-vars                  Add a list of environment variables to set before running jsc."
156     puts "                            Each environment variable should be separated by a space."
157     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
158     puts "                            it will override the default value."
159     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
160     puts "--basic                     Run with default and these additional modes: no-llint,"
161     puts "                            no-cjit-validate-phases, no-cjit-collect-continuously, dfg-eager"
162     puts "                            and for FTL platforms: no-ftl, ftl-eager-no-cjit and"
163     puts "                            ftl-no-cjit-small-pool."
164     exit 1
165 end
166
167 jscArg = nil
168
169 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
170                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
171                ['--no-copy', GetoptLong::NO_ARGUMENT],
172                ['--memory-limited', GetoptLong::NO_ARGUMENT],
173                ['--no-jit', GetoptLong::NO_ARGUMENT],
174                ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
175                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
176                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
177                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
178                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
179                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
180                ['--os', GetoptLong::REQUIRED_ARGUMENT],
181                ['--shell-runner', GetoptLong::NO_ARGUMENT],
182                ['--make-runner', GetoptLong::NO_ARGUMENT],
183                ['--ruby-runner', GetoptLong::NO_ARGUMENT],
184                ['--test-writer', GetoptLong::REQUIRED_ARGUMENT],
185                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
186                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
187                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
188                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
189                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
190                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
191                ['--debug', GetoptLong::NO_ARGUMENT],
192                ['--release', GetoptLong::NO_ARGUMENT],
193                ['--quick', '-q', GetoptLong::NO_ARGUMENT],
194                ['--basic', GetoptLong::NO_ARGUMENT]).each {
195     | opt, arg |
196     case opt
197     when '--help'
198         usage
199     when '--jsc'
200         jscArg = arg
201     when '--no-copy'
202         $doNotMessWithVMPath = true
203     when '--output-dir'
204         $outputDir = Pathname.new(arg)
205     when '--memory-limited'
206         $memoryLimited = true
207     when '--no-jit'
208         $jitTests = false
209     when '--force-collectContinuously'
210         $forceCollectContinuously = true;
211     when '--verbose'
212         $verbosity += 1
213     when '--run-bundle'
214         $bundle = Pathname.new(arg)
215     when '--tarball'
216         $tarball = true
217         $copyVM = true
218         $tarFileName = arg unless arg == ''
219     when '--force-vm-copy'
220         $copyVM = true
221     when '--shell-runner'
222         $testRunnerType = :shell
223     when '--make-runner'
224         $testRunnerType = :make
225     when '--ruby-runner'
226         $testRunnerType = :ruby
227     when '--test-writer'
228         $testWriter = arg
229     when '--remote'
230         $copyVM = true
231         $tarball = true
232         $remote = true
233         uri = URI("ssh://" + arg)
234         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
235     when '--remote-config-file'
236         $remoteConfigFile = arg
237     when '--child-processes'
238         $numChildProcesses = arg.to_i
239     when '--filter'
240         $filter = Regexp.new(arg)
241     when '--arch'
242         $architecture = arg
243     when '--os'
244         $hostOS = arg
245     when '--env-vars'
246         $envVars = arg.gsub(/\s+/, ' ').split(' ')
247     when '--quick'
248         $mode = "quick"
249     when '--basic'
250         $mode = "basic"
251     when '--debug'
252         $buildType = "debug"
253     when '--release'
254         $buildType = "release"
255     end
256 }
257
258 if $remoteConfigFile
259     file = File.read($remoteConfigFile)
260     config = JSON.parse(file)
261
262     if !$remote and config['remote']
263         $copyVM = true
264         $tarball = true
265         $remote = true
266         uri = URI("ssh://" + config['remote'])
267         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
268     end
269
270     if config['remoteDirectory']
271         $remoteDirectory = config['remoteDirectory']
272     end
273 end
274
275 unless jscArg
276     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
277     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
278     command += ($buildType == "release") ? " --release" : " --debug"
279     command += " --executablePath"
280
281     output = `#{command}`.split("\n")
282     if !output.length
283         $stderr.puts "Error: must specify --jsc <path>"
284         exit 1
285     end
286
287     output = output[0]
288     jscArg = Pathname.new(output).join("jsc")
289     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
290     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
291     if !File.file?(jscArg)
292         $stderr.puts "Error: must specify --jsc <path>"
293         exit 1
294     end
295
296     puts "Using the following jsc path: #{jscArg}"
297 end
298
299 if $doNotMessWithVMPath
300     $jscPath = Pathname.new(jscArg)
301 else
302     $jscPath = Pathname.new(jscArg).realpath
303 end
304
305 $progressMeter = ($verbosity == 0 and $stdout.tty?)
306
307 if $bundle
308     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
309     $outputDir = $bundle
310 end
311
312 # Try to determine architecture. Return nil on failure.
313 def machOArchitectureCode
314     begin 
315         otoolLines = `otool -afh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
316         otoolLines.each_with_index {
317             | value, index |
318             if value =~ /magic/ and value =~ /cputype/
319                 return otoolLines[index + 1].split[1].to_i
320             end
321         }
322     rescue
323         $stderr.puts "Warning: unable to execute otool."
324     end
325     $stderr.puts "Warning: unable to determine architecture."
326     nil
327 end
328
329 def determineArchitectureFromMachOBinary
330     code = machOArchitectureCode
331     return nil unless code
332     is64BitFlag = 0x01000000
333     case code
334     when 7
335         "x86"
336     when 7 | is64BitFlag
337         "x86-64"
338     when 12
339         "arm"
340     when 12 | is64BitFlag
341         "arm64"
342     else
343         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
344         nil
345     end
346 end
347
348 def determineArchitectureFromELFBinary
349     f = File.open($jscPath.to_s)
350     data = f.read(19)
351
352     if !(data[0,4] == "\x7F\x45\x4C\x46")
353         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
354         return nil
355     end
356
357     code = data[18].ord
358     case code
359     when 3
360         "x86"
361     when 8
362         "mips"
363     when 62
364         "x86-64"
365     when 40
366         "arm"
367     when 183
368         "arm64"
369     else
370         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
371         nil
372     end
373 end
374
375 def determineArchitectureFromPEBinary
376     f = File.open($jscPath.to_s)
377     data = f.read(1024)
378
379     if !(data[0, 2] == "MZ")
380         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
381         return nil
382     end
383
384     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
385
386     if !(data[peHeaderAddr, 4] == "PE\0\0")
387         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
388         return nil
389     end
390
391     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
392
393     case machine
394     when 0x014c
395         "x86"
396     when 0x8664
397         "x86-64"
398     else
399         $stderr.puts "Warning: unsupported machine type: #{machine}"
400         nil
401     end
402 end
403
404 def determineArchitecture
405     case $hostOS
406     when "darwin"
407         determineArchitectureFromMachOBinary
408     when "linux"
409         determineArchitectureFromELFBinary
410     when "windows"
411         determineArchitectureFromPEBinary
412     else
413         $stderr.puts "Warning: unable to determine architecture on this platform."
414         nil
415     end
416 end
417
418 def determineOS
419     case RbConfig::CONFIG["host_os"]
420     when /darwin/i
421         "darwin"
422     when /linux/i
423         "linux"
424     when /mswin|mingw|cygwin/
425         "windows"
426     else
427         $stderr.puts "Warning: unable to determine host operating system"
428         nil
429     end
430 end
431
432 $hostOS = determineOS unless $hostOS
433 $architecture = determineArchitecture unless $architecture
434 $isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $architecture == "mips" || $hostOS == "windows")
435
436 if !$testRunnerType
437     if $remote and $hostOS == "darwin"
438         $testRunnerType = :shell
439     else
440         $testRunnerType = :make
441     end
442 end
443
444 if $testWriter
445     if /[^-a-zA-Z0-9_]/.match($testWriter)
446         raise "Invalid test writer #{$testWriter} given"
447     end
448 end
449
450 $numFailures = 0
451 $numPasses = 0
452
453 # We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
454 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--validateExceptionChecks=true", "--maxPerThreadStackUsage=1572864"]
455 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
456 # NOTE: Tests rely on this using scribbleFreeCells.
457 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
458 B3O1_OPTIONS = ["--defaultB3OptLevel=1"]
459 FTL_OPTIONS = ["--useFTLJIT=true"]
460 PROBE_OSR_EXIT_OPTION = ["--useProbeOSRExit=true"]
461
462 require_relative "webkitruby/jsc-stress-test-writer-#{$testWriter}"
463
464 def shouldCollectContinuously?
465     $buildType == "release" or $forceCollectContinuously
466 end
467
468 COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false"] : []
469
470 $runlist = []
471
472 def frameworkFromJSCPath(jscPath)
473     parentDirectory = jscPath.dirname
474     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
475         parentDirectory.dirname
476     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
477         jscPath.dirname + "JavaScriptCore.framework"
478     else
479         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
480         nil
481     end
482 end
483
484 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
485     dir = Pathname.new(".")
486     $benchmarkDirectory.each_filename {
487         | pathComponent |
488         dir += ".."
489     }
490     dir + resourcePath
491 end
492
493 def pathToVM
494     pathToBundleResourceFromBenchmarkDirectory($jscPath)
495 end
496
497 def pathToHelpers
498     pathToBundleResourceFromBenchmarkDirectory(".helpers")
499 end
500
501 $runCommandOptions = {}
502
503 $uniqueFilenameCounter = 0
504 def uniqueFilename(extension)
505     payloadDir = $outputDir + "_payload"
506     Dir.mkdir payloadDir unless payloadDir.directory?
507     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
508     $uniqueFilenameCounter += 1
509     result
510 end
511
512 def baseOutputName(kind)
513     "#{$collectionName}/#{$benchmark}.#{kind}"
514 end
515
516 def addRunCommand(kind, command, outputHandler, errorHandler, *additionalEnv)
517     $didAddRunCommand = true
518     name = baseOutputName(kind)
519     if $filter and name !~ $filter
520         return
521     end
522     plan = Plan.new(
523         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
524         errorHandler)
525     plan.additionalEnv.push(*additionalEnv)
526     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
527         $runlist.unshift plan
528     else
529         $runlist << plan
530     end
531 end
532
533 # Returns true if there were run commands found in the file ($benchmarkDirectory +
534 # $benchmark), in which case those run commands have already been executed. Otherwise
535 # returns false, in which case you're supposed to add your own run commands.
536 def parseRunCommands
537     oldDidAddRunCommand = $didAddRunCommand
538     $didAddRunCommand = false
539
540     Dir.chdir($outputDir) {
541         File.open($benchmarkDirectory + $benchmark) {
542             | inp |
543             inp.each_line {
544                 | line |
545                 begin
546                     doesMatch = line =~ /^\/\/@/
547                 rescue Exception => e
548                     # Apparently this happens in the case of some UTF8 stuff in some files, where
549                     # Ruby tries to be strict and throw exceptions.
550                     next
551                 end
552                 next unless doesMatch
553                 eval $~.post_match
554             }
555         }
556     }
557
558     result = $didAddRunCommand
559     $didAddRunCommand = result or oldDidAddRunCommand
560     result
561 end
562
563 def slow!
564     $runCommandOptions[:isSlow] = true
565 end
566
567 def runWithOutputHandler(kind, outputHandler, *options)
568     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
569 end
570
571 def run(kind, *options)
572     runWithOutputHandler(kind, silentOutputHandler, *options)
573 end
574
575 def runNoFTL(*optionalTestSpecificOptions)
576     run("no-ftl", *optionalTestSpecificOptions)
577 end
578
579 def runWithRAMSize(size, *optionalTestSpecificOptions)
580     run("ram-size-#{size}", "--forceRAMSize=#{size}", *optionalTestSpecificOptions)
581 end
582
583 def runOneLargeHeap(*optionalTestSpecificOptions)
584     if $memoryLimited
585         $didAddRunCommand = true
586         puts "Skipping #{$collectionName}/#{$benchmark}"
587     else
588         run("default", *optionalTestSpecificOptions)
589     end
590 end
591
592 def runNoJIT(*optionalTestSpecificOptions)
593     run("no-jit", "--useJIT=false", *optionalTestSpecificOptions)
594 end
595
596 def runNoLLInt(*optionalTestSpecificOptions)
597     if $jitTests
598         run("no-llint", "--useLLInt=false", *optionalTestSpecificOptions)
599     end
600 end
601
602 # NOTE: Tests rely on this using scribbleFreeCells.
603 def runNoCJITValidate(*optionalTestSpecificOptions)
604     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
605 end
606
607 def runNoCJITValidatePhases(*optionalTestSpecificOptions)
608     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
609 end
610
611 def runNoCJITCollectContinuously(*optionalTestSpecificOptions)
612     run("no-cjit-collect-continuously", *(NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
613 end
614
615 def runDefault(*optionalTestSpecificOptions)
616     run("default", *(FTL_OPTIONS + optionalTestSpecificOptions))
617 end
618
619 def runFTLNoCJIT(*optionalTestSpecificOptions)
620     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
621 end
622
623 def runFTLNoCJITB3O1(*optionalTestSpecificOptions)
624     run("ftl-no-cjit-b3o1", "--useArrayAllocationProfiling=false", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
625 end
626
627 def runFTLNoCJITValidate(*optionalTestSpecificOptions)
628     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + PROBE_OSR_EXIT_OPTION + optionalTestSpecificOptions))
629 end
630
631 def runFTLNoCJITNoPutStackValidate(*optionalTestSpecificOptions)
632     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
633 end
634
635 def runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
636     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
637 end
638
639 def runFTLNoCJITOSRValidation(*optionalTestSpecificOptions)
640     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
641 end
642
643 def runDFGEager(*optionalTestSpecificOptions)
644     run("dfg-eager", *(EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + PROBE_OSR_EXIT_OPTION + optionalTestSpecificOptions))
645 end
646
647 def runDFGEagerNoCJITValidate(*optionalTestSpecificOptions)
648     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
649 end
650
651 def runFTLEager(*optionalTestSpecificOptions)
652     run("ftl-eager", "--airForceBriggsAllocator=true", "--forcePolyProto=true", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
653 end
654
655 def runFTLEagerWatchdog(*optionalTestSpecificOptions)
656     timeout = rand(100)
657     run("ftl-eager-watchdog-#{timeout}", "--watchdog=#{timeout}", "--watchdog-exception-ok", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
658 end
659
660 def runFTLEagerNoCJITValidate(*optionalTestSpecificOptions)
661     run("ftl-eager-no-cjit", "--validateGraph=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
662 end
663
664 def runFTLEagerNoCJITB3O1(*optionalTestSpecificOptions)
665     run("ftl-eager-no-cjit-b3o1", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
666 end
667
668 def runFTLEagerNoCJITOSRValidation(*optionalTestSpecificOptions)
669     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
670 end
671
672 def runNoCJITNoASO(*optionalTestSpecificOptions)
673     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
674 end
675
676 def runNoCJITNoAccessInlining(*optionalTestSpecificOptions)
677     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
678 end
679
680 def runFTLNoCJITNoAccessInlining(*optionalTestSpecificOptions)
681     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
682 end
683
684 def runFTLNoCJITSmallPool(*optionalTestSpecificOptions)
685     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
686 end
687
688 def runNoCJIT(*optionalTestSpecificOptions)
689     run("no-cjit", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
690 end
691
692 def runDFGMaximalFlushPhase(*optionalTestSpecificOptions)
693     run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
694 end
695
696 def runShadowChicken(*optionalTestSpecificOptions)
697     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true", *optionalTestSpecificOptions)
698 end
699
700 def defaultRun
701     if $mode == "quick"
702         defaultQuickRun
703     else
704         runDefault
705         if $jitTests
706             runNoLLInt
707             runNoCJITValidatePhases
708             runNoCJITCollectContinuously if shouldCollectContinuously?
709             runDFGEager
710             if $mode != "basic"
711                 runDFGEagerNoCJITValidate
712                 runDFGMaximalFlushPhase
713             end
714
715             return if !$isFTLPlatform
716
717             runNoFTL
718             runFTLEager
719             runFTLEagerNoCJITValidate
720             runFTLNoCJITSmallPool
721
722             return if $mode == "basic"
723
724             runFTLNoCJITValidate
725             runFTLNoCJITB3O1
726             runFTLNoCJITNoPutStackValidate
727             runFTLNoCJITNoInlineValidate
728             runFTLEagerNoCJITB3O1
729         end
730     end
731 end
732
733 def defaultNoNoLLIntRun
734     if $mode == "quick"
735         defaultQuickRun
736     else
737         runDefault
738         if $jitTests
739             runNoCJITValidatePhases
740             runNoCJITCollectContinuously if shouldCollectContinuously?
741             runDFGEager
742             if $mode != "basic"
743                 runDFGEagerNoCJITValidate
744                 runDFGMaximalFlushPhase
745             end
746
747             return if !$isFTLPlatform
748
749             runNoFTL
750             runFTLNoCJITValidate
751             runFTLNoCJITSmallPool
752
753             return if $mode == "basic"
754
755             runFTLNoCJITB3O1
756             runFTLNoCJITNoPutStackValidate
757             runFTLNoCJITNoInlineValidate
758             runFTLEager
759             runFTLEagerNoCJITValidate
760         end
761     end
762 end
763
764 def defaultQuickRun
765     runDefault
766     if $jitTests
767         runNoCJITValidate
768
769         return if !$isFTLPlatform
770
771         runNoFTL
772         runFTLNoCJITValidate
773     end
774 end
775
776 def defaultSpotCheckNoMaximalFlush
777     defaultQuickRun
778     runNoCJITNoAccessInlining
779
780     return if !$isFTLPlatform
781
782     runFTLNoCJITOSRValidation
783     runFTLNoCJITNoAccessInlining
784     runFTLNoCJITB3O1
785 end
786
787 def defaultSpotCheck
788     defaultSpotCheckNoMaximalFlush
789     runDFGMaximalFlushPhase
790 end
791
792 # This is expected to not do eager runs because eager runs can have a lot of recompilations
793 # for reasons that don't arise in the real world. It's used for tests that assert convergence
794 # by counting recompilations.
795 def defaultNoEagerRun
796     runDefault
797     if $jitTests
798         runNoLLInt
799         runNoCJITValidatePhases
800         runNoCJITCollectContinuously if shouldCollectContinuously?
801
802         return if !$isFTLPlatform
803
804         runNoFTL
805         runFTLNoCJITValidate
806
807         return if $mode == "basic"
808
809         runFTLNoCJITNoInlineValidate
810         runFTLNoCJITB3O1
811     end
812 end
813
814 def defaultNoSamplingProfilerRun
815     runDefault
816     if $jitTests
817         runNoLLInt
818         runNoCJITValidatePhases
819         runNoCJITCollectContinuously if shouldCollectContinuously?
820         runDFGEager
821         runDFGEagerNoCJITValidate
822         runDFGMaximalFlushPhase
823
824         return if !$isFTLPlatform
825
826         runNoFTL
827         runFTLNoCJITNoPutStackValidate
828         runFTLNoCJITNoInlineValidate
829         runFTLEager
830         runFTLEagerNoCJITValidate
831         runFTLNoCJITSmallPool
832     end
833 end
834
835 def runProfiler
836     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
837         skip
838         return
839     end
840
841     profilerOutput = uniqueFilename(".json")
842     if $canRunDisplayProfilerOutput
843         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
844     else
845         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
846         run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
847     end
848 end
849
850 def runExceptionFuzz
851     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
852     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
853 end
854
855 def runExecutableAllocationFuzz(name, *options)
856     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
857     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
858 end
859
860 def runTypeProfiler
861     if !$jitTests
862         return
863     end
864
865     run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
866     run("ftl-no-cjit-type-profiler-force-poly-proto", "--useTypeProfiler=true", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
867
868     return if !$isFTLPlatform
869
870     run("ftl-type-profiler-ftl-eager", "--useTypeProfiler=true", *(FTL_OPTIONS + EAGER_OPTIONS))
871 end
872
873 def runControlFlowProfiler
874     if !$jitTests
875         return
876     end
877
878     return if !$isFTLPlatform
879
880     run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
881 end
882
883 def runTest262(mode, exception, includeFiles, flags)
884     failsWithException = exception != "NoException"
885     isStrict = false
886     isModule = false
887     isAsync = false
888
889     flags.each {
890         | flag |
891         case flag
892         when :strict
893             isStrict = true
894         when :module
895             isModule = true
896         when :async
897             isAsync = true
898         else
899             raise "Invalid flag for runTest262, #{flag}"
900         end
901     }
902
903     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
904
905     args = [pathToVM.to_s] + BASE_OPTIONS
906     args << "--exception=" + exception if failsWithException
907     args << "--test262-async" if isAsync
908     args += includeFiles
909
910     case mode
911     when :normal
912         errorHandler = simpleErrorHandler
913         outputHandler = silentOutputHandler
914     when :fail
915         errorHandler = expectedFailErrorHandler
916         outputHandler = noisyOutputHandler
917     when :failDueToOutdatedOrBadTest
918         errorHandler = expectedFailErrorHandler
919         outputHandler = noisyOutputHandler
920     when :skip
921         return
922     else
923         raise "Invalid mode: #{mode}"
924     end
925
926     if isStrict
927         kind = "default-strict"
928         args << "--strict-file=#{$benchmark}"
929     else
930         kind = "default"
931         if isModule
932             args << "--module-file=#{$benchmark}"
933         else
934             args << $benchmark.to_s
935         end
936     end
937
938     addRunCommand(kind, args, outputHandler, errorHandler)
939 end
940
941 def prepareTest262Fixture
942     # This function is used to add the files used by Test262 modules tests.
943     prepareExtraRelativeFiles([""], $collection)
944 end
945
946 def runES6(mode)
947     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
948     case mode
949     when :normal
950         errorHandler = simpleErrorHandler
951     when :fail
952         errorHandler = expectedFailErrorHandler
953     when :failDueToOutdatedOrBadTest
954         errorHandler = expectedFailErrorHandler
955     when :skip
956         return
957     else
958         raise "Invalid mode: #{mode}"
959     end
960     addRunCommand("default", args, noisyOutputHandler, errorHandler)
961 end
962
963 def runModules
964     run("default-modules", "-m")
965
966     if !$jitTests
967         return
968     end
969
970     run("no-llint-modules", "-m", "--useLLInt=false")
971     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
972     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
973     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
974
975     return if !$isFTLPlatform
976
977     run("default-ftl-modules", "-m", *FTL_OPTIONS)
978     run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
979     run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
980     run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
981     run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
982     run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
983 end
984
985 def runWebAssembly
986     return if !$jitTests
987     return if !$isFTLPlatform
988     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
989     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
990     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
991     run("default-wasm", "-m", *FTL_OPTIONS)
992     if $mode != "quick"
993         run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
994         run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
995         run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
996         run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
997         run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
998     end
999 end
1000
1001 def runWebAssemblyEmscripten(mode)
1002     case mode
1003     when :skip
1004         return
1005     end
1006     return if !$jitTests
1007     return if !$isFTLPlatform
1008     wasm = $benchmark.to_s.sub! '.js', '.wasm'
1009     prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
1010     run("default-wasm", *FTL_OPTIONS)
1011     if $mode != "quick"
1012         run("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1013         run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1014         run("wasm-no-call-ic", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1015         run("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1016     end
1017 end
1018
1019 def runWebAssemblySpecTest(mode)
1020     case mode
1021     when :skip
1022         return
1023     end
1024     return if !$jitTests
1025     return if !$isFTLPlatform
1026     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1027
1028     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1029     prepareExtraRelativeFiles(modules.map { |f| "../../" + f }, $collection)
1030
1031     harness = Dir[WASMTESTS_PATH + "spec-harness/" + "*.js"].map { |f| File.basename(f) }
1032     prepareExtraRelativeFiles(harness.map { |f| "../../spec-harness/" + f }, $collection)
1033
1034     runWithOutputHandler("default-wasm", noisyOutputHandler, "../spec-harness.js", *FTL_OPTIONS)
1035     if $mode != "quick"
1036       runWithOutputHandler("wasm-no-cjit-yes-tls-context", noisyOutputHandler, "../spec-harness.js",  "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1037       runWithOutputHandler("wasm-eager-jettison", noisyOutputHandler, "../spec-harness.js", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1038       runWithOutputHandler("wasm-no-call-ic", noisyOutputHandler, "../spec-harness.js", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1039       runWithOutputHandler("wasm-no-tls-context", noisyOutputHandler, "../spec-harness.js", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1040     end
1041 end
1042
1043 def runWebAssemblyLowExecutableMemory(*optionalTestSpecificOptions)
1044     return if !$jitTests
1045     return if !$isFTLPlatform
1046     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1047     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1048     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1049     # Only let WebAssembly get executable memory.
1050     run("default-wasm", "--useConcurrentGC=0" , "--useConcurrentJIT=0", "--jitMemoryReservationSize=15000", "--useBaselineJIT=0", "--useDFGJIT=0", "--useFTLJIT=0", "-m")
1051 end
1052
1053 def runChakra(mode, exception, baselineFile, extraFiles)
1054     raise unless $benchmark.to_s =~ /\.js$/
1055     failsWithException = exception != "NoException"
1056     testName = $~.pre_match
1057
1058     prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
1059     prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)
1060
1061     args = [pathToVM.to_s] + BASE_OPTIONS
1062     args += FTL_OPTIONS if $isFTLPlatform
1063     args += EAGER_OPTIONS
1064     args << "--exception=" + exception if failsWithException
1065     args << "--dumpException" if failsWithException
1066     args += ["jsc-lib.js"]
1067
1068     case mode
1069     when :baseline
1070         prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
1071         errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
1072         outputHandler = noisyOutputHandler
1073     when :pass
1074         errorHandler = chakraPassFailErrorHandler
1075         outputHandler = noisyOutputHandler
1076     when :skipDueToOutdatedOrBadTest
1077         return
1078     when :skip
1079         return
1080     else
1081         raise "Invalid mode: #{mode}"
1082     end
1083
1084     kind = "default"
1085     args << $benchmark.to_s
1086
1087     addRunCommand(kind, args, outputHandler, errorHandler)
1088 end
1089
1090 def runLayoutTest(kind, *options)
1091     raise unless $benchmark.to_s =~ /\.js$/
1092     testName = $~.pre_match
1093     if kind
1094         kind = "layout-" + kind
1095     else
1096         kind = "layout"
1097     end
1098
1099     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1100     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1101
1102     args = [pathToVM.to_s] + BASE_OPTIONS + options +
1103         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1104          $benchmark.to_s,
1105          (Pathname.new("resources") + "standalone-post.js").to_s]
1106     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1107 end
1108
1109 def runLayoutTestNoFTL
1110     runLayoutTest("no-ftl")
1111 end
1112
1113 def runLayoutTestNoLLInt
1114     runLayoutTest("no-llint", "--useLLInt=false")
1115 end
1116
1117 def runLayoutTestNoCJIT
1118     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1119 end
1120
1121 def runLayoutTestDFGEagerNoCJIT
1122     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1123 end
1124
1125 def runLayoutTestDefault
1126     runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
1127 end
1128
1129 def runLayoutTestFTLNoCJIT
1130     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1131 end
1132
1133 def runLayoutTestFTLEagerNoCJIT
1134     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1135 end
1136
1137 def runLayoutTestFTLEagerNoCJITB3O1
1138     runLayoutTest("ftl-eager-no-cjit-b3o1", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS))
1139 end
1140
1141 def noFTLRunLayoutTest
1142     if !$jitTests
1143         return
1144     end
1145
1146     runLayoutTestNoLLInt
1147     runLayoutTestNoCJIT
1148     runLayoutTestDFGEagerNoCJIT
1149 end
1150
1151 def defaultQuickRunLayoutTest
1152     runLayoutTestDefault
1153     if $jitTests
1154         if $isFTLPlatform
1155             runLayoutTestNoFTL
1156             runLayoutTestFTLNoCJIT
1157             runLayoutTestFTLEagerNoCJIT
1158         else
1159             noFTLRunLayoutTest
1160         end
1161     end
1162 end
1163
1164 def defaultRunLayoutTest
1165     if $mode == "quick"
1166         defaultQuickRunLayoutTest
1167     else
1168         runLayoutTestDefault
1169         if $jitTests
1170             noFTLRunLayoutTest
1171
1172             return if !$isFTLPlatform
1173
1174             runLayoutTestNoFTL
1175             runLayoutTestFTLNoCJIT
1176             runLayoutTestFTLEagerNoCJIT
1177         end
1178     end
1179 end
1180
1181 def noEagerNoNoLLIntTestsRunLayoutTest
1182     runLayoutTestDefault
1183     if $jitTests
1184         runLayoutTestNoCJIT
1185
1186         return if !$isFTLPlatform
1187
1188         runLayoutTestNoFTL
1189         runLayoutTestFTLNoCJIT
1190     end
1191 end
1192
1193 def noNoLLIntRunLayoutTest
1194     runLayoutTestDefault
1195     if $jitTests
1196         runLayoutTestNoCJIT
1197         runLayoutTestDFGEagerNoCJIT
1198
1199         return if !$isFTLPlatform
1200
1201         runLayoutTestNoFTL
1202         runLayoutTestFTLNoCJIT
1203         runLayoutTestFTLEagerNoCJIT
1204     end
1205 end
1206
1207 def prepareExtraRelativeFiles(extraFiles, destination)
1208     Dir.chdir($outputDir) {
1209         extraFiles.each {
1210             | file |
1211             dest = destination + file
1212             FileUtils.mkdir_p(dest.dirname)
1213             FileUtils.cp $extraFilesBaseDir + file, dest
1214         }
1215     }
1216 end
1217
1218 def baseDirForCollection(collectionName)
1219     Pathname(".tests") + collectionName
1220 end
1221
1222 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1223     raise unless absoluteBase.absolute?
1224     Dir.chdir($outputDir) {
1225         collectionBaseDir = baseDirForCollection($collectionName)
1226         extraFiles.each {
1227             | file |
1228             destination = collectionBaseDir + file
1229             FileUtils.mkdir_p destination.dirname unless destination.directory?
1230             FileUtils.cp absoluteBase + file, destination
1231         }
1232     }
1233 end
1234
1235 def runMozillaTest(kind, mode, extraFiles, *options)
1236     if kind
1237         kind = "mozilla-" + kind
1238     else
1239         kind = "mozilla"
1240     end
1241     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1242     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1243     case mode
1244     when :normal
1245         errorHandler = mozillaErrorHandler
1246     when :negative
1247         errorHandler = mozillaExit3ErrorHandler
1248     when :fail
1249         errorHandler = mozillaFailErrorHandler
1250     when :failDueToOutdatedOrBadTest
1251         errorHandler = mozillaFailErrorHandler
1252     when :skip
1253         return
1254     else
1255         raise "Invalid mode: #{mode}"
1256     end
1257     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1258 end
1259
1260 def runMozillaTestDefault(mode, *extraFiles)
1261     runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
1262 end
1263
1264 def runMozillaTestNoFTL(mode, *extraFiles)
1265     runMozillaTest("no-ftl", mode, extraFiles)
1266 end
1267
1268 def runMozillaTestLLInt(mode, *extraFiles)
1269     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1270 end
1271
1272 def runMozillaTestBaselineJIT(mode, *extraFiles)
1273     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1274 end
1275
1276 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1277     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1278 end
1279
1280 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1281     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1282 end
1283
1284 def defaultQuickRunMozillaTest(mode, *extraFiles)
1285     if $jitTests
1286         runMozillaTestDefault(mode, *extraFiles)
1287         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1288     else
1289         runMozillaTestNoFTL(mode, *extraFiles)
1290         if $jitTests
1291             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1292         end
1293     end
1294 end
1295
1296 def defaultRunMozillaTest(mode, *extraFiles)
1297     if $mode == "quick"
1298         defaultQuickRunMozillaTest(mode, *extraFiles)
1299     else
1300         runMozillaTestNoFTL(mode, *extraFiles)
1301         if $jitTests
1302             runMozillaTestLLInt(mode, *extraFiles)
1303             runMozillaTestBaselineJIT(mode, *extraFiles)
1304             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1305             runMozillaTestDefault(mode, *extraFiles)
1306             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
1307         end
1308     end
1309 end
1310
1311 def runNoisyTestImpl(kind, options, additionalEnv)
1312     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler, *additionalEnv)
1313 end
1314
1315 def runNoisyTest(kind, *options)
1316     runNoisyTestImpl(kind, options, [])
1317 end
1318
1319 def runNoisyTestWithEnv(kind, *additionalEnv)
1320     runNoisyTestImpl(kind, [], additionalEnv)
1321 end
1322
1323 def runNoisyTestDefault
1324     runNoisyTest("default", *FTL_OPTIONS)
1325 end
1326
1327 def runNoisyTestNoFTL
1328     runNoisyTest("no-ftl")
1329 end
1330
1331 def runNoisyTestNoCJIT
1332     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1333 end
1334
1335 def runNoisyTestNoCJITB3O1
1336     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS))
1337 end
1338
1339 def runNoisyTestEagerNoCJIT
1340     runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1341 end
1342
1343 def defaultRunNoisyTest
1344     runNoisyTestDefault
1345     if $jitTests and $isFTLPlatform
1346         runNoisyTestNoFTL
1347         runNoisyTestNoCJIT
1348         runNoisyTestNoCJITB3O1
1349         runNoisyTestEagerNoCJIT
1350     end
1351 end
1352
1353 def skip
1354     $didAddRunCommand = true
1355     puts "Skipping #{$collectionName}/#{$benchmark}"
1356 end
1357
1358 def allJSFiles(path)
1359     if path.file?
1360         [path]
1361     else
1362         result = []
1363         Dir.foreach(path) {
1364             | filename |
1365             next unless filename =~ /\.js$/
1366             next unless (path + filename).file?
1367             result << path + filename
1368         }
1369         result
1370     end
1371 end
1372
1373 def uniqueifyName(names, name)
1374     result = name.to_s
1375     toAdd = 1
1376     while names[result]
1377         result = "#{name}-#{toAdd}"
1378         toAdd += 1
1379     end
1380     names[result] = true
1381     result
1382 end
1383
1384 def simplifyCollectionName(collectionPath)
1385     outerDir = collectionPath.dirname
1386     name = collectionPath.basename
1387     lastName = name
1388     if collectionPath.directory?
1389         while lastName.to_s =~ /test/
1390             lastName = outerDir.basename
1391             name = lastName + name
1392             outerDir = outerDir.dirname
1393         end
1394     end
1395     uniqueifyName($collectionNames, name)
1396 end
1397
1398 def prepareCollection(name)
1399     FileUtils.mkdir_p $outputDir + name
1400
1401     absoluteCollection = $collection.realpath
1402
1403     Dir.chdir($outputDir) {
1404         bundleDir = baseDirForCollection(name)
1405
1406         # Create the proper directory structures.
1407         FileUtils.mkdir_p bundleDir
1408         if bundleDir.basename == $collection.basename
1409             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1410             $collection = bundleDir
1411         else
1412             FileUtils.cp_r absoluteCollection, bundleDir
1413             $collection = bundleDir + $collection.basename
1414         end
1415
1416         $extraFilesBaseDir = absoluteCollection
1417     }
1418 end
1419
1420 $collectionNames = {}
1421
1422 def handleCollectionFile(collection)
1423     collectionName = simplifyCollectionName(collection)
1424    
1425     paths = {}
1426     subCollections = []
1427     YAML::load(IO::read(collection)).each {
1428         | entry |
1429         if entry["collection"]
1430             subCollections << entry["collection"]
1431             next
1432         end
1433         
1434         if Pathname.new(entry["path"]).absolute?
1435             raise "Absolute path: " + entry["path"] + " in #{collection}"
1436         end
1437         
1438         if paths[entry["path"]]
1439             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1440         end
1441         
1442         subCollection = collection.dirname + entry["path"]
1443         
1444         if subCollection.file?
1445             subCollectionName = Pathname.new(entry["path"]).dirname
1446         else
1447             subCollectionName = entry["path"]
1448         end
1449         
1450         $collection = subCollection
1451         $collectionName = Pathname.new(collectionName)
1452         Pathname.new(subCollectionName).each_filename {
1453             | filename |
1454             next if filename =~ /^\./
1455             $collectionName += filename
1456         }
1457         $collectionName = $collectionName.to_s
1458         
1459         prepareCollection($collectionName)
1460       
1461         Dir.chdir($outputDir) {
1462             pathsToSearch = [$collection]
1463             if entry["tests"]
1464                 if entry["tests"].is_a? Array
1465                     pathsToSearch = entry["tests"].map {
1466                         | testName |
1467                         pathsToSearch[0] + testName
1468                     }
1469                 else
1470                     pathsToSearch[0] += entry["tests"]
1471                 end
1472             end
1473             pathsToSearch.each {
1474                 | pathToSearch |
1475                 allJSFiles(pathToSearch).each {
1476                     | path |
1477                     
1478                     $benchmark = path.basename
1479                     $benchmarkDirectory = path.dirname
1480                     
1481                     $runCommandOptions = {}
1482                     eval entry["cmd"]
1483                 }
1484             }
1485         }
1486     }
1487     
1488     subCollections.each {
1489         | subCollection |
1490         handleCollection(collection.dirname + subCollection)
1491     }
1492 end
1493
1494 def handleCollectionDirectory(collection)
1495     collectionName = simplifyCollectionName(collection)
1496     
1497     $collection = collection
1498     $collectionName = collectionName
1499     prepareCollection(collectionName)
1500    
1501     Dir.chdir($outputDir) {
1502         $benchmarkDirectory = $collection
1503         allJSFiles($collection).each {
1504             | path |
1505             
1506             $benchmark = path.basename
1507             
1508             $runCommandOptions = {}
1509             defaultRun unless parseRunCommands
1510         }
1511     }
1512 end
1513
1514 def handleCollection(collection)
1515     collection = Pathname.new(collection)
1516     
1517     if collection.file?
1518         handleCollectionFile(collection)
1519     else
1520         handleCollectionDirectory(collection)
1521     end
1522 end
1523
1524 def appendFailure(plan)
1525     File.open($outputDir + "failed", "a") {
1526         | outp |
1527         outp.puts plan.name
1528     }
1529     $numFailures += 1
1530 end
1531
1532 def appendPass(plan)
1533     File.open($outputDir + "passed", "a") {
1534         | outp |
1535         outp.puts plan.name
1536     }
1537     $numPasses += 1
1538 end
1539
1540 def appendResult(plan, didPass)
1541     File.open($outputDir + "results", "a") {
1542         | outp |
1543         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1544     }
1545 end
1546
1547 def prepareBundle
1548     raise if $bundle
1549
1550     if $doNotMessWithVMPath
1551         if !$remote and !$tarball
1552             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1553             $jscPath = Pathname.new($jscPath).realpath
1554         else
1555             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1556         end
1557     else
1558         originalJSCPath = $jscPath
1559         vmDir = $outputDir + ".vm"
1560         FileUtils.mkdir_p vmDir
1561         
1562         frameworkPath = frameworkFromJSCPath($jscPath)
1563         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1564         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1565         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1566
1567         if frameworkPath
1568             source = frameworkPath
1569             destination = Pathname.new(".vm")
1570         else
1571             source = originalJSCPath
1572             destination = $jscPath
1573
1574             Dir.chdir($outputDir) {
1575                 FileUtils.mkdir_p $jscPath.dirname
1576             }
1577         end
1578
1579         Dir.chdir($outputDir) {
1580             if $copyVM
1581                 FileUtils.cp_r source, destination
1582             else
1583                 begin 
1584                     FileUtils.ln_s source, destination
1585                 rescue Exception
1586                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1587                     FileUtils.cp_r source, destination
1588                 end
1589             end
1590
1591             if $remote and $hostOS == "linux"
1592                 begin
1593                     dependencies = `ldd #{source}`
1594                     dependencies.split(/\n/).each {
1595                         | dependency |
1596                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1597                     }
1598                 rescue
1599                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1600                 end
1601             end
1602         }
1603     end
1604     
1605     Dir.chdir($outputDir) {
1606         FileUtils.cp_r HELPERS_PATH, ".helpers"
1607     }
1608
1609     ARGV.each {
1610         | collection |
1611         handleCollection(collection)
1612     }
1613
1614     puts
1615 end
1616
1617 def cleanOldResults
1618     raise unless $bundle
1619
1620     eachResultFile($outputDir) {
1621         | path |
1622         FileUtils.rm_f path
1623     }
1624 end
1625
1626 def cleanEmptyResultFiles
1627     eachResultFile($outputDir) {
1628         | path |
1629         next unless path.basename.to_s =~ /\.out$/
1630         next unless FileTest.size(path) == 0
1631         FileUtils.rm_f path
1632     }
1633 end
1634
1635 def eachResultFile(startingDir, &block)
1636     dirsToClean = [startingDir]
1637     until dirsToClean.empty?
1638         nextDir = dirsToClean.pop
1639         Dir.foreach(nextDir) {
1640             | entry |
1641             next if entry =~ /^\./
1642             path = nextDir + entry
1643             if path.directory?
1644                 dirsToClean.push(path)
1645             else
1646                 block.call(path)
1647             end
1648         }
1649     end
1650 end
1651
1652 def prepareTestRunner
1653     raise if $bundle
1654
1655     $runlist.each_with_index {
1656         | plan, index |
1657         plan.index = index
1658     }
1659
1660     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1661     toDelete = []
1662     Dir.foreach($runnerDir) {
1663         | filename |
1664         if filename =~ /^test_/
1665             toDelete << filename
1666         end
1667     }
1668     
1669     toDelete.each {
1670         | filename |
1671         File.unlink($runnerDir + filename)
1672     }
1673
1674     $runlist.each {
1675         | plan |
1676         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1677     }
1678
1679     case $testRunnerType
1680     when :make
1681         prepareMakeTestRunner
1682     when :shell
1683         prepareShellTestRunner
1684     when :ruby
1685         prepareRubyTestRunner
1686     else
1687         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1688     end
1689 end
1690
1691 def cleanRunnerDirectory
1692     raise unless $bundle
1693     Dir.foreach($runnerDir) {
1694         | filename |
1695         next unless filename =~ /^test_fail/
1696         FileUtils.rm_f $runnerDir + filename
1697     }
1698 end
1699
1700 def sshRead(cmd)
1701     raise unless $remote
1702
1703     result = ""
1704     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1705       | inp |
1706       inp.each_line {
1707         | line |
1708         result += line
1709       }
1710     }
1711     raise "#{$?}" unless $?.success?
1712     result
1713 end
1714
1715 def runCommandOnTester(cmd)
1716     if $remote
1717         result = sshRead(cmd)
1718     else
1719         result = `#{cmd}`
1720     end
1721 end
1722
1723 def numberOfProcessors
1724     if $hostOS == "windows"
1725         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1726     else
1727         begin
1728             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1729         rescue
1730             numProcessors = 0
1731         end
1732
1733         if numProcessors == 0
1734             begin
1735                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1736             rescue
1737                 numProcessors == 0
1738             end
1739         end
1740     end
1741
1742     if numProcessors == 0
1743         numProcessors = 1
1744     end
1745     return numProcessors
1746 end
1747
1748 def runAndMonitorTestRunnerCommand(*cmd)
1749     numberOfTests = 0
1750     Dir.chdir($runnerDir) {
1751         # -1 for the runscript, and -2 for '..' and '.'
1752         numberOfTests = Dir.entries(".").count - 3
1753     }
1754     unless $progressMeter
1755         mysys(cmd.join(' '))
1756     else
1757        running = {}
1758        didRun = {}
1759        didFail = {}
1760        blankLine = true
1761        prevStringLength = 0
1762        IO.popen(cmd.join(' '), mode="r") {
1763            | inp |
1764            inp.each_line {
1765                | line |
1766                line = line.scrub.chomp
1767                if line =~ /^Running /
1768                    running[$~.post_match] = true
1769                elsif line =~ /^PASS: /
1770                    didRun[$~.post_match] = true
1771                elsif line =~ /^FAIL: /
1772                    didRun[$~.post_match] = true
1773                    didFail[$~.post_match] = true
1774                else
1775                    unless blankLine
1776                        print("\r" + " " * prevStringLength + "\r")
1777                    end
1778                    puts line
1779                    blankLine = true
1780                end
1781
1782                def lpad(str, chars)
1783                    str = str.to_s
1784                    if str.length > chars
1785                        str
1786                    else
1787                       "%#{chars}s"%(str)
1788                    end
1789                end
1790
1791                string  = ""
1792                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1793                unless didFail.empty?
1794                    string += " (failed #{didFail.size})"
1795                end
1796                string += " "
1797                (running.size - didRun.size).times {
1798                    string += "."
1799                }
1800                if string.length < prevStringLength
1801                    print string
1802                    print(" " * (prevStringLength - string.length))
1803                end
1804                print string
1805                prevStringLength = string.length
1806                blankLine = false
1807                $stdout.flush
1808            }
1809        }
1810        puts
1811        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1812     end
1813 end
1814
1815 def runTestRunner
1816     if $remote
1817         if !$remoteDirectory
1818             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1819         end
1820         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1821         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1822         remoteScript = "\""
1823         remoteScript += "cd #{$remoteDirectory} && "
1824         remoteScript += "rm -rf #{$outputDir.basename} && "
1825         remoteScript += "tar xzf #{$tarFileName} && "
1826         remoteScript += "cd #{$outputDir.basename}/.runner && "
1827         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1828         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1829         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
1830         $envVars.each { |var| remoteScript += "export " << var << "\n" }
1831         remoteScript += "#{testRunnerCommand}\""
1832         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1833     else
1834         Dir.chdir($runnerDir) {
1835             runAndMonitorTestRunnerCommand(testRunnerCommand)
1836         }
1837     end
1838 end
1839
1840 def detectFailures
1841     raise if $bundle
1842
1843     failures = []
1844     
1845     if $remote
1846         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1847         output.split(/\n/).each {
1848             | line |
1849             next unless line =~ /test_fail_/
1850             failures << $~.post_match.to_i
1851         }
1852     else
1853         Dir.foreach($runnerDir) {
1854             | filename |
1855             next unless filename =~ /test_fail_/
1856             failures << $~.post_match.to_i
1857         }
1858     end
1859
1860     failureSet = {}
1861
1862     failures.each {
1863         | failure | 
1864         appendFailure($runlist[failure])
1865         failureSet[failure] = true
1866     }
1867
1868     familyMap = {}
1869     $runlist.each_with_index {
1870         | plan, index |
1871         unless familyMap[plan.family]
1872             familyMap[plan.family] = []
1873         end
1874         if failureSet[index]
1875             appendResult(plan, false)
1876             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
1877             next
1878         else
1879             appendResult(plan, true)
1880             familyMap[plan.family] << {:result => "PASS", :plan => plan};
1881         end
1882         appendPass(plan)
1883     }
1884
1885     File.open($outputDir + "resultsByFamily", "w") {
1886         | outp |
1887         first = true
1888         familyMap.keys.sort.each {
1889             | familyName |
1890             if first
1891                 first = false
1892             else
1893                 outp.puts
1894             end
1895             
1896             outp.print "#{familyName}:"
1897
1898             numPassed = 0
1899             familyMap[familyName].each {
1900                 | entry |
1901                 if entry[:result] == "PASS"
1902                     numPassed += 1
1903                 end
1904             }
1905
1906             if numPassed == familyMap[familyName].size
1907                 outp.puts " PASSED"
1908             elsif numPassed == 0
1909                 outp.puts " FAILED"
1910             else
1911                 outp.puts
1912                 familyMap[familyName].each {
1913                     | entry |
1914                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
1915                 }
1916             end
1917         }
1918     }
1919 end
1920
1921 def compressBundle
1922     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
1923     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1924     raise unless system(cmd)
1925 end
1926
1927 def clean(file)
1928     FileUtils.rm_rf file unless $bundle
1929 end
1930
1931 clean($outputDir + "failed")
1932 clean($outputDir + "passed")
1933 clean($outputDir + "results")
1934 clean($outputDir + "resultsByFamily")
1935 clean($outputDir + ".vm")
1936 clean($outputDir + ".helpers")
1937 clean($outputDir + ".runner")
1938 clean($outputDir + ".tests")
1939 clean($outputDir + "_payload")
1940
1941 Dir.mkdir($outputDir) unless $outputDir.directory?
1942
1943 $outputDir = $outputDir.realpath
1944 $runnerDir = $outputDir + ".runner"
1945
1946 if !$numChildProcesses
1947     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
1948         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
1949     else
1950         $numChildProcesses = numberOfProcessors
1951     end
1952 end
1953
1954 if ENV["JSCTEST_timeout"]
1955     # In the worst case, the processors just interfere with each other.
1956     # Increase the timeout proportionally to the number of processors.
1957     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
1958 end
1959
1960 def runBundle
1961     raise unless $bundle
1962
1963     cleanRunnerDirectory
1964     cleanOldResults
1965     runTestRunner
1966     cleanEmptyResultFiles
1967 end
1968
1969 def runNormal
1970     raise if $bundle or $tarball
1971
1972     prepareBundle
1973     prepareTestRunner
1974     runTestRunner
1975     cleanEmptyResultFiles
1976     detectFailures
1977 end
1978
1979 def runTarball
1980     raise unless $tarball
1981
1982     prepareBundle 
1983     prepareTestRunner
1984     compressBundle
1985 end
1986
1987 def runRemote
1988     raise unless $remote
1989
1990     prepareBundle
1991     prepareTestRunner
1992     compressBundle
1993     runTestRunner
1994     detectFailures
1995 end
1996
1997 puts
1998 if $bundle
1999     runBundle
2000 elsif $remote
2001     runRemote
2002 elsif $tarball
2003     runTarball
2004 else
2005     runNormal
2006 end