Enable gigacage on iOS
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
index 44d272e..4a707c7 100755 (executable)
@@ -1,6 +1,6 @@
 #!/usr/bin/env ruby
 
-# Copyright (C) 2013-2015 Apple Inc. All rights reserved.
+# Copyright (C) 2013-2016 Apple Inc. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -37,10 +37,18 @@ module URI
     @@schemes['SSH'] = SSH
 end
 
+class String
+    def scrub
+        encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
+    end
+end
+
 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
+WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
+CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
 
@@ -62,6 +70,7 @@ begin
     $canRunDisplayProfilerOutput = true
 rescue Exception => e
     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
+    $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
     $stderr.puts "Error: #{e.inspect}"
 end
 
@@ -90,7 +99,6 @@ end
 
 $jscPath = nil
 $doNotMessWithVMPath = false
-$enableFTL = false
 $jitTests = true
 $memoryLimited = false
 $outputDir = Pathname.new("results")
@@ -100,6 +108,7 @@ $tarball = false
 $tarFileName = "payload.tar.gz"
 $copyVM = false
 $testRunnerType = nil
+$testWriter = "default"
 $remoteUser = nil
 $remoteHost = nil
 $remotePort = nil
@@ -107,7 +116,10 @@ $remoteDirectory = nil
 $architecture = nil
 $hostOS = nil
 $filter = nil
-
+$envVars = []
+$mode = "full"
+$buildType = "release"
+$forceCollectContinuously = false
 
 def usage
     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
@@ -115,10 +127,11 @@ def usage
     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
     puts "                            --jsc specifies an already present JavaScriptCore to test."
-    puts "--ftl-jit                   Indicate that we have the FTL JIT."
     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
-    puts "                            Skip tests tagged with //@large-heap"
+    puts "                            Skip tests tagged with //@skip if $memoryLimited"
     puts "--no-jit                    Do not run JIT specific tests."
+    puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
+    puts "                            platform."
     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
     puts "--verbose            (-v)   Print more things while running."
     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
@@ -130,11 +143,24 @@ def usage
     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
     puts "                            In general the shell runner is slower than the make runner."
     puts "--make-runner               Uses the faster make-based runner."
+    puts "--ruby-runner               Uses the ruby runner for machines without unix shell or make."
+    puts "--test-writer [writer]      Specifies the test script format."
+    puts "                            default is to use shell scripts to run the tests"
+    puts "                            \"ruby\" to use ruby scripts for systems without a unix shell."
     puts "--remote                    Specify a remote host on which to run tests from command line argument."
     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
     puts "--child-processes    (-c)   Specify the number of child processes."
     puts "--filter                    Only run tests whose name matches the given regular expression."
     puts "--help               (-h)   Print this message."
+    puts "--env-vars                  Add a list of environment variables to set before running jsc."
+    puts "                            Each environment variable should be separated by a space."
+    puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
+    puts "                            it will override the default value."
+    puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
+    puts "--basic                     Run with default and these additional modes: no-llint,"
+    puts "                            no-cjit-validate-phases, no-cjit-collect-continuously, dfg-eager"
+    puts "                            and for FTL platforms: no-ftl, ftl-eager-no-cjit and"
+    puts "                            ftl-no-cjit-small-pool."
     exit 1
 end
 
@@ -143,9 +169,9 @@ jscArg = nil
 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
                ['--no-copy', GetoptLong::NO_ARGUMENT],
-               ['--ftl-jit', GetoptLong::NO_ARGUMENT],
                ['--memory-limited', GetoptLong::NO_ARGUMENT],
                ['--no-jit', GetoptLong::NO_ARGUMENT],
+               ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
@@ -154,11 +180,18 @@ GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
                ['--os', GetoptLong::REQUIRED_ARGUMENT],
                ['--shell-runner', GetoptLong::NO_ARGUMENT],
                ['--make-runner', GetoptLong::NO_ARGUMENT],
+               ['--ruby-runner', GetoptLong::NO_ARGUMENT],
+               ['--test-writer', GetoptLong::REQUIRED_ARGUMENT],
                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
-               ['--verbose', '-v', GetoptLong::NO_ARGUMENT]).each {
+               ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
+               ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
+               ['--debug', GetoptLong::NO_ARGUMENT],
+               ['--release', GetoptLong::NO_ARGUMENT],
+               ['--quick', '-q', GetoptLong::NO_ARGUMENT],
+               ['--basic', GetoptLong::NO_ARGUMENT]).each {
     | opt, arg |
     case opt
     when '--help'
@@ -169,12 +202,12 @@ GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
         $doNotMessWithVMPath = true
     when '--output-dir'
         $outputDir = Pathname.new(arg)
-    when '--ftl-jit'
-        $enableFTL = true
     when '--memory-limited'
         $memoryLimited = true
     when '--no-jit'
         $jitTests = false
+    when '--force-collectContinuously'
+        $forceCollectContinuously = true;
     when '--verbose'
         $verbosity += 1
     when '--run-bundle'
@@ -189,6 +222,10 @@ GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
         $testRunnerType = :shell
     when '--make-runner'
         $testRunnerType = :make
+    when '--ruby-runner'
+        $testRunnerType = :ruby
+    when '--test-writer'
+        $testWriter = arg
     when '--remote'
         $copyVM = true
         $tarball = true
@@ -205,6 +242,16 @@ GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
         $architecture = arg
     when '--os'
         $hostOS = arg
+    when '--env-vars'
+        $envVars = arg.gsub(/\s+/, ' ').split(' ')
+    when '--quick'
+        $mode = "quick"
+    when '--basic'
+        $mode = "basic"
+    when '--debug'
+        $buildType = "debug"
+    when '--release'
+        $buildType = "release"
     end
 }
 
@@ -226,13 +273,27 @@ if $remoteConfigFile
 end
 
 unless jscArg
-    $stderr.puts "Error: must specify -jsc <path>"
-    exit 1
-end
+    # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
+    command = SCRIPTS_PATH.join("webkit-build-directory").to_s
+    command += ($buildType == "release") ? " --release" : " --debug"
+    command += " --executablePath"
+
+    output = `#{command}`.split("\n")
+    if !output.length
+        $stderr.puts "Error: must specify --jsc <path>"
+        exit 1
+    end
 
-if $enableFTL and !$jitTests
-    $stderr.puts "Error: can only specify one of --no-jit and --ftl-jit"
-    exit 1
+    output = output[0]
+    jscArg = Pathname.new(output).join("jsc")
+    jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
+    jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
+    if !File.file?(jscArg)
+        $stderr.puts "Error: must specify --jsc <path>"
+        exit 1
+    end
+
+    puts "Using the following jsc path: #{jscArg}"
 end
 
 if $doNotMessWithVMPath
@@ -251,7 +312,7 @@ end
 # Try to determine architecture. Return nil on failure.
 def machOArchitectureCode
     begin 
-        otoolLines = `otool -aSfh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
+        otoolLines = `otool -afh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
         otoolLines.each_with_index {
             | value, index |
             if value =~ /magic/ and value =~ /cputype/
@@ -297,6 +358,8 @@ def determineArchitectureFromELFBinary
     case code
     when 3
         "x86"
+    when 8
+        "mips"
     when 62
         "x86-64"
     when 40
@@ -309,12 +372,43 @@ def determineArchitectureFromELFBinary
     end
 end
 
+def determineArchitectureFromPEBinary
+    f = File.open($jscPath.to_s)
+    data = f.read(1024)
+
+    if !(data[0, 2] == "MZ")
+        $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
+        return nil
+    end
+
+    peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
+
+    if !(data[peHeaderAddr, 4] == "PE\0\0")
+        $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
+        return nil
+    end
+
+    machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
+
+    case machine
+    when 0x014c
+        "x86"
+    when 0x8664
+        "x86-64"
+    else
+        $stderr.puts "Warning: unsupported machine type: #{machine}"
+        nil
+    end
+end
+
 def determineArchitecture
     case $hostOS
     when "darwin"
         determineArchitectureFromMachOBinary
     when "linux"
         determineArchitectureFromELFBinary
+    when "windows"
+        determineArchitectureFromPEBinary
     else
         $stderr.puts "Warning: unable to determine architecture on this platform."
         nil
@@ -337,6 +431,7 @@ end
 
 $hostOS = determineOS unless $hostOS
 $architecture = determineArchitecture unless $architecture
+$isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $architecture == "mips" || $hostOS == "windows")
 
 if !$testRunnerType
     if $remote and $hostOS == "darwin"
@@ -346,12 +441,31 @@ if !$testRunnerType
     end
 end
 
+if $testWriter
+    if /[^-a-zA-Z0-9_]/.match($testWriter)
+        raise "Invalid test writer #{$testWriter} given"
+    end
+end
+
 $numFailures = 0
+$numPasses = 0
+
+# We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
+BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--validateExceptionChecks=true", "--maxPerThreadStackUsage=1572864"]
+EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
+# NOTE: Tests rely on this using scribbleFreeCells.
+NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
+B3O1_OPTIONS = ["--defaultB3OptLevel=1"]
+FTL_OPTIONS = ["--useFTLJIT=true"]
+PROBE_OSR_EXIT_OPTION = ["--useProbeOSRExit=true"]
+
+require_relative "webkitruby/jsc-stress-test-writer-#{$testWriter}"
 
-BASE_OPTIONS = ["--useFTLJIT=false", "--enableFunctionDotArguments=true"]
-EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000"]
-NO_CJIT_OPTIONS = ["--enableConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100"]
-FTL_OPTIONS = ["--useFTLJIT=true", "--ftlCrashesIfCantInitializeLLVM=true"]
+def shouldCollectContinuously?
+    $buildType == "release" or $forceCollectContinuously
+end
+
+COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false"] : []
 
 $runlist = []
 
@@ -384,248 +498,8 @@ def pathToHelpers
     pathToBundleResourceFromBenchmarkDirectory(".helpers")
 end
 
-def prefixCommand(prefix)
-    "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
-end
-
-def redirectAndPrefixCommand(prefix)
-    prefixCommand(prefix) + " 2>&1"
-end
-
-def pipeAndPrefixCommand(outputFilename, prefix)
-    "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
-end
-
-# Output handler for tests that are expected to be silent.
-def silentOutputHandler
-    Proc.new {
-        | name |
-        " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
-    }
-end
-
-# Output handler for tests that are expected to produce meaningful output.
-def noisyOutputHandler
-    Proc.new {
-        | name |
-        " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
-    }
-end
-
-# Error handler for tests that fail exactly when they return non-zero exit status.
-# This is useful when a test is expected to fail.
-def simpleErrorHandler
-    Proc.new {
-        | outp, plan |
-        outp.puts "if test -e #{plan.failFile}"
-        outp.puts "then"
-        outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
-        outp.puts "    " + plan.failCommand
-        outp.puts "else"
-        outp.puts "    " + plan.successCommand
-        outp.puts "fi"
-    }
-end
-
-# Error handler for tests that fail exactly when they return zero exit status.
-def expectedFailErrorHandler
-    Proc.new {
-        | outp, plan |
-        outp.puts "if test -e #{plan.failFile}"
-        outp.puts "then"
-        outp.puts "    " + plan.successCommand
-        outp.puts "else"
-        outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
-        outp.puts "    " + plan.failCommand
-        outp.puts "fi"
-    }
-end
-
-# Error handler for tests that fail exactly when they return non-zero exit status and produce
-# lots of spew. This will echo that spew when the test fails.
-def noisyErrorHandler
-    Proc.new {
-        | outp, plan |
-        outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
-    
-        outp.puts "if test -e #{plan.failFile}"
-        outp.puts "then"
-        outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
-        outp.puts "    " + plan.failCommand
-        outp.puts "else"
-        outp.puts "    " + plan.successCommand
-        outp.puts "fi"
-    }
-end
-
-# Error handler for tests that diff their output with some expectation.
-def diffErrorHandler(expectedFilename)
-    Proc.new {
-        | outp, plan |
-        outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
-        diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
-        
-        outp.puts "if test -e #{plan.failFile}"
-        outp.puts "then"
-        outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
-        outp.puts "    " + plan.failCommand
-        outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
-        outp.puts "then"
-        outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
-        outp.puts "    if [ $? -eq 0 ]"
-        outp.puts "    then"
-        outp.puts "    " + plan.successCommand
-        outp.puts "    else"
-        outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
-        outp.puts "        " + plan.failCommand
-        outp.puts "    fi"
-        outp.puts "else"
-        outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
-        outp.puts "    " + plan.failCommand
-        outp.puts "fi"
-    }
-end
-
-# Error handler for tests that report error by saying "failed!". This is used by Mozilla
-# tests.
-def mozillaErrorHandler
-    Proc.new {
-        | outp, plan |
-        outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
-
-        outp.puts "if test -e #{plan.failFile}"
-        outp.puts "then"
-        outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
-        outp.puts "    " + plan.failCommand
-        outp.puts "elif grep -i -q failed! #{outputFilename}"
-        outp.puts "then"
-        outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
-        outp.puts "    " + plan.failCommand
-        outp.puts "else"
-        outp.puts "    " + plan.successCommand
-        outp.puts "fi"
-    }
-end
-
-# Error handler for tests that report error by saying "failed!", and are expected to
-# fail. This is used by Mozilla tests.
-def mozillaFailErrorHandler
-    Proc.new {
-        | outp, plan |
-        outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
-
-        outp.puts "if test -e #{plan.failFile}"
-        outp.puts "then"
-        outp.puts "    " + plan.successCommand
-        outp.puts "elif grep -i -q failed! #{outputFilename}"
-        outp.puts "then"
-        outp.puts "    " + plan.successCommand
-        outp.puts "else"
-        outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
-        outp.puts "    " + plan.failCommand
-        outp.puts "fi"
-    }
-end
-
-# Error handler for tests that report error by saying "failed!", and are expected to have
-# an exit code of 3.
-def mozillaExit3ErrorHandler
-    Proc.new {
-        | outp, plan |
-        outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
-
-        outp.puts "if test -e #{plan.failFile}"
-        outp.puts "then"
-        outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
-        outp.puts "    then"
-        outp.puts "        if grep -i -q failed! #{outputFilename}"
-        outp.puts "        then"
-        outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
-        outp.puts "            " + plan.failCommand
-        outp.puts "        else"
-        outp.puts "            " + plan.successCommand
-        outp.puts "        fi"
-        outp.puts "    else"
-        outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
-        outp.puts "        " + plan.failCommand
-        outp.puts "    fi"
-        outp.puts "else"
-        outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
-        outp.puts "    " + plan.failCommand
-        outp.puts "fi"
-    }
-end
-
 $runCommandOptions = {}
 
-class Plan
-    attr_reader :directory, :arguments, :name, :outputHandler, :errorHandler
-    attr_accessor :index
-    
-    def initialize(directory, arguments, name, outputHandler, errorHandler)
-        @directory = directory
-        @arguments = arguments
-        @name = name
-        @outputHandler = outputHandler
-        @errorHandler = errorHandler
-        @isSlow = !!$runCommandOptions[:isSlow]
-    end
-    
-    def shellCommand
-        # It's important to remember that the test is actually run in a subshell, so if we change directory
-        # in the subshell when we return we will be in our original directory. This is nice because we don't
-        # have to bend over backwards to do things relative to the root.
-        "(cd ../#{Shellwords.shellescape(@directory.to_s)} && \"$@\" " + escapeAll(@arguments) + ")"
-    end
-    
-    def reproScriptCommand
-        # We have to find our way back to the .runner directory since that's where all of the relative
-        # paths assume they start out from.
-        script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
-        script += "cd $CURRENT_DIR\n"
-        Pathname.new(@name).dirname.each_filename {
-            | pathComponent |
-            script += "cd ..\n"
-        }
-        script += "cd .runner\n"
-
-        script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
-        script += "export JSC_timeout=#{Shellwords.shellescape(ENV['JSC_timeout'])}\n"
-        script += "#{shellCommand} || exit 1"
-        "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
-    end
-    
-    def failCommand
-        "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
-    end
-    
-    def successCommand
-        if $progressMeter or $verbosity >= 2
-            "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
-        else
-            "rm -f #{failFile}"
-        end
-    end
-    
-    def failFile
-        "test_fail_#{@index}"
-    end
-    
-    def writeRunScript(filename)
-        File.open(filename, "w") {
-            | outp |
-            outp.puts "echo Running #{Shellwords.shellescape(@name)}"
-            cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
-            cmd += @outputHandler.call(@name)
-            if $verbosity >= 3
-                outp.puts "echo #{Shellwords.shellescape(cmd)}"
-            end
-            outp.puts cmd
-            @errorHandler.call(outp, self)
-        }
-    end
-end
-
 $uniqueFilenameCounter = 0
 def uniqueFilename(extension)
     payloadDir = $outputDir + "_payload"
@@ -639,13 +513,16 @@ def baseOutputName(kind)
     "#{$collectionName}/#{$benchmark}.#{kind}"
 end
 
-def addRunCommand(kind, command, outputHandler, errorHandler)
+def addRunCommand(kind, command, outputHandler, errorHandler, *additionalEnv)
     $didAddRunCommand = true
     name = baseOutputName(kind)
     if $filter and name !~ $filter
         return
     end
-    plan = Plan.new($benchmarkDirectory, command, name, outputHandler, errorHandler)
+    plan = Plan.new(
+        $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
+        errorHandler)
+    plan.additionalEnv.push(*additionalEnv)
     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
         $runlist.unshift plan
     else
@@ -687,139 +564,229 @@ def slow!
     $runCommandOptions[:isSlow] = true
 end
 
+def runWithOutputHandler(kind, outputHandler, *options)
+    addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
+end
+
 def run(kind, *options)
-    addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
+    runWithOutputHandler(kind, silentOutputHandler, *options)
+end
+
+def runNoFTL(*optionalTestSpecificOptions)
+    run("no-ftl", *optionalTestSpecificOptions)
+end
+
+def runWithRAMSize(size, *optionalTestSpecificOptions)
+    run("ram-size-#{size}", "--forceRAMSize=#{size}", *optionalTestSpecificOptions)
 end
 
-def runDefault
-    run("default")
+def runOneLargeHeap(*optionalTestSpecificOptions)
+    if $memoryLimited
+        $didAddRunCommand = true
+        puts "Skipping #{$collectionName}/#{$benchmark}"
+    else
+        run("default", *optionalTestSpecificOptions)
+    end
 end
 
-def runWithRAMSize(size)
-    run("ram-size-#{size}", "--forceRAMSize=#{size}")
+def runNoJIT(*optionalTestSpecificOptions)
+    run("no-jit", "--useJIT=false", *optionalTestSpecificOptions)
 end
 
-def runNoLLInt
+def runNoLLInt(*optionalTestSpecificOptions)
     if $jitTests
-        run("no-llint", "--useLLInt=false")
+        run("no-llint", "--useLLInt=false", *optionalTestSpecificOptions)
     end
 end
 
-def runNoCJITValidate
-    run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *NO_CJIT_OPTIONS)
+# NOTE: Tests rely on this using scribbleFreeCells.
+def runNoCJITValidate(*optionalTestSpecificOptions)
+    run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runNoCJITValidatePhases
-    run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
+def runNoCJITValidatePhases(*optionalTestSpecificOptions)
+    run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runDefaultFTL
-    run("default-ftl", *FTL_OPTIONS) if $enableFTL
+def runNoCJITCollectContinuously(*optionalTestSpecificOptions)
+    run("no-cjit-collect-continuously", *(NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runFTLNoCJIT
-    run("ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
+def runDefault(*optionalTestSpecificOptions)
+    run("default", *(FTL_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runFTLNoCJITValidate
-    run("ftl-no-cjit-validate", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
+def runFTLNoCJIT(*optionalTestSpecificOptions)
+    run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runFTLNoCJITNoInlineValidate
-    run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
+def runFTLNoCJITB3O1(*optionalTestSpecificOptions)
+    run("ftl-no-cjit-b3o1", "--useArrayAllocationProfiling=false", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runFTLNoCJITOSRValidation
-    run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
+def runFTLNoCJITValidate(*optionalTestSpecificOptions)
+    run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + PROBE_OSR_EXIT_OPTION + optionalTestSpecificOptions))
 end
 
-def runDFGEager
-    run("dfg-eager", *EAGER_OPTIONS)
+def runFTLNoCJITNoPutStackValidate(*optionalTestSpecificOptions)
+    run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runDFGEagerNoCJITValidate
-    run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
+def runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
+    run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runFTLEager
-    run("ftl-eager", *(FTL_OPTIONS + EAGER_OPTIONS)) if $enableFTL
+def runFTLNoCJITOSRValidation(*optionalTestSpecificOptions)
+    run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runFTLEagerNoCJITValidate
-    run("ftl-eager-no-cjit", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
+def runDFGEager(*optionalTestSpecificOptions)
+    run("dfg-eager", *(EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + PROBE_OSR_EXIT_OPTION + optionalTestSpecificOptions))
 end
 
-def runFTLEagerNoCJITOSRValidation
-    run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
+def runDFGEagerNoCJITValidate(*optionalTestSpecificOptions)
+    run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runAlwaysTriggerCopyPhase
-    run("always-trigger-copy-phase", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
+def runFTLEager(*optionalTestSpecificOptions)
+    run("ftl-eager", "--airForceBriggsAllocator=true", "--forcePolyProto=true", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runNoCJITNoASO
-    run("no-cjit-no-aso", "--enableArchitectureSpecificOptimizations=false", *NO_CJIT_OPTIONS)
+def runFTLEagerWatchdog(*optionalTestSpecificOptions)
+    timeout = rand(100)
+    run("ftl-eager-watchdog-#{timeout}", "--watchdog=#{timeout}", "--watchdog-exception-ok", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runFTLNoCJITNoSimpleOpt
-    run("ftl-no-cjit-no-simple-opt", "--llvmSimpleOpt=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
+def runFTLEagerNoCJITValidate(*optionalTestSpecificOptions)
+    run("ftl-eager-no-cjit", "--validateGraph=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runNoCJITNoAccessInlining
-    run("no-cjit-no-access-inlining", "--enableAccessInlining=false", *NO_CJIT_OPTIONS)
+def runFTLEagerNoCJITB3O1(*optionalTestSpecificOptions)
+    run("ftl-eager-no-cjit-b3o1", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runFTLNoCJITNoAccessInlining
-    run("ftl-no-cjit-no-access-inlining", "--enableAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
+def runFTLEagerNoCJITOSRValidation(*optionalTestSpecificOptions)
+    run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runFTLNoCJITSmallPool
-    run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
+def runNoCJITNoASO(*optionalTestSpecificOptions)
+    run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runMiscNoCJITTest(*options)
-    run("misc-no-cjit", *(NO_CJIT_OPTIONS + options))
+def runNoCJITNoAccessInlining(*optionalTestSpecificOptions)
+    run("no-cjit-no-access-inlining", "--useAccessInlining=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
 end
 
-def runMiscFTLNoCJITTest(*options)
-    run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + options))
+def runFTLNoCJITNoAccessInlining(*optionalTestSpecificOptions)
+    run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
+end
+
+def runFTLNoCJITSmallPool(*optionalTestSpecificOptions)
+    run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
+end
+
+def runNoCJIT(*optionalTestSpecificOptions)
+    run("no-cjit", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
+end
+
+def runDFGMaximalFlushPhase(*optionalTestSpecificOptions)
+    run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
+end
+
+def runShadowChicken(*optionalTestSpecificOptions)
+    run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true", *optionalTestSpecificOptions)
 end
 
 def defaultRun
-    runDefault
-    runAlwaysTriggerCopyPhase
-    if $jitTests
-        runNoLLInt
-        runNoCJITValidatePhases
-        runDFGEager
-        runDFGEagerNoCJITValidate
-        runDefaultFTL
-        runFTLNoCJITValidate
-        runFTLNoCJITNoInlineValidate
-        runFTLEager
-        runFTLEagerNoCJITValidate
-        runFTLNoCJITSmallPool
+    if $mode == "quick"
+        defaultQuickRun
+    else
+        runDefault
+        if $jitTests
+            runNoLLInt
+            runNoCJITValidatePhases
+            runNoCJITCollectContinuously if shouldCollectContinuously?
+            runDFGEager
+            if $mode != "basic"
+                runDFGEagerNoCJITValidate
+                runDFGMaximalFlushPhase
+            end
+
+            return if !$isFTLPlatform
+
+            runNoFTL
+            runFTLEager
+            runFTLEagerNoCJITValidate
+            runFTLNoCJITSmallPool
+
+            return if $mode == "basic"
+
+            runFTLNoCJITValidate
+            runFTLNoCJITB3O1
+            runFTLNoCJITNoPutStackValidate
+            runFTLNoCJITNoInlineValidate
+            runFTLEagerNoCJITB3O1
+        end
     end
 end
 
-def defaultQuickRun
-    if $enableFTL
-        runDefaultFTL
-        runFTLNoCJITValidate
+def defaultNoNoLLIntRun
+    if $mode == "quick"
+        defaultQuickRun
     else
         runDefault
         if $jitTests
-            runNoCJITValidate
+            runNoCJITValidatePhases
+            runNoCJITCollectContinuously if shouldCollectContinuously?
+            runDFGEager
+            if $mode != "basic"
+                runDFGEagerNoCJITValidate
+                runDFGMaximalFlushPhase
+            end
+
+            return if !$isFTLPlatform
+
+            runNoFTL
+            runFTLNoCJITValidate
+            runFTLNoCJITSmallPool
+
+            return if $mode == "basic"
+
+            runFTLNoCJITB3O1
+            runFTLNoCJITNoPutStackValidate
+            runFTLNoCJITNoInlineValidate
+            runFTLEager
+            runFTLEagerNoCJITValidate
         end
     end
 end
 
-def defaultSpotCheck
+def defaultQuickRun
+    runDefault
+    if $jitTests
+        runNoCJITValidate
+
+        return if !$isFTLPlatform
+
+        runNoFTL
+        runFTLNoCJITValidate
+    end
+end
+
+def defaultSpotCheckNoMaximalFlush
     defaultQuickRun
-    runFTLNoCJITNoSimpleOpt
-    runFTLNoCJITOSRValidation
     runNoCJITNoAccessInlining
+
+    return if !$isFTLPlatform
+
+    runFTLNoCJITOSRValidation
     runFTLNoCJITNoAccessInlining
+    runFTLNoCJITB3O1
+end
+
+def defaultSpotCheck
+    defaultSpotCheckNoMaximalFlush
+    runDFGMaximalFlushPhase
 end
 
 # This is expected to not do eager runs because eager runs can have a lot of recompilations
@@ -827,13 +794,41 @@ end
 # by counting recompilations.
 def defaultNoEagerRun
     runDefault
-    runAlwaysTriggerCopyPhase
     if $jitTests
         runNoLLInt
         runNoCJITValidatePhases
-        runDefaultFTL
+        runNoCJITCollectContinuously if shouldCollectContinuously?
+
+        return if !$isFTLPlatform
+
+        runNoFTL
         runFTLNoCJITValidate
+
+        return if $mode == "basic"
+
         runFTLNoCJITNoInlineValidate
+        runFTLNoCJITB3O1
+    end
+end
+
+def defaultNoSamplingProfilerRun
+    runDefault
+    if $jitTests
+        runNoLLInt
+        runNoCJITValidatePhases
+        runNoCJITCollectContinuously if shouldCollectContinuously?
+        runDFGEager
+        runDFGEagerNoCJITValidate
+        runDFGMaximalFlushPhase
+
+        return if !$isFTLPlatform
+
+        runNoFTL
+        runFTLNoCJITNoPutStackValidate
+        runFTLNoCJITNoInlineValidate
+        runFTLEager
+        runFTLEagerNoCJITValidate
+        runFTLNoCJITSmallPool
     end
 end
 
@@ -845,10 +840,10 @@ def runProfiler
 
     profilerOutput = uniqueFilename(".json")
     if $canRunDisplayProfilerOutput
-        addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
+        addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
     else
         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
-        run("profiler-simple", "-p", profilerOutput.to_s)
+        run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
     end
 end
 
@@ -867,11 +862,12 @@ def runTypeProfiler
         return
     end
 
-    if $enableFTL
-        run("ftl-no-cjit-type-profiler", "--enableTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
-    else
-        run("no-cjit-type-profiler", "--enableTypeProfiler=true", *NO_CJIT_OPTIONS)
-    end
+    run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
+    run("ftl-no-cjit-type-profiler-force-poly-proto", "--useTypeProfiler=true", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
+
+    return if !$isFTLPlatform
+
+    run("ftl-type-profiler-ftl-eager", "--useTypeProfiler=true", *(FTL_OPTIONS + EAGER_OPTIONS))
 end
 
 def runControlFlowProfiler
@@ -879,11 +875,72 @@ def runControlFlowProfiler
         return
     end
 
-    if $enableFTL
-        run("ftl-no-cjit-type-profiler", "--enableControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
+    return if !$isFTLPlatform
+
+    run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
+end
+
+def runTest262(mode, exception, includeFiles, flags)
+    failsWithException = exception != "NoException"
+    isStrict = false
+    isModule = false
+    isAsync = false
+
+    flags.each {
+        | flag |
+        case flag
+        when :strict
+            isStrict = true
+        when :module
+            isModule = true
+        when :async
+            isAsync = true
+        else
+            raise "Invalid flag for runTest262, #{flag}"
+        end
+    }
+
+    prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
+
+    args = [pathToVM.to_s] + BASE_OPTIONS
+    args << "--exception=" + exception if failsWithException
+    args << "--test262-async" if isAsync
+    args += includeFiles
+
+    case mode
+    when :normal
+        errorHandler = simpleErrorHandler
+        outputHandler = silentOutputHandler
+    when :fail
+        errorHandler = expectedFailErrorHandler
+        outputHandler = noisyOutputHandler
+    when :failDueToOutdatedOrBadTest
+        errorHandler = expectedFailErrorHandler
+        outputHandler = noisyOutputHandler
+    when :skip
+        return
+    else
+        raise "Invalid mode: #{mode}"
+    end
+
+    if isStrict
+        kind = "default-strict"
+        args << "--strict-file=#{$benchmark}"
     else
-        run("no-cjit-type-profiler", "--enableControlFlowProfiler=true", *NO_CJIT_OPTIONS)
+        kind = "default"
+        if isModule
+            args << "--module-file=#{$benchmark}"
+        else
+            args << $benchmark.to_s
+        end
     end
+
+    addRunCommand(kind, args, outputHandler, errorHandler)
+end
+
+def prepareTest262Fixture
+    # This function is used to add the files used by Test262 modules tests.
+    prepareExtraRelativeFiles([""], $collection)
 end
 
 def runES6(mode)
@@ -893,12 +950,143 @@ def runES6(mode)
         errorHandler = simpleErrorHandler
     when :fail
         errorHandler = expectedFailErrorHandler
+    when :failDueToOutdatedOrBadTest
+        errorHandler = expectedFailErrorHandler
+    when :skip
+        return
     else
         raise "Invalid mode: #{mode}"
     end
     addRunCommand("default", args, noisyOutputHandler, errorHandler)
 end
 
+def runModules
+    run("default-modules", "-m")
+
+    if !$jitTests
+        return
+    end
+
+    run("no-llint-modules", "-m", "--useLLInt=false")
+    run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
+    run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
+    run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
+
+    return if !$isFTLPlatform
+
+    run("default-ftl-modules", "-m", *FTL_OPTIONS)
+    run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
+    run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
+    run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
+    run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
+    run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
+end
+
+def runWebAssembly
+    return if !$jitTests
+    return if !$isFTLPlatform
+    modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
+    prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
+    prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
+    run("default-wasm", "-m", *FTL_OPTIONS)
+    if $mode != "quick"
+        run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
+        run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
+        run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
+        run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
+        run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
+    end
+end
+
+def runWebAssemblyEmscripten(mode)
+    case mode
+    when :skip
+        return
+    end
+    return if !$jitTests
+    return if !$isFTLPlatform
+    wasm = $benchmark.to_s.sub! '.js', '.wasm'
+    prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
+    run("default-wasm", *FTL_OPTIONS)
+    if $mode != "quick"
+        run("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
+        run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
+        run("wasm-no-call-ic", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
+        run("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
+    end
+end
+
+def runWebAssemblySpecTest(mode)
+    case mode
+    when :skip
+        return
+    end
+    return if !$jitTests
+    return if !$isFTLPlatform
+    prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
+
+    modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
+    prepareExtraRelativeFiles(modules.map { |f| "../../" + f }, $collection)
+
+    harness = Dir[WASMTESTS_PATH + "spec-harness/" + "*.js"].map { |f| File.basename(f) }
+    prepareExtraRelativeFiles(harness.map { |f| "../../spec-harness/" + f }, $collection)
+
+    runWithOutputHandler("default-wasm", noisyOutputHandler, "../spec-harness.js", *FTL_OPTIONS)
+    if $mode != "quick"
+      runWithOutputHandler("wasm-no-cjit-yes-tls-context", noisyOutputHandler, "../spec-harness.js",  "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
+      runWithOutputHandler("wasm-eager-jettison", noisyOutputHandler, "../spec-harness.js", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
+      runWithOutputHandler("wasm-no-call-ic", noisyOutputHandler, "../spec-harness.js", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
+      runWithOutputHandler("wasm-no-tls-context", noisyOutputHandler, "../spec-harness.js", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
+    end
+end
+
+def runWebAssemblyLowExecutableMemory(*optionalTestSpecificOptions)
+    return if !$jitTests
+    return if !$isFTLPlatform
+    modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
+    prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
+    prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
+    # Only let WebAssembly get executable memory.
+    run("default-wasm", "--useConcurrentGC=0" , "--useConcurrentJIT=0", "--jitMemoryReservationSize=15000", "--useBaselineJIT=0", "--useDFGJIT=0", "--useFTLJIT=0", "-m")
+end
+
+def runChakra(mode, exception, baselineFile, extraFiles)
+    raise unless $benchmark.to_s =~ /\.js$/
+    failsWithException = exception != "NoException"
+    testName = $~.pre_match
+
+    prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
+    prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)
+
+    args = [pathToVM.to_s] + BASE_OPTIONS
+    args += FTL_OPTIONS if $isFTLPlatform
+    args += EAGER_OPTIONS
+    args << "--exception=" + exception if failsWithException
+    args << "--dumpException" if failsWithException
+    args += ["jsc-lib.js"]
+
+    case mode
+    when :baseline
+        prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
+        errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
+        outputHandler = noisyOutputHandler
+    when :pass
+        errorHandler = chakraPassFailErrorHandler
+        outputHandler = noisyOutputHandler
+    when :skipDueToOutdatedOrBadTest
+        return
+    when :skip
+        return
+    else
+        raise "Invalid mode: #{mode}"
+    end
+
+    kind = "default"
+    args << $benchmark.to_s
+
+    addRunCommand(kind, args, outputHandler, errorHandler)
+end
+
 def runLayoutTest(kind, *options)
     raise unless $benchmark.to_s =~ /\.js$/
     testName = $~.pre_match
@@ -918,8 +1106,8 @@ def runLayoutTest(kind, *options)
     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
 end
 
-def runLayoutTestDefault
-    runLayoutTest(nil)
+def runLayoutTestNoFTL
+    runLayoutTest("no-ftl")
 end
 
 def runLayoutTestNoLLInt
@@ -934,16 +1122,20 @@ def runLayoutTestDFGEagerNoCJIT
     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
 end
 
-def runLayoutTestDefaultFTL
-    runLayoutTest("ftl", "--testTheFTL=true", *FTL_OPTIONS) if $enableFTL
+def runLayoutTestDefault
+    runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
 end
 
 def runLayoutTestFTLNoCJIT
-    runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
+    runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
 end
 
 def runLayoutTestFTLEagerNoCJIT
-    runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
+    runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
+end
+
+def runLayoutTestFTLEagerNoCJITB3O1
+    runLayoutTest("ftl-eager-no-cjit-b3o1", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS))
 end
 
 def noFTLRunLayoutTest
@@ -956,13 +1148,45 @@ def noFTLRunLayoutTest
     runLayoutTestDFGEagerNoCJIT
 end
 
+def defaultQuickRunLayoutTest
+    runLayoutTestDefault
+    if $jitTests
+        if $isFTLPlatform
+            runLayoutTestNoFTL
+            runLayoutTestFTLNoCJIT
+            runLayoutTestFTLEagerNoCJIT
+        else
+            noFTLRunLayoutTest
+        end
+    end
+end
+
 def defaultRunLayoutTest
+    if $mode == "quick"
+        defaultQuickRunLayoutTest
+    else
+        runLayoutTestDefault
+        if $jitTests
+            noFTLRunLayoutTest
+
+            return if !$isFTLPlatform
+
+            runLayoutTestNoFTL
+            runLayoutTestFTLNoCJIT
+            runLayoutTestFTLEagerNoCJIT
+        end
+    end
+end
+
+def noEagerNoNoLLIntTestsRunLayoutTest
     runLayoutTestDefault
     if $jitTests
-        noFTLRunLayoutTest
-        runLayoutTestDefaultFTL
+        runLayoutTestNoCJIT
+
+        return if !$isFTLPlatform
+
+        runLayoutTestNoFTL
         runLayoutTestFTLNoCJIT
-        runLayoutTestFTLEagerNoCJIT
     end
 end
 
@@ -971,7 +1195,10 @@ def noNoLLIntRunLayoutTest
     if $jitTests
         runLayoutTestNoCJIT
         runLayoutTestDFGEagerNoCJIT
-        runLayoutTestDefaultFTL
+
+        return if !$isFTLPlatform
+
+        runLayoutTestNoFTL
         runLayoutTestFTLNoCJIT
         runLayoutTestFTLEagerNoCJIT
     end
@@ -981,7 +1208,9 @@ def prepareExtraRelativeFiles(extraFiles, destination)
     Dir.chdir($outputDir) {
         extraFiles.each {
             | file |
-            FileUtils.cp $extraFilesBaseDir + file, destination + file
+            dest = destination + file
+            FileUtils.mkdir_p(dest.dirname)
+            FileUtils.cp $extraFilesBaseDir + file, dest
         }
     }
 end
@@ -1018,6 +1247,8 @@ def runMozillaTest(kind, mode, extraFiles, *options)
         errorHandler = mozillaExit3ErrorHandler
     when :fail
         errorHandler = mozillaFailErrorHandler
+    when :failDueToOutdatedOrBadTest
+        errorHandler = mozillaFailErrorHandler
     when :skip
         return
     else
@@ -1027,11 +1258,11 @@ def runMozillaTest(kind, mode, extraFiles, *options)
 end
 
 def runMozillaTestDefault(mode, *extraFiles)
-    runMozillaTest(nil, mode, extraFiles)
+    runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
 end
 
-def runMozillaTestDefaultFTL(mode, *extraFiles)
-    runMozillaTest("ftl", mode, extraFiles, *FTL_OPTIONS) if $enableFTL
+def runMozillaTestNoFTL(mode, *extraFiles)
+    runMozillaTest("no-ftl", mode, extraFiles)
 end
 
 def runMozillaTestLLInt(mode, *extraFiles)
@@ -1047,45 +1278,74 @@ def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
 end
 
 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
-    runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
+    runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
 end
 
-def defaultRunMozillaTest(mode, *extraFiles)
-    runMozillaTestDefault(mode, *extraFiles)
+def defaultQuickRunMozillaTest(mode, *extraFiles)
     if $jitTests
-        runMozillaTestLLInt(mode, *extraFiles)
-        runMozillaTestBaselineJIT(mode, *extraFiles)
-        runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
-        runMozillaTestDefaultFTL(mode, *extraFiles)
+        runMozillaTestDefault(mode, *extraFiles)
         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
+    else
+        runMozillaTestNoFTL(mode, *extraFiles)
+        if $jitTests
+            runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
+        end
+    end
+end
+
+def defaultRunMozillaTest(mode, *extraFiles)
+    if $mode == "quick"
+        defaultQuickRunMozillaTest(mode, *extraFiles)
+    else
+        runMozillaTestNoFTL(mode, *extraFiles)
+        if $jitTests
+            runMozillaTestLLInt(mode, *extraFiles)
+            runMozillaTestBaselineJIT(mode, *extraFiles)
+            runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
+            runMozillaTestDefault(mode, *extraFiles)
+            runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
+        end
     end
 end
 
+def runNoisyTestImpl(kind, options, additionalEnv)
+    addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler, *additionalEnv)
+end
+
 def runNoisyTest(kind, *options)
-    addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
+    runNoisyTestImpl(kind, options, [])
+end
+
+def runNoisyTestWithEnv(kind, *additionalEnv)
+    runNoisyTestImpl(kind, [], additionalEnv)
 end
 
 def runNoisyTestDefault
-    runNoisyTest("default")
+    runNoisyTest("default", *FTL_OPTIONS)
 end
 
-def runNoisyTestDefaultFTL
-    runNoisyTest("ftl", *FTL_OPTIONS) if $enableFTL
+def runNoisyTestNoFTL
+    runNoisyTest("no-ftl")
 end
 
 def runNoisyTestNoCJIT
-    runNoisyTest($enableFTL ? "ftl-no-cjit" : "no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(($enableFTL ? FTL_OPTIONS : []) + NO_CJIT_OPTIONS))
+    runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
+end
+
+def runNoisyTestNoCJITB3O1
+    runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS))
 end
 
 def runNoisyTestEagerNoCJIT
-    runNoisyTest($enableFTL ? "ftl-eager-no-cjit" : "eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(($enableFTL ? FTL_OPTIONS : []) + NO_CJIT_OPTIONS + EAGER_OPTIONS))
+    runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
 end
 
 def defaultRunNoisyTest
     runNoisyTestDefault
-    if $jitTests
-        runNoisyTestDefaultFTL
+    if $jitTests and $isFTLPlatform
+        runNoisyTestNoFTL
         runNoisyTestNoCJIT
+        runNoisyTestNoCJITB3O1
         runNoisyTestEagerNoCJIT
     end
 end
@@ -1095,13 +1355,6 @@ def skip
     puts "Skipping #{$collectionName}/#{$benchmark}"
 end
 
-def largeHeap
-    if $memoryLimited
-        $didAddRunCommand = true
-        puts "Skipping #{$collectionName}/#{$benchmark}"
-    end
-end
-
 def allJSFiles(path)
     if path.file?
         [path]
@@ -1276,6 +1529,21 @@ def appendFailure(plan)
     $numFailures += 1
 end
 
+def appendPass(plan)
+    File.open($outputDir + "passed", "a") {
+        | outp |
+        outp.puts plan.name
+    }
+    $numPasses += 1
+end
+
+def appendResult(plan, didPass)
+    File.open($outputDir + "results", "a") {
+        | outp |
+        outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
+    }
+end
+
 def prepareBundle
     raise if $bundle
 
@@ -1321,13 +1589,6 @@ def prepareBundle
             end
 
             if $remote and $hostOS == "linux"
-                if $enableFTL
-                    begin
-                        FileUtils.cp_r originalJSCPath + "../../lib/libllvmForJSC.so" , $jscPath.dirname
-                    rescue
-                        $stderr.puts "Warning: unable to copy libllvmForJSC.so to the bundle."
-                    end
-                end
                 begin
                     dependencies = `ldd #{source}`
                     dependencies.split(/\n/).each {
@@ -1420,71 +1681,13 @@ def prepareTestRunner
         prepareMakeTestRunner
     when :shell
         prepareShellTestRunner
+    when :ruby
+        prepareRubyTestRunner
     else
         raise "Unknown test runner type: #{$testRunnerType.to_s}"
     end
 end
 
-def prepareShellTestRunner
-    FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
-end
-
-def prepareMakeTestRunner
-    # The goals of our parallel test runner are scalability and simplicity. The
-    # simplicity part is particularly important. We don't want to have to have
-    # a full-time contributor just philosophising about parallel testing.
-    #
-    # As such, we just pass off all of the hard work to 'make'. This creates a
-    # dummy directory ("$outputDir/.runner") in which we create a dummy
-    # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
-    # That is, for each test we know we will run, there is a rule in the
-    # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
-    # Makefile results in 'make' doing all of the hard work:
-    #
-    # - Load balancing just works. Most systems have a great load balancer in
-    #   'make'. If your system doesn't then just install a real 'make'.
-    #
-    # - Interruptions just work. For example Ctrl-C handling in 'make' is
-    #   exactly right. You don't have to worry about zombie processes.
-    #
-    # We then do some tricks to make failure detection work and to make this
-    # totally sound. If a test fails, we don't want the whole 'make' job to
-    # stop. We also don't have any facility for makefile-escaping of path names.
-    # We do have such a thing for shell-escaping, though. We fix both problems
-    # by having the actual work for each of the test rules be done in a shell
-    # script on the side. There is one such script per test. The script responds
-    # to failure by printing something on the console and then touching a
-    # failure file for that test, but then still returns 0. This makes 'make'
-    # continue past that failure and complete all the tests anyway.
-    #
-    # In the end, this script collects all of the failures by searching for
-    # files in the .runner directory whose name matches /^test_fail_/, where
-    # the thing after the 'fail_' is the test index. Those are the files that
-    # would be created by the test scripts if they detect failure. We're
-    # basically using the filesystem as a concurrent database of test failures.
-    # Even if two tests fail at the same time, since they're touching different
-    # files we won't miss any failures.
-    runIndices = []
-    $runlist.each {
-        | plan |
-        runIndices << plan.index
-    }
-    
-    File.open($runnerDir + "Makefile", "w") {
-        | outp |
-        outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
-        runIndices.each {
-            | index |
-            plan = $runlist[index]
-            outp.puts "test_done_#{index}:"
-            outp.puts "\tsh test_script_#{plan.index}"
-        }
-    }
-end
-
-    
-puts
-
 def cleanRunnerDirectory
     raise unless $bundle
     Dir.foreach($runnerDir) {
@@ -1518,17 +1721,21 @@ def runCommandOnTester(cmd)
 end
 
 def numberOfProcessors
-    begin
-        numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
-    rescue
-        numProcessors = 0
-    end
-
-    if numProcessors == 0
+    if $hostOS == "windows"
+        numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
+    else
         begin
-            numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
+            numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
         rescue
-            numProcessors == 0
+            numProcessors = 0
+        end
+
+        if numProcessors == 0
+            begin
+                numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
+            rescue
+                numProcessors == 0
+            end
         end
     end
 
@@ -1556,7 +1763,7 @@ def runAndMonitorTestRunnerCommand(*cmd)
            | inp |
            inp.each_line {
                | line |
-               line.chomp!
+               line = line.scrub.chomp
                if line =~ /^Running /
                    running[$~.post_match] = true
                elsif line =~ /^PASS: /
@@ -1606,15 +1813,6 @@ def runAndMonitorTestRunnerCommand(*cmd)
 end
 
 def runTestRunner
-    case $testRunnerType
-    when :shell
-        testRunnerCommand = "sh runscript"
-    when :make
-        testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
-    else
-        raise "Unknown test runner type: #{$testRunnerType.to_s}"
-    end
-
     if $remote
         if !$remoteDirectory
             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
@@ -1628,7 +1826,8 @@ def runTestRunner
         remoteScript += "cd #{$outputDir.basename}/.runner && "
         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
-        remoteScript += "export JSC_timeout=#{Shellwords.shellescape(ENV['JSC_timeout'])} && "
+        remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
+        $envVars.each { |var| remoteScript += "export " << var << "\n" }
         remoteScript += "#{testRunnerCommand}\""
         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
     else
@@ -1641,20 +1840,82 @@ end
 def detectFailures
     raise if $bundle
 
+    failures = []
+    
     if $remote
         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
         output.split(/\n/).each {
             | line |
             next unless line =~ /test_fail_/
-            appendFailure($runlist[$~.post_match.to_i])
+            failures << $~.post_match.to_i
         }
     else
         Dir.foreach($runnerDir) {
             | filename |
             next unless filename =~ /test_fail_/
-            appendFailure($runlist[$~.post_match.to_i])
+            failures << $~.post_match.to_i
         }
     end
+
+    failureSet = {}
+
+    failures.each {
+        | failure | 
+        appendFailure($runlist[failure])
+        failureSet[failure] = true
+    }
+
+    familyMap = {}
+    $runlist.each_with_index {
+        | plan, index |
+        unless familyMap[plan.family]
+            familyMap[plan.family] = []
+        end
+        if failureSet[index]
+            appendResult(plan, false)
+            familyMap[plan.family] << {:result => "FAIL", :plan => plan};
+            next
+        else
+            appendResult(plan, true)
+            familyMap[plan.family] << {:result => "PASS", :plan => plan};
+        end
+        appendPass(plan)
+    }
+
+    File.open($outputDir + "resultsByFamily", "w") {
+        | outp |
+        first = true
+        familyMap.keys.sort.each {
+            | familyName |
+            if first
+                first = false
+            else
+                outp.puts
+            end
+            
+            outp.print "#{familyName}:"
+
+            numPassed = 0
+            familyMap[familyName].each {
+                | entry |
+                if entry[:result] == "PASS"
+                    numPassed += 1
+                end
+            }
+
+            if numPassed == familyMap[familyName].size
+                outp.puts " PASSED"
+            elsif numPassed == 0
+                outp.puts " FAILED"
+            else
+                outp.puts
+                familyMap[familyName].each {
+                    | entry |
+                    outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
+                }
+            end
+        }
+    }
 end
 
 def compressBundle
@@ -1668,6 +1929,9 @@ def clean(file)
 end
 
 clean($outputDir + "failed")
+clean($outputDir + "passed")
+clean($outputDir + "results")
+clean($outputDir + "resultsByFamily")
 clean($outputDir + ".vm")
 clean($outputDir + ".helpers")
 clean($outputDir + ".runner")
@@ -1687,16 +1951,10 @@ if !$numChildProcesses
     end
 end
 
-if $enableFTL and ENV["JSC_timeout"]
-    # Currently, using the FTL is a performance regression particularly in real
-    # (i.e. non-loopy) benchmarks. Account for this in the timeout.
-    ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i * 2).to_s
-end
-
-if ENV["JSC_timeout"]
+if ENV["JSCTEST_timeout"]
     # In the worst case, the processors just interfere with each other.
     # Increase the timeout proportionally to the number of processors.
-    ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
+    ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
 end
 
 def runBundle
@@ -1736,6 +1994,7 @@ def runRemote
     detectFailures
 end
 
+puts
 if $bundle
     runBundle
 elsif $remote