[WinCairo] Support --no-copy for jsc tests in wincairo tests, add copying of dlls...
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2016 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
47 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
48 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
49 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
50 WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
51 CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
52 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
53 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
54
55 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
56
57 begin
58     require 'shellwords'
59 rescue Exception => e
60     $stderr.puts "Warning: did not find shellwords, not running any tests."
61     exit 0
62 end
63
64 $canRunDisplayProfilerOutput = false
65
66 begin
67     require 'rubygems'
68     require 'json'
69     require 'highline'
70     $canRunDisplayProfilerOutput = true
71 rescue Exception => e
72     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
73     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
74     $stderr.puts "Error: #{e.inspect}"
75 end
76
77 def printCommandArray(*cmd)
78     begin
79         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
80     rescue
81         commandArray = cmd.join(' ')
82     end
83     $stderr.puts ">> #{commandArray}"
84 end
85
86 def mysys(*cmd)
87     printCommandArray(*cmd) if $verbosity >= 1
88     raise "Command failed: #{$?.inspect}" unless system(*cmd)
89 end
90
91 def escapeAll(array)
92     array.map {
93         | v |
94         raise "Detected a non-string in #{inspect}" unless v.is_a? String
95         Shellwords.shellescape(v)
96     }.join(' ')
97 end
98
99
100 $jscPath = nil
101 $doNotMessWithVMPath = false
102 $jitTests = true
103 $memoryLimited = false
104 $outputDir = Pathname.new("results")
105 $verbosity = 0
106 $bundle = nil
107 $tarball = false
108 $tarFileName = "payload.tar.gz"
109 $copyVM = false
110 $testRunnerType = nil
111 $testWriter = "default"
112 $remoteUser = nil
113 $remoteHost = nil
114 $remotePort = nil
115 $remoteDirectory = nil
116 $architecture = nil
117 $hostOS = nil
118 $filter = nil
119 $envVars = []
120 $mode = "full"
121 $buildType = "release"
122 $forceCollectContinuously = false
123
124 def usage
125     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
126     puts
127     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
128     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
129     puts "                            --jsc specifies an already present JavaScriptCore to test."
130     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
131     puts "                            Skip tests tagged with //@skip if $memoryLimited"
132     puts "--no-jit                    Do not run JIT specific tests."
133     puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
134     puts "                            platform."
135     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
136     puts "--verbose            (-v)   Print more things while running."
137     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
138     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
139     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
140     puts "                            e.g. x86, x86_64, arm."
141     puts "--os                        Specify os instead of determining from JavaScriptCore build."
142     puts "                            e.g. darwin, linux & windows."
143     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
144     puts "                            In general the shell runner is slower than the make runner."
145     puts "--make-runner               Uses the faster make-based runner."
146     puts "--ruby-runner               Uses the ruby runner for machines without unix shell or make."
147     puts "--test-writer [writer]      Specifies the test script format."
148     puts "                            default is to use shell scripts to run the tests"
149     puts "                            \"ruby\" to use ruby scripts for systems without a unix shell."
150     puts "--remote                    Specify a remote host on which to run tests from command line argument."
151     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
152     puts "--child-processes    (-c)   Specify the number of child processes."
153     puts "--filter                    Only run tests whose name matches the given regular expression."
154     puts "--help               (-h)   Print this message."
155     puts "--env-vars                  Add a list of environment variables to set before running jsc."
156     puts "                            Each environment variable should be separated by a space."
157     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
158     puts "                            it will override the default value."
159     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
160     puts "--basic                     Run with default and these additional modes: no-llint,"
161     puts "                            no-cjit-validate-phases, no-cjit-collect-continuously, dfg-eager"
162     puts "                            and for FTL platforms: no-ftl, ftl-eager-no-cjit and"
163     puts "                            ftl-no-cjit-small-pool."
164     exit 1
165 end
166
167 jscArg = nil
168
169 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
170                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
171                ['--no-copy', GetoptLong::NO_ARGUMENT],
172                ['--memory-limited', GetoptLong::NO_ARGUMENT],
173                ['--no-jit', GetoptLong::NO_ARGUMENT],
174                ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
175                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
176                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
177                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
178                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
179                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
180                ['--os', GetoptLong::REQUIRED_ARGUMENT],
181                ['--shell-runner', GetoptLong::NO_ARGUMENT],
182                ['--make-runner', GetoptLong::NO_ARGUMENT],
183                ['--ruby-runner', GetoptLong::NO_ARGUMENT],
184                ['--test-writer', GetoptLong::REQUIRED_ARGUMENT],
185                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
186                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
187                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
188                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
189                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
190                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
191                ['--debug', GetoptLong::NO_ARGUMENT],
192                ['--release', GetoptLong::NO_ARGUMENT],
193                ['--quick', '-q', GetoptLong::NO_ARGUMENT],
194                ['--basic', GetoptLong::NO_ARGUMENT]).each {
195     | opt, arg |
196     case opt
197     when '--help'
198         usage
199     when '--jsc'
200         jscArg = arg
201     when '--no-copy'
202         $doNotMessWithVMPath = true
203     when '--output-dir'
204         $outputDir = Pathname.new(arg)
205     when '--memory-limited'
206         $memoryLimited = true
207     when '--no-jit'
208         $jitTests = false
209     when '--force-collectContinuously'
210         $forceCollectContinuously = true;
211     when '--verbose'
212         $verbosity += 1
213     when '--run-bundle'
214         $bundle = Pathname.new(arg)
215     when '--tarball'
216         $tarball = true
217         $copyVM = true
218         $tarFileName = arg unless arg == ''
219     when '--force-vm-copy'
220         $copyVM = true
221     when '--shell-runner'
222         $testRunnerType = :shell
223     when '--make-runner'
224         $testRunnerType = :make
225     when '--ruby-runner'
226         $testRunnerType = :ruby
227     when '--test-writer'
228         $testWriter = arg
229     when '--remote'
230         $copyVM = true
231         $tarball = true
232         $remote = true
233         uri = URI("ssh://" + arg)
234         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
235     when '--remote-config-file'
236         $remoteConfigFile = arg
237     when '--child-processes'
238         $numChildProcesses = arg.to_i
239     when '--filter'
240         $filter = Regexp.new(arg)
241     when '--arch'
242         $architecture = arg
243     when '--os'
244         $hostOS = arg
245     when '--env-vars'
246         $envVars = arg.gsub(/\s+/, ' ').split(' ')
247     when '--quick'
248         $mode = "quick"
249     when '--basic'
250         $mode = "basic"
251     when '--debug'
252         $buildType = "debug"
253     when '--release'
254         $buildType = "release"
255     end
256 }
257
258 if $remoteConfigFile
259     file = File.read($remoteConfigFile)
260     config = JSON.parse(file)
261
262     if !$remote and config['remote']
263         $copyVM = true
264         $tarball = true
265         $remote = true
266         uri = URI("ssh://" + config['remote'])
267         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
268     end
269
270     if config['remoteDirectory']
271         $remoteDirectory = config['remoteDirectory']
272     end
273 end
274
275 unless jscArg
276     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
277     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
278     command += ($buildType == "release") ? " --release" : " --debug"
279     command += " --executablePath"
280
281     output = `#{command}`.split("\n")
282     if !output.length
283         $stderr.puts "Error: must specify --jsc <path>"
284         exit 1
285     end
286
287     output = output[0]
288     jscArg = Pathname.new(output).join("jsc")
289     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
290     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
291     if !File.file?(jscArg)
292         $stderr.puts "Error: must specify --jsc <path>"
293         exit 1
294     end
295
296     puts "Using the following jsc path: #{jscArg}"
297 end
298
299 if $doNotMessWithVMPath
300     $jscPath = Pathname.new(jscArg)
301 else
302     $jscPath = Pathname.new(jscArg).realpath
303 end
304
305 $progressMeter = ($verbosity == 0 and $stdout.tty?)
306
307 if $bundle
308     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
309     $outputDir = $bundle
310 end
311
312 # Try to determine architecture. Return nil on failure.
313 def machOArchitectureCode
314     begin 
315         otoolLines = `otool -afh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
316         otoolLines.each_with_index {
317             | value, index |
318             if value =~ /magic/ and value =~ /cputype/
319                 return otoolLines[index + 1].split[1].to_i
320             end
321         }
322     rescue
323         $stderr.puts "Warning: unable to execute otool."
324     end
325     $stderr.puts "Warning: unable to determine architecture."
326     nil
327 end
328
329 def determineArchitectureFromMachOBinary
330     code = machOArchitectureCode
331     return nil unless code
332     is64BitFlag = 0x01000000
333     case code
334     when 7
335         "x86"
336     when 7 | is64BitFlag
337         "x86-64"
338     when 12
339         "arm"
340     when 12 | is64BitFlag
341         "arm64"
342     else
343         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
344         nil
345     end
346 end
347
348 def determineArchitectureFromELFBinary
349     f = File.open($jscPath.to_s)
350     data = f.read(19)
351
352     if !(data[0,4] == "\x7F\x45\x4C\x46")
353         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
354         return nil
355     end
356
357     code = data[18].ord
358     case code
359     when 3
360         "x86"
361     when 8
362         "mips"
363     when 62
364         "x86-64"
365     when 40
366         "arm"
367     when 183
368         "arm64"
369     else
370         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
371         nil
372     end
373 end
374
375 def determineArchitectureFromPEBinary
376     f = File.open($jscPath.to_s)
377     data = f.read(1024)
378
379     if !(data[0, 2] == "MZ")
380         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
381         return nil
382     end
383
384     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
385
386     if !(data[peHeaderAddr, 4] == "PE\0\0")
387         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
388         return nil
389     end
390
391     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
392
393     case machine
394     when 0x014c
395         "x86"
396     when 0x8664
397         "x86-64"
398     else
399         $stderr.puts "Warning: unsupported machine type: #{machine}"
400         nil
401     end
402 end
403
404 def determineArchitecture
405     case $hostOS
406     when "darwin"
407         determineArchitectureFromMachOBinary
408     when "linux"
409         determineArchitectureFromELFBinary
410     when "windows"
411         determineArchitectureFromPEBinary
412     else
413         $stderr.puts "Warning: unable to determine architecture on this platform."
414         nil
415     end
416 end
417
418 def determineOS
419     case RbConfig::CONFIG["host_os"]
420     when /darwin/i
421         "darwin"
422     when /linux/i
423         "linux"
424     when /mswin|mingw|cygwin/
425         "windows"
426     else
427         $stderr.puts "Warning: unable to determine host operating system"
428         nil
429     end
430 end
431
432 $hostOS = determineOS unless $hostOS
433 $architecture = determineArchitecture unless $architecture
434 $isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $architecture == "mips" || $hostOS == "windows")
435
436 def isFTLEnabled
437     $jitTests && $isFTLPlatform
438 end
439
440 if !$testRunnerType
441     if $remote and $hostOS == "darwin"
442         $testRunnerType = :shell
443     else
444         $testRunnerType = :make
445     end
446 end
447
448 if $testWriter
449     if /[^-a-zA-Z0-9_]/.match($testWriter)
450         raise "Invalid test writer #{$testWriter} given"
451     end
452 end
453
454 $numFailures = 0
455 $numPasses = 0
456
457 # We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
458 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--validateExceptionChecks=true", "--useDollarVM=true", "--maxPerThreadStackUsage=1572864"]
459 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
460 # NOTE: Tests rely on this using scribbleFreeCells.
461 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
462 B3O1_OPTIONS = ["--defaultB3OptLevel=1"]
463 FTL_OPTIONS = ["--useFTLJIT=true"]
464 PROBE_OSR_EXIT_OPTION = ["--useProbeOSRExit=true"]
465
466 require_relative "webkitruby/jsc-stress-test-writer-#{$testWriter}"
467
468 def shouldCollectContinuously?
469     $buildType == "release" or $forceCollectContinuously
470 end
471
472 COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false"] : []
473
474 $runlist = []
475
476 def frameworkFromJSCPath(jscPath)
477     parentDirectory = jscPath.dirname
478     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
479         parentDirectory.dirname
480     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
481         jscPath.dirname + "JavaScriptCore.framework"
482     else
483         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
484         nil
485     end
486 end
487
488 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
489     dir = Pathname.new(".")
490     $benchmarkDirectory.each_filename {
491         | pathComponent |
492         dir += ".."
493     }
494     dir + resourcePath
495 end
496
497 def pathToVM
498     pathToBundleResourceFromBenchmarkDirectory($jscPath)
499 end
500
501 def pathToHelpers
502     pathToBundleResourceFromBenchmarkDirectory(".helpers")
503 end
504
505 $runCommandOptions = {}
506
507 $uniqueFilenameCounter = 0
508 def uniqueFilename(extension)
509     payloadDir = $outputDir + "_payload"
510     Dir.mkdir payloadDir unless payloadDir.directory?
511     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
512     $uniqueFilenameCounter += 1
513     result
514 end
515
516 def baseOutputName(kind)
517     "#{$collectionName}/#{$benchmark}.#{kind}"
518 end
519
520 def addRunCommand(kind, command, outputHandler, errorHandler, *additionalEnv)
521     $didAddRunCommand = true
522     name = baseOutputName(kind)
523     if $filter and name !~ $filter
524         return
525     end
526     plan = Plan.new(
527         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
528         errorHandler)
529     plan.additionalEnv.push(*additionalEnv)
530     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
531         $runlist.unshift plan
532     else
533         $runlist << plan
534     end
535 end
536
537 # Returns true if there were run commands found in the file ($benchmarkDirectory +
538 # $benchmark), in which case those run commands have already been executed. Otherwise
539 # returns false, in which case you're supposed to add your own run commands.
540 def parseRunCommands
541     oldDidAddRunCommand = $didAddRunCommand
542     $didAddRunCommand = false
543     $skipped = false
544
545     Dir.chdir($outputDir) {
546         File.open($benchmarkDirectory + $benchmark) {
547             | inp |
548             inp.each_line {
549                 | line |
550                 begin
551                     doesMatch = line =~ /^\/\/@/
552                 rescue Exception => e
553                     # Apparently this happens in the case of some UTF8 stuff in some files, where
554                     # Ruby tries to be strict and throw exceptions.
555                     next
556                 end
557                 next unless doesMatch
558                 eval $~.post_match
559                 if $skipped
560                     break
561                 end
562             }
563         }
564     }
565
566     result = $didAddRunCommand
567     $didAddRunCommand = result or oldDidAddRunCommand
568     result
569 end
570
571 def slow!
572     $runCommandOptions[:isSlow] = true
573 end
574
575 def runWithOutputHandler(kind, outputHandler, *options)
576     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
577 end
578
579 def run(kind, *options)
580     runWithOutputHandler(kind, silentOutputHandler, *options)
581 end
582
583 def runNoFTL(*optionalTestSpecificOptions)
584     run("no-ftl", *optionalTestSpecificOptions)
585 end
586
587 def runWithRAMSize(size, *optionalTestSpecificOptions)
588     run("ram-size-#{size}", "--forceRAMSize=#{size}", *optionalTestSpecificOptions)
589 end
590
591 def runOneLargeHeap(*optionalTestSpecificOptions)
592     if $memoryLimited
593         $didAddRunCommand = true
594         puts "Skipping #{$collectionName}/#{$benchmark}"
595     else
596         run("default", *optionalTestSpecificOptions)
597     end
598 end
599
600 def runNoJIT(*optionalTestSpecificOptions)
601     run("no-jit", "--useJIT=false", *optionalTestSpecificOptions)
602 end
603
604 def runNoLLInt(*optionalTestSpecificOptions)
605     if $jitTests
606         run("no-llint", "--useLLInt=false", *optionalTestSpecificOptions)
607     end
608 end
609
610 # NOTE: Tests rely on this using scribbleFreeCells.
611 def runNoCJITValidate(*optionalTestSpecificOptions)
612     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
613 end
614
615 def runNoCJITValidatePhases(*optionalTestSpecificOptions)
616     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
617 end
618
619 def runNoCJITCollectContinuously(*optionalTestSpecificOptions)
620     run("no-cjit-collect-continuously", *(NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
621 end
622
623 def runDefault(*optionalTestSpecificOptions)
624     run("default", *(FTL_OPTIONS + optionalTestSpecificOptions))
625 end
626
627 def runBigIntEnabled(*optionalTestSpecificOptions)
628     run("big-int-enabled", "--useBigInt=true" , *(FTL_OPTIONS + optionalTestSpecificOptions))
629 end
630
631 def runFTLNoCJIT(*optionalTestSpecificOptions)
632     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
633 end
634
635 def runFTLNoCJITB3O1(*optionalTestSpecificOptions)
636     run("ftl-no-cjit-b3o1", "--useArrayAllocationProfiling=false", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
637 end
638
639 def runFTLNoCJITValidate(*optionalTestSpecificOptions)
640     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + PROBE_OSR_EXIT_OPTION + optionalTestSpecificOptions))
641 end
642
643 def runFTLNoCJITNoPutStackValidate(*optionalTestSpecificOptions)
644     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
645 end
646
647 def runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
648     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
649 end
650
651 def runFTLNoCJITOSRValidation(*optionalTestSpecificOptions)
652     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
653 end
654
655 def runDFGEager(*optionalTestSpecificOptions)
656     run("dfg-eager", *(EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + PROBE_OSR_EXIT_OPTION + optionalTestSpecificOptions))
657 end
658
659 def runDFGEagerNoCJITValidate(*optionalTestSpecificOptions)
660     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
661 end
662
663 def runFTLEager(*optionalTestSpecificOptions)
664     run("ftl-eager", "--airForceBriggsAllocator=true", "--forcePolyProto=true", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
665 end
666
667 def runFTLEagerWatchdog(*optionalTestSpecificOptions)
668     timeout = rand(100)
669     run("ftl-eager-watchdog-#{timeout}", "--watchdog=#{timeout}", "--watchdog-exception-ok", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
670 end
671
672 def runFTLEagerNoCJITValidate(*optionalTestSpecificOptions)
673     run("ftl-eager-no-cjit", "--validateGraph=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
674 end
675
676 def runFTLEagerNoCJITB3O1(*optionalTestSpecificOptions)
677     run("ftl-eager-no-cjit-b3o1", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
678 end
679
680 def runFTLEagerNoCJITOSRValidation(*optionalTestSpecificOptions)
681     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
682 end
683
684 def runNoCJITNoASO(*optionalTestSpecificOptions)
685     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
686 end
687
688 def runNoCJITNoAccessInlining(*optionalTestSpecificOptions)
689     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
690 end
691
692 def runFTLNoCJITNoAccessInlining(*optionalTestSpecificOptions)
693     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
694 end
695
696 def runFTLNoCJITSmallPool(*optionalTestSpecificOptions)
697     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
698 end
699
700 def runNoCJIT(*optionalTestSpecificOptions)
701     run("no-cjit", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
702 end
703
704 def runDFGMaximalFlushPhase(*optionalTestSpecificOptions)
705     run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
706 end
707
708 def runShadowChicken(*optionalTestSpecificOptions)
709     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true", *optionalTestSpecificOptions)
710 end
711
712 def defaultRun
713     if $mode == "quick"
714         defaultQuickRun
715     else
716         runDefault
717         if $jitTests
718             runNoLLInt
719             runNoCJITValidatePhases
720             runNoCJITCollectContinuously if shouldCollectContinuously?
721             runDFGEager
722             if $mode != "basic"
723                 runDFGEagerNoCJITValidate
724                 runDFGMaximalFlushPhase
725             end
726
727             return if !$isFTLPlatform
728
729             runNoFTL
730             runFTLEager
731             runFTLEagerNoCJITValidate
732             runFTLNoCJITSmallPool
733
734             return if $mode == "basic"
735
736             runFTLNoCJITValidate
737             runFTLNoCJITB3O1
738             runFTLNoCJITNoPutStackValidate
739             runFTLNoCJITNoInlineValidate
740             runFTLEagerNoCJITB3O1
741         end
742     end
743 end
744
745 def defaultNoNoLLIntRun
746     if $mode == "quick"
747         defaultQuickRun
748     else
749         runDefault
750         if $jitTests
751             runNoCJITValidatePhases
752             runNoCJITCollectContinuously if shouldCollectContinuously?
753             runDFGEager
754             if $mode != "basic"
755                 runDFGEagerNoCJITValidate
756                 runDFGMaximalFlushPhase
757             end
758
759             return if !$isFTLPlatform
760
761             runNoFTL
762             runFTLNoCJITValidate
763             runFTLNoCJITSmallPool
764
765             return if $mode == "basic"
766
767             runFTLNoCJITB3O1
768             runFTLNoCJITNoPutStackValidate
769             runFTLNoCJITNoInlineValidate
770             runFTLEager
771             runFTLEagerNoCJITValidate
772         end
773     end
774 end
775
776 def defaultQuickRun
777     runDefault
778     if $jitTests
779         runNoCJITValidate
780
781         return if !$isFTLPlatform
782
783         runNoFTL
784         runFTLNoCJITValidate
785     end
786 end
787
788 def defaultSpotCheckNoMaximalFlush
789     defaultQuickRun
790     runNoCJITNoAccessInlining
791
792     return if !$isFTLPlatform
793
794     runFTLNoCJITOSRValidation
795     runFTLNoCJITNoAccessInlining
796     runFTLNoCJITB3O1
797 end
798
799 def defaultSpotCheck
800     defaultSpotCheckNoMaximalFlush
801     runDFGMaximalFlushPhase
802 end
803
804 # This is expected to not do eager runs because eager runs can have a lot of recompilations
805 # for reasons that don't arise in the real world. It's used for tests that assert convergence
806 # by counting recompilations.
807 def defaultNoEagerRun
808     runDefault
809     if $jitTests
810         runNoLLInt
811         runNoCJITValidatePhases
812         runNoCJITCollectContinuously if shouldCollectContinuously?
813
814         return if !$isFTLPlatform
815
816         runNoFTL
817         runFTLNoCJITValidate
818
819         return if $mode == "basic"
820
821         runFTLNoCJITNoInlineValidate
822         runFTLNoCJITB3O1
823     end
824 end
825
826 def defaultNoSamplingProfilerRun
827     runDefault
828     if $jitTests
829         runNoLLInt
830         runNoCJITValidatePhases
831         runNoCJITCollectContinuously if shouldCollectContinuously?
832         runDFGEager
833         runDFGEagerNoCJITValidate
834         runDFGMaximalFlushPhase
835
836         return if !$isFTLPlatform
837
838         runNoFTL
839         runFTLNoCJITNoPutStackValidate
840         runFTLNoCJITNoInlineValidate
841         runFTLEager
842         runFTLEagerNoCJITValidate
843         runFTLNoCJITSmallPool
844     end
845 end
846
847 def runProfiler
848     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
849         skip
850         return
851     end
852
853     profilerOutput = uniqueFilename(".json")
854     if $canRunDisplayProfilerOutput
855         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
856     else
857         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
858         run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
859     end
860 end
861
862 def runExceptionFuzz
863     subCommand = escapeAll([pathToVM.to_s, "--useDollarVM=true", $benchmark.to_s])
864     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
865 end
866
867 def runExecutableAllocationFuzz(name, *options)
868     subCommand = escapeAll([pathToVM.to_s, "--useDollarVM=true", $benchmark.to_s] + options)
869     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
870 end
871
872 def runTypeProfiler
873     if !$jitTests
874         return
875     end
876
877     run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
878     run("ftl-no-cjit-type-profiler-force-poly-proto", "--useTypeProfiler=true", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
879
880     return if !$isFTLPlatform
881
882     run("ftl-type-profiler-ftl-eager", "--useTypeProfiler=true", *(FTL_OPTIONS + EAGER_OPTIONS))
883 end
884
885 def runControlFlowProfiler
886     if !$jitTests
887         return
888     end
889
890     return if !$isFTLPlatform
891
892     run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
893 end
894
895 def runTest262(mode, exception, includeFiles, flags)
896     failsWithException = exception != "NoException"
897     isStrict = false
898     isModule = false
899     isAsync = false
900
901     flags.each {
902         | flag |
903         case flag
904         when :strict
905             isStrict = true
906         when :module
907             isModule = true
908         when :async
909             isAsync = true
910         else
911             raise "Invalid flag for runTest262, #{flag}"
912         end
913     }
914
915     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
916
917     args = [pathToVM.to_s] + BASE_OPTIONS
918     args << "--exception=" + exception if failsWithException
919     args << "--test262-async" if isAsync
920     args += includeFiles
921
922     case mode
923     when :normal
924         errorHandler = simpleErrorHandler
925         outputHandler = silentOutputHandler
926     when :fail
927         errorHandler = expectedFailErrorHandler
928         outputHandler = noisyOutputHandler
929     when :failDueToOutdatedOrBadTest
930         errorHandler = expectedFailErrorHandler
931         outputHandler = noisyOutputHandler
932     when :skip
933         return
934     else
935         raise "Invalid mode: #{mode}"
936     end
937
938     if isStrict
939         kind = "default-strict"
940         args << "--strict-file=#{$benchmark}"
941     else
942         kind = "default"
943         if isModule
944             args << "--module-file=#{$benchmark}"
945         else
946             args << $benchmark.to_s
947         end
948     end
949
950     addRunCommand(kind, args, outputHandler, errorHandler)
951 end
952
953 def prepareTest262Fixture
954     # This function is used to add the files used by Test262 modules tests.
955     prepareExtraRelativeFiles([""], $collection)
956 end
957
958 def runES6(mode)
959     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
960     case mode
961     when :normal
962         errorHandler = simpleErrorHandler
963     when :fail
964         errorHandler = expectedFailErrorHandler
965     when :failDueToOutdatedOrBadTest
966         errorHandler = expectedFailErrorHandler
967     when :skip
968         return
969     else
970         raise "Invalid mode: #{mode}"
971     end
972     addRunCommand("default", args, noisyOutputHandler, errorHandler)
973 end
974
975 def defaultRunModules(noLLInt: true)
976     run("default-modules", "-m")
977
978     if !$jitTests
979         return
980     end
981
982     run("no-llint-modules", "-m", "--useLLInt=false") if noLLInt
983     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
984     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
985     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
986
987     return if !$isFTLPlatform
988
989     run("default-ftl-modules", "-m", *FTL_OPTIONS)
990     run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
991     run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
992     run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
993     run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
994     run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
995 end
996
997 def noNoLLIntRunModules
998     defaultRunModules(noLLInt: false)
999 end
1000
1001 def runWebAssembly
1002     return if !$jitTests
1003     return if !$isFTLPlatform
1004     run("default-wasm", "-m", *FTL_OPTIONS)
1005     if $mode != "quick"
1006         run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1007         run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1008         run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1009         run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1010         run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
1011     end
1012 end
1013
1014 def runWebAssemblySuite
1015     return if !$jitTests
1016     return if !$isFTLPlatform
1017     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1018     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1019     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1020     run("default-wasm", "-m", *FTL_OPTIONS)
1021     if $mode != "quick"
1022         run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1023         run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1024         run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1025         run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1026         run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
1027     end
1028 end
1029
1030 def runWebAssemblyEmscripten(mode)
1031     case mode
1032     when :skip
1033         return
1034     end
1035     return if !$jitTests
1036     return if !$isFTLPlatform
1037     wasm = $benchmark.to_s.sub! '.js', '.wasm'
1038     prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
1039     run("default-wasm", *FTL_OPTIONS)
1040     if $mode != "quick"
1041         run("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1042         run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1043         run("wasm-no-call-ic", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1044         run("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1045     end
1046 end
1047
1048 def runWebAssemblySpecTest(mode)
1049     case mode
1050     when :skip
1051         return
1052     end
1053     return if !$jitTests
1054     return if !$isFTLPlatform
1055     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1056
1057     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1058     prepareExtraRelativeFiles(modules.map { |f| "../../" + f }, $collection)
1059
1060     harness = Dir[WASMTESTS_PATH + "spec-harness/" + "*.js"].map { |f| File.basename(f) }
1061     prepareExtraRelativeFiles(harness.map { |f| "../../spec-harness/" + f }, $collection)
1062
1063     runWithOutputHandler("default-wasm", noisyOutputHandler, "../spec-harness.js", *FTL_OPTIONS)
1064     if $mode != "quick"
1065       runWithOutputHandler("wasm-no-cjit-yes-tls-context", noisyOutputHandler, "../spec-harness.js",  "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1066       runWithOutputHandler("wasm-eager-jettison", noisyOutputHandler, "../spec-harness.js", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1067       runWithOutputHandler("wasm-no-call-ic", noisyOutputHandler, "../spec-harness.js", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1068       runWithOutputHandler("wasm-no-tls-context", noisyOutputHandler, "../spec-harness.js", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1069     end
1070 end
1071
1072 def runWebAssemblyLowExecutableMemory(*optionalTestSpecificOptions)
1073     return if !$jitTests
1074     return if !$isFTLPlatform
1075     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1076     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1077     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1078     # Only let WebAssembly get executable memory.
1079     run("default-wasm", "--useConcurrentGC=0" , "--useConcurrentJIT=0", "--jitMemoryReservationSize=15000", "--useBaselineJIT=0", "--useDFGJIT=0", "--useFTLJIT=0", "-m")
1080 end
1081
1082 def runChakra(mode, exception, baselineFile, extraFiles)
1083     raise unless $benchmark.to_s =~ /\.js$/
1084     failsWithException = exception != "NoException"
1085     testName = $~.pre_match
1086
1087     prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
1088     prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)
1089
1090     args = [pathToVM.to_s] + BASE_OPTIONS
1091     args += FTL_OPTIONS if $isFTLPlatform
1092     args += EAGER_OPTIONS
1093     args << "--exception=" + exception if failsWithException
1094     args << "--dumpException" if failsWithException
1095     args += ["jsc-lib.js"]
1096
1097     case mode
1098     when :baseline
1099         prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
1100         errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
1101         outputHandler = noisyOutputHandler
1102     when :pass
1103         errorHandler = chakraPassFailErrorHandler
1104         outputHandler = noisyOutputHandler
1105     when :skipDueToOutdatedOrBadTest
1106         return
1107     when :skip
1108         return
1109     else
1110         raise "Invalid mode: #{mode}"
1111     end
1112
1113     kind = "default"
1114     args << $benchmark.to_s
1115
1116     addRunCommand(kind, args, outputHandler, errorHandler)
1117 end
1118
1119 def runLayoutTest(kind, *options)
1120     raise unless $benchmark.to_s =~ /\.js$/
1121     testName = $~.pre_match
1122     if kind
1123         kind = "layout-" + kind
1124     else
1125         kind = "layout"
1126     end
1127
1128     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1129     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1130
1131     args = [pathToVM.to_s] + BASE_OPTIONS + options +
1132         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1133          $benchmark.to_s,
1134          (Pathname.new("resources") + "standalone-post.js").to_s]
1135     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1136 end
1137
1138 def runLayoutTestNoFTL
1139     runLayoutTest("no-ftl")
1140 end
1141
1142 def runLayoutTestNoLLInt
1143     runLayoutTest("no-llint", "--useLLInt=false")
1144 end
1145
1146 def runLayoutTestNoCJIT
1147     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1148 end
1149
1150 def runLayoutTestDFGEagerNoCJIT
1151     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1152 end
1153
1154 def runLayoutTestDefault
1155     runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
1156 end
1157
1158 def runLayoutTestFTLNoCJIT
1159     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1160 end
1161
1162 def runLayoutTestFTLEagerNoCJIT
1163     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1164 end
1165
1166 def runLayoutTestFTLEagerNoCJITB3O1
1167     runLayoutTest("ftl-eager-no-cjit-b3o1", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS))
1168 end
1169
1170 def noFTLRunLayoutTest
1171     if !$jitTests
1172         return
1173     end
1174
1175     runLayoutTestNoLLInt
1176     runLayoutTestNoCJIT
1177     runLayoutTestDFGEagerNoCJIT
1178 end
1179
1180 def defaultQuickRunLayoutTest
1181     runLayoutTestDefault
1182     if $jitTests
1183         if $isFTLPlatform
1184             runLayoutTestNoFTL
1185             runLayoutTestFTLNoCJIT
1186             runLayoutTestFTLEagerNoCJIT
1187         else
1188             noFTLRunLayoutTest
1189         end
1190     end
1191 end
1192
1193 def defaultRunLayoutTest
1194     if $mode == "quick"
1195         defaultQuickRunLayoutTest
1196     else
1197         runLayoutTestDefault
1198         if $jitTests
1199             noFTLRunLayoutTest
1200
1201             return if !$isFTLPlatform
1202
1203             runLayoutTestNoFTL
1204             runLayoutTestFTLNoCJIT
1205             runLayoutTestFTLEagerNoCJIT
1206         end
1207     end
1208 end
1209
1210 def noEagerNoNoLLIntTestsRunLayoutTest
1211     runLayoutTestDefault
1212     if $jitTests
1213         runLayoutTestNoCJIT
1214
1215         return if !$isFTLPlatform
1216
1217         runLayoutTestNoFTL
1218         runLayoutTestFTLNoCJIT
1219     end
1220 end
1221
1222 def noNoLLIntRunLayoutTest
1223     runLayoutTestDefault
1224     if $jitTests
1225         runLayoutTestNoCJIT
1226         runLayoutTestDFGEagerNoCJIT
1227
1228         return if !$isFTLPlatform
1229
1230         runLayoutTestNoFTL
1231         runLayoutTestFTLNoCJIT
1232         runLayoutTestFTLEagerNoCJIT
1233     end
1234 end
1235
1236 def prepareExtraRelativeFiles(extraFiles, destination)
1237     Dir.chdir($outputDir) {
1238         extraFiles.each {
1239             | file |
1240             dest = destination + file
1241             FileUtils.mkdir_p(dest.dirname)
1242             FileUtils.cp $extraFilesBaseDir + file, dest
1243         }
1244     }
1245 end
1246
1247 def baseDirForCollection(collectionName)
1248     Pathname(".tests") + collectionName
1249 end
1250
1251 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1252     raise unless absoluteBase.absolute?
1253     Dir.chdir($outputDir) {
1254         collectionBaseDir = baseDirForCollection($collectionName)
1255         extraFiles.each {
1256             | file |
1257             destination = collectionBaseDir + file
1258             FileUtils.mkdir_p destination.dirname unless destination.directory?
1259             FileUtils.cp absoluteBase + file, destination
1260         }
1261     }
1262 end
1263
1264 def runMozillaTest(kind, mode, extraFiles, *options)
1265     if kind
1266         kind = "mozilla-" + kind
1267     else
1268         kind = "mozilla"
1269     end
1270     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1271     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1272     case mode
1273     when :normal
1274         errorHandler = mozillaErrorHandler
1275     when :negative
1276         errorHandler = mozillaExit3ErrorHandler
1277     when :fail
1278         errorHandler = mozillaFailErrorHandler
1279     when :failDueToOutdatedOrBadTest
1280         errorHandler = mozillaFailErrorHandler
1281     when :skip
1282         return
1283     else
1284         raise "Invalid mode: #{mode}"
1285     end
1286     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1287 end
1288
1289 def runMozillaTestDefault(mode, *extraFiles)
1290     runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
1291 end
1292
1293 def runMozillaTestNoFTL(mode, *extraFiles)
1294     runMozillaTest("no-ftl", mode, extraFiles)
1295 end
1296
1297 def runMozillaTestLLInt(mode, *extraFiles)
1298     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1299 end
1300
1301 def runMozillaTestBaselineJIT(mode, *extraFiles)
1302     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1303 end
1304
1305 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1306     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1307 end
1308
1309 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1310     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1311 end
1312
1313 def defaultQuickRunMozillaTest(mode, *extraFiles)
1314     if $jitTests
1315         runMozillaTestDefault(mode, *extraFiles)
1316         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1317     else
1318         runMozillaTestNoFTL(mode, *extraFiles)
1319         if $jitTests
1320             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1321         end
1322     end
1323 end
1324
1325 def defaultRunMozillaTest(mode, *extraFiles)
1326     if $mode == "quick"
1327         defaultQuickRunMozillaTest(mode, *extraFiles)
1328     else
1329         runMozillaTestNoFTL(mode, *extraFiles)
1330         if $jitTests
1331             runMozillaTestLLInt(mode, *extraFiles)
1332             runMozillaTestBaselineJIT(mode, *extraFiles)
1333             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1334             runMozillaTestDefault(mode, *extraFiles)
1335             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
1336         end
1337     end
1338 end
1339
1340 def runNoisyTestImpl(kind, options, additionalEnv)
1341     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler, *additionalEnv)
1342 end
1343
1344 def runNoisyTest(kind, *options)
1345     runNoisyTestImpl(kind, options, [])
1346 end
1347
1348 def runNoisyTestWithEnv(kind, *additionalEnv)
1349     runNoisyTestImpl(kind, [], additionalEnv)
1350 end
1351
1352 def runNoisyTestDefault
1353     runNoisyTest("default", *FTL_OPTIONS)
1354 end
1355
1356 def runNoisyTestNoFTL
1357     runNoisyTest("no-ftl")
1358 end
1359
1360 def runNoisyTestNoCJIT
1361     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1362 end
1363
1364 def runNoisyTestNoCJITB3O1
1365     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS))
1366 end
1367
1368 def runNoisyTestEagerNoCJIT
1369     runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1370 end
1371
1372 def defaultRunNoisyTest
1373     runNoisyTestDefault
1374     if $jitTests and $isFTLPlatform
1375         runNoisyTestNoFTL
1376         runNoisyTestNoCJIT
1377         runNoisyTestNoCJITB3O1
1378         runNoisyTestEagerNoCJIT
1379     end
1380 end
1381
1382 def skip
1383     $didAddRunCommand = true
1384     $skipped = true
1385     puts "Skipping #{$collectionName}/#{$benchmark}"
1386 end
1387
1388 def allJSFiles(path)
1389     if path.file?
1390         [path]
1391     else
1392         result = []
1393         Dir.foreach(path) {
1394             | filename |
1395             next unless filename =~ /\.js$/
1396             next unless (path + filename).file?
1397             result << path + filename
1398         }
1399         result
1400     end
1401 end
1402
1403 def uniqueifyName(names, name)
1404     result = name.to_s
1405     toAdd = 1
1406     while names[result]
1407         result = "#{name}-#{toAdd}"
1408         toAdd += 1
1409     end
1410     names[result] = true
1411     result
1412 end
1413
1414 def simplifyCollectionName(collectionPath)
1415     outerDir = collectionPath.dirname
1416     name = collectionPath.basename
1417     lastName = name
1418     if collectionPath.directory?
1419         while lastName.to_s =~ /test/
1420             lastName = outerDir.basename
1421             name = lastName + name
1422             outerDir = outerDir.dirname
1423         end
1424     end
1425     uniqueifyName($collectionNames, name)
1426 end
1427
1428 def prepareCollection(name)
1429     FileUtils.mkdir_p $outputDir + name
1430
1431     absoluteCollection = $collection.realpath
1432
1433     Dir.chdir($outputDir) {
1434         bundleDir = baseDirForCollection(name)
1435
1436         # Create the proper directory structures.
1437         FileUtils.mkdir_p bundleDir
1438         if bundleDir.basename == $collection.basename
1439             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1440             $collection = bundleDir
1441         else
1442             FileUtils.cp_r absoluteCollection, bundleDir
1443             $collection = bundleDir + $collection.basename
1444         end
1445
1446         $extraFilesBaseDir = absoluteCollection
1447     }
1448 end
1449
1450 $collectionNames = {}
1451
1452 def handleCollectionFile(collection)
1453     collectionName = simplifyCollectionName(collection)
1454    
1455     paths = {}
1456     subCollections = []
1457     YAML::load(IO::read(collection)).each {
1458         | entry |
1459         if entry["collection"]
1460             subCollections << entry["collection"]
1461             next
1462         end
1463         
1464         if Pathname.new(entry["path"]).absolute?
1465             raise "Absolute path: " + entry["path"] + " in #{collection}"
1466         end
1467         
1468         if paths[entry["path"]]
1469             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1470         end
1471         
1472         subCollection = collection.dirname + entry["path"]
1473         
1474         if subCollection.file?
1475             subCollectionName = Pathname.new(entry["path"]).dirname
1476         else
1477             subCollectionName = entry["path"]
1478         end
1479         
1480         $collection = subCollection
1481         $collectionName = Pathname.new(collectionName)
1482         Pathname.new(subCollectionName).each_filename {
1483             | filename |
1484             next if filename =~ /^\./
1485             $collectionName += filename
1486         }
1487         $collectionName = $collectionName.to_s
1488         
1489         prepareCollection($collectionName)
1490       
1491         Dir.chdir($outputDir) {
1492             pathsToSearch = [$collection]
1493             if entry["tests"]
1494                 if entry["tests"].is_a? Array
1495                     pathsToSearch = entry["tests"].map {
1496                         | testName |
1497                         pathsToSearch[0] + testName
1498                     }
1499                 else
1500                     pathsToSearch[0] += entry["tests"]
1501                 end
1502             end
1503             pathsToSearch.each {
1504                 | pathToSearch |
1505                 allJSFiles(pathToSearch).each {
1506                     | path |
1507                     
1508                     $benchmark = path.basename
1509                     $benchmarkDirectory = path.dirname
1510                     
1511                     $runCommandOptions = {}
1512                     eval entry["cmd"]
1513                 }
1514             }
1515         }
1516     }
1517     
1518     subCollections.each {
1519         | subCollection |
1520         handleCollection(collection.dirname + subCollection)
1521     }
1522 end
1523
1524 def handleCollectionDirectory(collection)
1525     collectionName = simplifyCollectionName(collection)
1526     
1527     $collection = collection
1528     $collectionName = collectionName
1529     prepareCollection(collectionName)
1530    
1531     Dir.chdir($outputDir) {
1532         $benchmarkDirectory = $collection
1533         allJSFiles($collection).each {
1534             | path |
1535             
1536             $benchmark = path.basename
1537             
1538             $runCommandOptions = {}
1539             defaultRun unless parseRunCommands
1540         }
1541     }
1542 end
1543
1544 def handleCollection(collection)
1545     collection = Pathname.new(collection)
1546     
1547     if collection.file?
1548         handleCollectionFile(collection)
1549     else
1550         handleCollectionDirectory(collection)
1551     end
1552 end
1553
1554 def appendFailure(plan)
1555     File.open($outputDir + "failed", "a") {
1556         | outp |
1557         outp.puts plan.name
1558     }
1559     $numFailures += 1
1560 end
1561
1562 def appendPass(plan)
1563     File.open($outputDir + "passed", "a") {
1564         | outp |
1565         outp.puts plan.name
1566     }
1567     $numPasses += 1
1568 end
1569
1570 def appendResult(plan, didPass)
1571     File.open($outputDir + "results", "a") {
1572         | outp |
1573         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1574     }
1575 end
1576
1577 def prepareBundle
1578     raise if $bundle
1579
1580     if $doNotMessWithVMPath
1581         if !$remote and !$tarball
1582             $testingFrameworkPath = (frameworkFromJSCPath($jscPath) || $jscPath.dirname).realpath
1583             $jscPath = Pathname.new($jscPath).realpath
1584         else
1585             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1586         end
1587     else
1588         originalJSCPath = $jscPath
1589         vmDir = $outputDir + ".vm"
1590         FileUtils.mkdir_p vmDir
1591         
1592         frameworkPath = frameworkFromJSCPath($jscPath)
1593         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1594         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1595         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1596
1597         if frameworkPath
1598             source = frameworkPath
1599             destination = Pathname.new(".vm")
1600         elsif $hostOS == "windows"
1601             # Make sure to copy dll along with jsc on Windows
1602             originalJSCDir = File.dirname(originalJSCPath)
1603             source = [originalJSCPath] + [originalJSCDir + "/jscLib.dll"]
1604
1605             # Check for and copy JavaScriptCore.dll and WTF.dll for dynamic builds
1606             javaScriptCoreDLLPath = File.join(originalJSCDir, "JavaScriptCore.dll")
1607             wtfDLLPath = File.join(originalJSCDir, "WTF.dll")
1608             if (File.exists?(javaScriptCoreDLLPath))
1609                 source = source + [javaScriptCoreDLLPath]
1610             end
1611             if (File.exists?(wtfDLLPath))
1612                 source = source + [wtfDLLPath]
1613             end
1614
1615             destination = $jscPath.dirname
1616
1617             Dir.chdir($outputDir) {
1618                 FileUtils.mkdir_p destination
1619             }
1620         else
1621             source = originalJSCPath
1622             destination = $jscPath
1623
1624             Dir.chdir($outputDir) {
1625                 FileUtils.mkdir_p $jscPath.dirname
1626             }
1627         end
1628
1629         Dir.chdir($outputDir) {
1630             if $copyVM
1631                 FileUtils.cp_r source, destination
1632             else
1633                 begin 
1634                     FileUtils.ln_s source, destination
1635                 rescue Exception
1636                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1637                     FileUtils.cp_r source, destination
1638                 end
1639             end
1640
1641             if $remote and $hostOS == "linux"
1642                 begin
1643                     dependencies = `ldd #{source}`
1644                     dependencies.split(/\n/).each {
1645                         | dependency |
1646                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1647                     }
1648                 rescue
1649                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1650                 end
1651             end
1652         }
1653     end
1654     
1655     Dir.chdir($outputDir) {
1656         FileUtils.cp_r HELPERS_PATH, ".helpers"
1657     }
1658
1659     ARGV.each {
1660         | collection |
1661         handleCollection(collection)
1662     }
1663
1664     puts
1665 end
1666
1667 def cleanOldResults
1668     raise unless $bundle
1669
1670     eachResultFile($outputDir) {
1671         | path |
1672         FileUtils.rm_f path
1673     }
1674 end
1675
1676 def cleanEmptyResultFiles
1677     eachResultFile($outputDir) {
1678         | path |
1679         next unless path.basename.to_s =~ /\.out$/
1680         next unless FileTest.size(path) == 0
1681         FileUtils.rm_f path
1682     }
1683 end
1684
1685 def eachResultFile(startingDir, &block)
1686     dirsToClean = [startingDir]
1687     until dirsToClean.empty?
1688         nextDir = dirsToClean.pop
1689         Dir.foreach(nextDir) {
1690             | entry |
1691             next if entry =~ /^\./
1692             path = nextDir + entry
1693             if path.directory?
1694                 dirsToClean.push(path)
1695             else
1696                 block.call(path)
1697             end
1698         }
1699     end
1700 end
1701
1702 def prepareTestRunner
1703     raise if $bundle
1704
1705     $runlist.each_with_index {
1706         | plan, index |
1707         plan.index = index
1708     }
1709
1710     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1711     toDelete = []
1712     Dir.foreach($runnerDir) {
1713         | filename |
1714         if filename =~ /^test_/
1715             toDelete << filename
1716         end
1717     }
1718     
1719     toDelete.each {
1720         | filename |
1721         File.unlink($runnerDir + filename)
1722     }
1723
1724     $runlist.each {
1725         | plan |
1726         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1727     }
1728
1729     case $testRunnerType
1730     when :make
1731         prepareMakeTestRunner
1732     when :shell
1733         prepareShellTestRunner
1734     when :ruby
1735         prepareRubyTestRunner
1736     else
1737         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1738     end
1739 end
1740
1741 def cleanRunnerDirectory
1742     raise unless $bundle
1743     Dir.foreach($runnerDir) {
1744         | filename |
1745         next unless filename =~ /^test_fail/
1746         FileUtils.rm_f $runnerDir + filename
1747     }
1748 end
1749
1750 def sshRead(cmd)
1751     raise unless $remote
1752
1753     result = ""
1754     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1755       | inp |
1756       inp.each_line {
1757         | line |
1758         result += line
1759       }
1760     }
1761     raise "#{$?}" unless $?.success?
1762     result
1763 end
1764
1765 def runCommandOnTester(cmd)
1766     if $remote
1767         result = sshRead(cmd)
1768     else
1769         result = `#{cmd}`
1770     end
1771 end
1772
1773 def numberOfProcessors
1774     if $hostOS == "windows"
1775         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1776     else
1777         begin
1778             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1779         rescue
1780             numProcessors = 0
1781         end
1782
1783         if numProcessors == 0
1784             begin
1785                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1786             rescue
1787                 numProcessors == 0
1788             end
1789         end
1790     end
1791
1792     if numProcessors == 0
1793         numProcessors = 1
1794     end
1795     return numProcessors
1796 end
1797
1798 def runAndMonitorTestRunnerCommand(*cmd)
1799     numberOfTests = 0
1800     Dir.chdir($runnerDir) {
1801         # -1 for the runscript, and -2 for '..' and '.'
1802         numberOfTests = Dir.entries(".").count - 3
1803     }
1804     unless $progressMeter
1805         mysys(cmd.join(' '))
1806     else
1807        running = {}
1808        didRun = {}
1809        didFail = {}
1810        blankLine = true
1811        prevStringLength = 0
1812        IO.popen(cmd.join(' '), mode="r") {
1813            | inp |
1814            inp.each_line {
1815                | line |
1816                line = line.scrub.chomp
1817                if line =~ /^Running /
1818                    running[$~.post_match] = true
1819                elsif line =~ /^PASS: /
1820                    didRun[$~.post_match] = true
1821                elsif line =~ /^FAIL: /
1822                    didRun[$~.post_match] = true
1823                    didFail[$~.post_match] = true
1824                else
1825                    unless blankLine
1826                        print("\r" + " " * prevStringLength + "\r")
1827                    end
1828                    puts line
1829                    blankLine = true
1830                end
1831
1832                def lpad(str, chars)
1833                    str = str.to_s
1834                    if str.length > chars
1835                        str
1836                    else
1837                       "%#{chars}s"%(str)
1838                    end
1839                end
1840
1841                string  = ""
1842                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1843                unless didFail.empty?
1844                    string += " (failed #{didFail.size})"
1845                end
1846                string += " "
1847                (running.size - didRun.size).times {
1848                    string += "."
1849                }
1850                if string.length < prevStringLength
1851                    print string
1852                    print(" " * (prevStringLength - string.length))
1853                end
1854                print string
1855                prevStringLength = string.length
1856                blankLine = false
1857                $stdout.flush
1858            }
1859        }
1860        puts
1861        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1862     end
1863 end
1864
1865 def runTestRunner
1866     if $remote
1867         if !$remoteDirectory
1868             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1869         end
1870         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1871         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1872         remoteScript = "\""
1873         remoteScript += "cd #{$remoteDirectory} && "
1874         remoteScript += "rm -rf #{$outputDir.basename} && "
1875         remoteScript += "tar xzf #{$tarFileName} && "
1876         remoteScript += "cd #{$outputDir.basename}/.runner && "
1877         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1878         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1879         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
1880         $envVars.each { |var| remoteScript += "export " << var << "\n" }
1881         remoteScript += "#{testRunnerCommand}\""
1882         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1883     else
1884         Dir.chdir($runnerDir) {
1885             runAndMonitorTestRunnerCommand(testRunnerCommand)
1886         }
1887     end
1888 end
1889
1890 def detectFailures
1891     raise if $bundle
1892
1893     failures = []
1894     
1895     if $remote
1896         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1897         output.split(/\n/).each {
1898             | line |
1899             next unless line =~ /test_fail_/
1900             failures << $~.post_match.to_i
1901         }
1902     else
1903         Dir.foreach($runnerDir) {
1904             | filename |
1905             next unless filename =~ /test_fail_/
1906             failures << $~.post_match.to_i
1907         }
1908     end
1909
1910     failureSet = {}
1911
1912     failures.each {
1913         | failure | 
1914         appendFailure($runlist[failure])
1915         failureSet[failure] = true
1916     }
1917
1918     familyMap = {}
1919     $runlist.each_with_index {
1920         | plan, index |
1921         unless familyMap[plan.family]
1922             familyMap[plan.family] = []
1923         end
1924         if failureSet[index]
1925             appendResult(plan, false)
1926             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
1927             next
1928         else
1929             appendResult(plan, true)
1930             familyMap[plan.family] << {:result => "PASS", :plan => plan};
1931         end
1932         appendPass(plan)
1933     }
1934
1935     File.open($outputDir + "resultsByFamily", "w") {
1936         | outp |
1937         first = true
1938         familyMap.keys.sort.each {
1939             | familyName |
1940             if first
1941                 first = false
1942             else
1943                 outp.puts
1944             end
1945             
1946             outp.print "#{familyName}:"
1947
1948             numPassed = 0
1949             familyMap[familyName].each {
1950                 | entry |
1951                 if entry[:result] == "PASS"
1952                     numPassed += 1
1953                 end
1954             }
1955
1956             if numPassed == familyMap[familyName].size
1957                 outp.puts " PASSED"
1958             elsif numPassed == 0
1959                 outp.puts " FAILED"
1960             else
1961                 outp.puts
1962                 familyMap[familyName].each {
1963                     | entry |
1964                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
1965                 }
1966             end
1967         }
1968     }
1969 end
1970
1971 def compressBundle
1972     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
1973     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1974     raise unless system(cmd)
1975 end
1976
1977 def clean(file)
1978     FileUtils.rm_rf file unless $bundle
1979 end
1980
1981 clean($outputDir + "failed")
1982 clean($outputDir + "passed")
1983 clean($outputDir + "results")
1984 clean($outputDir + "resultsByFamily")
1985 clean($outputDir + ".vm")
1986 clean($outputDir + ".helpers")
1987 clean($outputDir + ".runner")
1988 clean($outputDir + ".tests")
1989 clean($outputDir + "_payload")
1990
1991 Dir.mkdir($outputDir) unless $outputDir.directory?
1992
1993 $outputDir = $outputDir.realpath
1994 $runnerDir = $outputDir + ".runner"
1995
1996 if !$numChildProcesses
1997     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
1998         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
1999     else
2000         $numChildProcesses = numberOfProcessors
2001     end
2002 end
2003
2004 if ENV["JSCTEST_timeout"]
2005     # In the worst case, the processors just interfere with each other.
2006     # Increase the timeout proportionally to the number of processors.
2007     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
2008 end
2009
2010 def runBundle
2011     raise unless $bundle
2012
2013     cleanRunnerDirectory
2014     cleanOldResults
2015     runTestRunner
2016     cleanEmptyResultFiles
2017 end
2018
2019 def runNormal
2020     raise if $bundle or $tarball
2021
2022     prepareBundle
2023     prepareTestRunner
2024     runTestRunner
2025     cleanEmptyResultFiles
2026     detectFailures
2027 end
2028
2029 def runTarball
2030     raise unless $tarball
2031
2032     prepareBundle 
2033     prepareTestRunner
2034     compressBundle
2035 end
2036
2037 def runRemote
2038     raise unless $remote
2039
2040     prepareBundle
2041     prepareTestRunner
2042     compressBundle
2043     runTestRunner
2044     detectFailures
2045 end
2046
2047 puts
2048 if $bundle
2049     runBundle
2050 elsif $remote
2051     runRemote
2052 elsif $tarball
2053     runTarball
2054 else
2055     runNormal
2056 end