f55746f94a75c464daf30a8baaa86ca2d8eb5004
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2016 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
47 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
48 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
49 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
50 WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
51 CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
52 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
53 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
54
55 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
56
57 begin
58     require 'shellwords'
59 rescue Exception => e
60     $stderr.puts "Warning: did not find shellwords, not running any tests."
61     exit 0
62 end
63
64 $canRunDisplayProfilerOutput = false
65
66 begin
67     require 'rubygems'
68     require 'json'
69     require 'highline'
70     $canRunDisplayProfilerOutput = true
71 rescue Exception => e
72     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
73     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
74     $stderr.puts "Error: #{e.inspect}"
75 end
76
77 def printCommandArray(*cmd)
78     begin
79         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
80     rescue
81         commandArray = cmd.join(' ')
82     end
83     $stderr.puts ">> #{commandArray}"
84 end
85
86 def mysys(*cmd)
87     printCommandArray(*cmd) if $verbosity >= 1
88     raise "Command failed: #{$?.inspect}" unless system(*cmd)
89 end
90
91 def escapeAll(array)
92     array.map {
93         | v |
94         raise "Detected a non-string in #{inspect}" unless v.is_a? String
95         Shellwords.shellescape(v)
96     }.join(' ')
97 end
98
99
100 $jscPath = nil
101 $doNotMessWithVMPath = false
102 $jitTests = true
103 $memoryLimited = false
104 $outputDir = Pathname.new("results")
105 $verbosity = 0
106 $bundle = nil
107 $tarball = false
108 $tarFileName = "payload.tar.gz"
109 $copyVM = false
110 $testRunnerType = nil
111 $testWriter = "default"
112 $remoteUser = nil
113 $remoteHost = nil
114 $remotePort = nil
115 $remoteDirectory = nil
116 $architecture = nil
117 $hostOS = nil
118 $filter = nil
119 $envVars = []
120 $mode = "full"
121 $buildType = "release"
122 $forceCollectContinuously = false
123
124 def usage
125     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
126     puts
127     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
128     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
129     puts "                            --jsc specifies an already present JavaScriptCore to test."
130     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
131     puts "                            Skip tests tagged with //@skip if $memoryLimited"
132     puts "--no-jit                    Do not run JIT specific tests."
133     puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
134     puts "                            platform."
135     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
136     puts "--verbose            (-v)   Print more things while running."
137     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
138     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
139     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
140     puts "                            e.g. x86, x86_64, arm."
141     puts "--os                        Specify os instead of determining from JavaScriptCore build."
142     puts "                            e.g. darwin, linux & windows."
143     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
144     puts "                            In general the shell runner is slower than the make runner."
145     puts "--make-runner               Uses the faster make-based runner."
146     puts "--ruby-runner               Uses the ruby runner for machines without unix shell or make."
147     puts "--test-writer [writer]      Specifies the test script format."
148     puts "                            default is to use shell scripts to run the tests"
149     puts "                            \"ruby\" to use ruby scripts for systems without a unix shell."
150     puts "--remote                    Specify a remote host on which to run tests from command line argument."
151     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
152     puts "--child-processes    (-c)   Specify the number of child processes."
153     puts "--filter                    Only run tests whose name matches the given regular expression."
154     puts "--help               (-h)   Print this message."
155     puts "--env-vars                  Add a list of environment variables to set before running jsc."
156     puts "                            Each environment variable should be separated by a space."
157     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
158     puts "                            it will override the default value."
159     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
160     puts "--basic                     Run with default and these additional modes: no-llint,"
161     puts "                            no-cjit-validate-phases, no-cjit-collect-continuously, dfg-eager"
162     puts "                            and for FTL platforms: no-ftl, ftl-eager-no-cjit and"
163     puts "                            ftl-no-cjit-small-pool."
164     exit 1
165 end
166
167 jscArg = nil
168
169 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
170                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
171                ['--no-copy', GetoptLong::NO_ARGUMENT],
172                ['--memory-limited', GetoptLong::NO_ARGUMENT],
173                ['--no-jit', GetoptLong::NO_ARGUMENT],
174                ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
175                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
176                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
177                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
178                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
179                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
180                ['--os', GetoptLong::REQUIRED_ARGUMENT],
181                ['--shell-runner', GetoptLong::NO_ARGUMENT],
182                ['--make-runner', GetoptLong::NO_ARGUMENT],
183                ['--ruby-runner', GetoptLong::NO_ARGUMENT],
184                ['--test-writer', GetoptLong::REQUIRED_ARGUMENT],
185                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
186                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
187                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
188                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
189                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
190                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
191                ['--debug', GetoptLong::NO_ARGUMENT],
192                ['--release', GetoptLong::NO_ARGUMENT],
193                ['--quick', '-q', GetoptLong::NO_ARGUMENT],
194                ['--basic', GetoptLong::NO_ARGUMENT]).each {
195     | opt, arg |
196     case opt
197     when '--help'
198         usage
199     when '--jsc'
200         jscArg = arg
201     when '--no-copy'
202         $doNotMessWithVMPath = true
203     when '--output-dir'
204         $outputDir = Pathname.new(arg)
205     when '--memory-limited'
206         $memoryLimited = true
207     when '--no-jit'
208         $jitTests = false
209     when '--force-collectContinuously'
210         $forceCollectContinuously = true;
211     when '--verbose'
212         $verbosity += 1
213     when '--run-bundle'
214         $bundle = Pathname.new(arg)
215     when '--tarball'
216         $tarball = true
217         $copyVM = true
218         $tarFileName = arg unless arg == ''
219     when '--force-vm-copy'
220         $copyVM = true
221     when '--shell-runner'
222         $testRunnerType = :shell
223     when '--make-runner'
224         $testRunnerType = :make
225     when '--ruby-runner'
226         $testRunnerType = :ruby
227     when '--test-writer'
228         $testWriter = arg
229     when '--remote'
230         $copyVM = true
231         $tarball = true
232         $remote = true
233         uri = URI("ssh://" + arg)
234         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
235     when '--remote-config-file'
236         $remoteConfigFile = arg
237     when '--child-processes'
238         $numChildProcesses = arg.to_i
239     when '--filter'
240         $filter = Regexp.new(arg)
241     when '--arch'
242         $architecture = arg
243     when '--os'
244         $hostOS = arg
245     when '--env-vars'
246         $envVars = arg.gsub(/\s+/, ' ').split(' ')
247     when '--quick'
248         $mode = "quick"
249     when '--basic'
250         $mode = "basic"
251     when '--debug'
252         $buildType = "debug"
253     when '--release'
254         $buildType = "release"
255     end
256 }
257
258 if $remoteConfigFile
259     file = File.read($remoteConfigFile)
260     config = JSON.parse(file)
261
262     if !$remote and config['remote']
263         $copyVM = true
264         $tarball = true
265         $remote = true
266         uri = URI("ssh://" + config['remote'])
267         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
268     end
269
270     if config['remoteDirectory']
271         $remoteDirectory = config['remoteDirectory']
272     end
273 end
274
275 unless jscArg
276     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
277     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
278     command += ($buildType == "release") ? " --release" : " --debug"
279     command += " --executablePath"
280
281     output = `#{command}`.split("\n")
282     if !output.length
283         $stderr.puts "Error: must specify --jsc <path>"
284         exit 1
285     end
286
287     output = output[0]
288     jscArg = Pathname.new(output).join("jsc")
289     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
290     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
291     if !File.file?(jscArg)
292         $stderr.puts "Error: must specify --jsc <path>"
293         exit 1
294     end
295
296     puts "Using the following jsc path: #{jscArg}"
297 end
298
299 if $doNotMessWithVMPath
300     $jscPath = Pathname.new(jscArg)
301 else
302     $jscPath = Pathname.new(jscArg).realpath
303 end
304
305 $progressMeter = ($verbosity == 0 and $stdout.tty?)
306
307 if $bundle
308     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
309     $outputDir = $bundle
310 end
311
312 # Try to determine architecture. Return nil on failure.
313 def machOArchitectureCode
314     begin 
315         otoolLines = `otool -afh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
316         otoolLines.each_with_index {
317             | value, index |
318             if value =~ /magic/ and value =~ /cputype/
319                 return otoolLines[index + 1].split[1].to_i
320             end
321         }
322     rescue
323         $stderr.puts "Warning: unable to execute otool."
324     end
325     $stderr.puts "Warning: unable to determine architecture."
326     nil
327 end
328
329 def determineArchitectureFromMachOBinary
330     code = machOArchitectureCode
331     return nil unless code
332     is64BitFlag = 0x01000000
333     case code
334     when 7
335         "x86"
336     when 7 | is64BitFlag
337         "x86-64"
338     when 12
339         "arm"
340     when 12 | is64BitFlag
341         "arm64"
342     else
343         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
344         nil
345     end
346 end
347
348 def determineArchitectureFromELFBinary
349     f = File.open($jscPath.to_s)
350     data = f.read(19)
351
352     if !(data[0,4] == "\x7F\x45\x4C\x46")
353         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
354         return nil
355     end
356
357     code = data[18].ord
358     case code
359     when 3
360         "x86"
361     when 8
362         "mips"
363     when 62
364         "x86-64"
365     when 40
366         "arm"
367     when 183
368         "arm64"
369     else
370         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
371         nil
372     end
373 end
374
375 def determineArchitectureFromPEBinary
376     f = File.open($jscPath.to_s)
377     data = f.read(1024)
378
379     if !(data[0, 2] == "MZ")
380         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
381         return nil
382     end
383
384     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
385
386     if !(data[peHeaderAddr, 4] == "PE\0\0")
387         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
388         return nil
389     end
390
391     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
392
393     case machine
394     when 0x014c
395         "x86"
396     when 0x8664
397         "x86-64"
398     else
399         $stderr.puts "Warning: unsupported machine type: #{machine}"
400         nil
401     end
402 end
403
404 def determineArchitecture
405     case $hostOS
406     when "darwin"
407         determineArchitectureFromMachOBinary
408     when "linux"
409         determineArchitectureFromELFBinary
410     when "windows"
411         determineArchitectureFromPEBinary
412     else
413         $stderr.puts "Warning: unable to determine architecture on this platform."
414         nil
415     end
416 end
417
418 def determineOS
419     case RbConfig::CONFIG["host_os"]
420     when /darwin/i
421         "darwin"
422     when /linux/i
423         "linux"
424     when /mswin|mingw|cygwin/
425         "windows"
426     else
427         $stderr.puts "Warning: unable to determine host operating system"
428         nil
429     end
430 end
431
432 $hostOS = determineOS unless $hostOS
433 $architecture = determineArchitecture unless $architecture
434 $isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $architecture == "mips" || $hostOS == "windows")
435
436 if !$testRunnerType
437     if $remote and $hostOS == "darwin"
438         $testRunnerType = :shell
439     else
440         $testRunnerType = :make
441     end
442 end
443
444 if $testWriter
445     if /[^-a-zA-Z0-9_]/.match($testWriter)
446         raise "Invalid test writer #{$testWriter} given"
447     end
448 end
449
450 $numFailures = 0
451 $numPasses = 0
452
453 # We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
454 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--maxPerThreadStackUsage=1572864"]
455 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
456 # NOTE: Tests rely on this using scribbleFreeCells.
457 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
458 B3O1_OPTIONS = ["--defaultB3OptLevel=1"]
459 FTL_OPTIONS = ["--useFTLJIT=true"]
460
461 require_relative "webkitruby/jsc-stress-test-writer-#{$testWriter}"
462
463 def shouldCollectContinuously?
464     $buildType == "release" or $forceCollectContinuously
465 end
466
467 COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false"] : []
468
469 $runlist = []
470
471 def frameworkFromJSCPath(jscPath)
472     parentDirectory = jscPath.dirname
473     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
474         parentDirectory.dirname
475     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
476         jscPath.dirname + "JavaScriptCore.framework"
477     else
478         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
479         nil
480     end
481 end
482
483 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
484     dir = Pathname.new(".")
485     $benchmarkDirectory.each_filename {
486         | pathComponent |
487         dir += ".."
488     }
489     dir + resourcePath
490 end
491
492 def pathToVM
493     pathToBundleResourceFromBenchmarkDirectory($jscPath)
494 end
495
496 def pathToHelpers
497     pathToBundleResourceFromBenchmarkDirectory(".helpers")
498 end
499
500 $runCommandOptions = {}
501
502 $uniqueFilenameCounter = 0
503 def uniqueFilename(extension)
504     payloadDir = $outputDir + "_payload"
505     Dir.mkdir payloadDir unless payloadDir.directory?
506     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
507     $uniqueFilenameCounter += 1
508     result
509 end
510
511 def baseOutputName(kind)
512     "#{$collectionName}/#{$benchmark}.#{kind}"
513 end
514
515 def addRunCommand(kind, command, outputHandler, errorHandler)
516     $didAddRunCommand = true
517     name = baseOutputName(kind)
518     if $filter and name !~ $filter
519         return
520     end
521     plan = Plan.new(
522         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
523         errorHandler)
524     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
525         $runlist.unshift plan
526     else
527         $runlist << plan
528     end
529 end
530
531 # Returns true if there were run commands found in the file ($benchmarkDirectory +
532 # $benchmark), in which case those run commands have already been executed. Otherwise
533 # returns false, in which case you're supposed to add your own run commands.
534 def parseRunCommands
535     oldDidAddRunCommand = $didAddRunCommand
536     $didAddRunCommand = false
537
538     Dir.chdir($outputDir) {
539         File.open($benchmarkDirectory + $benchmark) {
540             | inp |
541             inp.each_line {
542                 | line |
543                 begin
544                     doesMatch = line =~ /^\/\/@/
545                 rescue Exception => e
546                     # Apparently this happens in the case of some UTF8 stuff in some files, where
547                     # Ruby tries to be strict and throw exceptions.
548                     next
549                 end
550                 next unless doesMatch
551                 eval $~.post_match
552             }
553         }
554     }
555
556     result = $didAddRunCommand
557     $didAddRunCommand = result or oldDidAddRunCommand
558     result
559 end
560
561 def slow!
562     $runCommandOptions[:isSlow] = true
563 end
564
565 def runWithOutputHandler(kind, outputHandler, *options)
566     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
567 end
568
569 def run(kind, *options)
570     runWithOutputHandler(kind, silentOutputHandler, *options)
571 end
572
573 def runNoFTL(*optionalTestSpecificOptions)
574     run("no-ftl", *optionalTestSpecificOptions)
575 end
576
577 def runWithRAMSize(size, *optionalTestSpecificOptions)
578     run("ram-size-#{size}", "--forceRAMSize=#{size}", *optionalTestSpecificOptions)
579 end
580
581 def runOneLargeHeap(*optionalTestSpecificOptions)
582     if $memoryLimited
583         $didAddRunCommand = true
584         puts "Skipping #{$collectionName}/#{$benchmark}"
585     else
586         run("default", *optionalTestSpecificOptions)
587     end
588 end
589
590 def runNoJIT(*optionalTestSpecificOptions)
591     run("no-jit", "--useJIT=false", *optionalTestSpecificOptions)
592 end
593
594 def runNoLLInt(*optionalTestSpecificOptions)
595     if $jitTests
596         run("no-llint", "--useLLInt=false", *optionalTestSpecificOptions)
597     end
598 end
599
600 # NOTE: Tests rely on this using scribbleFreeCells.
601 def runNoCJITValidate(*optionalTestSpecificOptions)
602     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
603 end
604
605 def runNoCJITValidatePhases(*optionalTestSpecificOptions)
606     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
607 end
608
609 def runNoCJITCollectContinuously(*optionalTestSpecificOptions)
610     run("no-cjit-collect-continuously", *(NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
611 end
612
613 def runDefault(*optionalTestSpecificOptions)
614     run("default", *(FTL_OPTIONS + optionalTestSpecificOptions))
615 end
616
617 def runFTLNoCJIT(*optionalTestSpecificOptions)
618     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
619 end
620
621 def runFTLNoCJITB3O1(*optionalTestSpecificOptions)
622     run("ftl-no-cjit-b3o1", "--useArrayAllocationProfiling=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
623 end
624
625 def runFTLNoCJITValidate(*optionalTestSpecificOptions)
626     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
627 end
628
629 def runFTLNoCJITNoPutStackValidate(*optionalTestSpecificOptions)
630     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
631 end
632
633 def runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
634     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
635 end
636
637 def runFTLNoCJITOSRValidation(*optionalTestSpecificOptions)
638     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
639 end
640
641 def runDFGEager(*optionalTestSpecificOptions)
642     run("dfg-eager", *(EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
643 end
644
645 def runDFGEagerNoCJITValidate(*optionalTestSpecificOptions)
646     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
647 end
648
649 def runFTLEager(*optionalTestSpecificOptions)
650     run("ftl-eager", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
651 end
652
653 def runFTLEagerWatchdog(*optionalTestSpecificOptions)
654     timeout = rand(100)
655     run("ftl-eager-watchdog-#{timeout}", "--watchdog=#{timeout}", "--watchdog-exception-ok", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
656 end
657
658 def runFTLEagerNoCJITValidate(*optionalTestSpecificOptions)
659     run("ftl-eager-no-cjit", "--validateGraph=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
660 end
661
662 def runFTLEagerNoCJITB3O1(*optionalTestSpecificOptions)
663     run("ftl-eager-no-cjit-b3o1", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
664 end
665
666 def runFTLEagerNoCJITOSRValidation(*optionalTestSpecificOptions)
667     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
668 end
669
670 def runNoCJITNoASO(*optionalTestSpecificOptions)
671     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
672 end
673
674 def runNoCJITNoAccessInlining(*optionalTestSpecificOptions)
675     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
676 end
677
678 def runFTLNoCJITNoAccessInlining(*optionalTestSpecificOptions)
679     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
680 end
681
682 def runFTLNoCJITSmallPool(*optionalTestSpecificOptions)
683     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
684 end
685
686 def runNoCJIT(*optionalTestSpecificOptions)
687     run("no-cjit", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
688 end
689
690 def runDFGMaximalFlushPhase(*optionalTestSpecificOptions)
691     run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
692 end
693
694 def runShadowChicken(*optionalTestSpecificOptions)
695     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true", *optionalTestSpecificOptions)
696 end
697
698 def defaultRun
699     if $mode == "quick"
700         defaultQuickRun
701     else
702         runDefault
703         if $jitTests
704             runNoLLInt
705             runNoCJITValidatePhases
706             runNoCJITCollectContinuously if shouldCollectContinuously?
707             runDFGEager
708             if $mode != "basic"
709                 runDFGEagerNoCJITValidate
710                 runDFGMaximalFlushPhase
711             end
712
713             return if !$isFTLPlatform
714
715             runNoFTL
716             runFTLEager
717             runFTLEagerNoCJITValidate
718             runFTLNoCJITSmallPool
719
720             return if $mode == "basic"
721
722             runFTLNoCJITValidate
723             runFTLNoCJITB3O1
724             runFTLNoCJITNoPutStackValidate
725             runFTLNoCJITNoInlineValidate
726             runFTLEagerNoCJITB3O1
727         end
728     end
729 end
730
731 def defaultNoNoLLIntRun
732     if $mode == "quick"
733         defaultQuickRun
734     else
735         runDefault
736         if $jitTests
737             runNoCJITValidatePhases
738             runNoCJITCollectContinuously if shouldCollectContinuously?
739             runDFGEager
740             if $mode != "basic"
741                 runDFGEagerNoCJITValidate
742                 runDFGMaximalFlushPhase
743             end
744
745             return if !$isFTLPlatform
746
747             runNoFTL
748             runFTLNoCJITValidate
749             runFTLNoCJITSmallPool
750
751             return if $mode == "basic"
752
753             runFTLNoCJITB3O1
754             runFTLNoCJITNoPutStackValidate
755             runFTLNoCJITNoInlineValidate
756             runFTLEager
757             runFTLEagerNoCJITValidate
758         end
759     end
760 end
761
762 def defaultQuickRun
763     runDefault
764     if $jitTests
765         runNoCJITValidate
766
767         return if !$isFTLPlatform
768
769         runNoFTL
770         runFTLNoCJITValidate
771     end
772 end
773
774 def defaultSpotCheckNoMaximalFlush
775     defaultQuickRun
776     runNoCJITNoAccessInlining
777
778     return if !$isFTLPlatform
779
780     runFTLNoCJITOSRValidation
781     runFTLNoCJITNoAccessInlining
782     runFTLNoCJITB3O1
783 end
784
785 def defaultSpotCheck
786     defaultSpotCheckNoMaximalFlush
787     runDFGMaximalFlushPhase
788 end
789
790 # This is expected to not do eager runs because eager runs can have a lot of recompilations
791 # for reasons that don't arise in the real world. It's used for tests that assert convergence
792 # by counting recompilations.
793 def defaultNoEagerRun
794     runDefault
795     if $jitTests
796         runNoLLInt
797         runNoCJITValidatePhases
798         runNoCJITCollectContinuously if shouldCollectContinuously?
799
800         return if !$isFTLPlatform
801
802         runNoFTL
803         runFTLNoCJITValidate
804
805         return if $mode == "basic"
806
807         runFTLNoCJITNoInlineValidate
808         runFTLNoCJITB3O1
809     end
810 end
811
812 def defaultNoSamplingProfilerRun
813     runDefault
814     if $jitTests
815         runNoLLInt
816         runNoCJITValidatePhases
817         runNoCJITCollectContinuously if shouldCollectContinuously?
818         runDFGEager
819         runDFGEagerNoCJITValidate
820         runDFGMaximalFlushPhase
821
822         return if !$isFTLPlatform
823
824         runNoFTL
825         runFTLNoCJITNoPutStackValidate
826         runFTLNoCJITNoInlineValidate
827         runFTLEager
828         runFTLEagerNoCJITValidate
829         runFTLNoCJITSmallPool
830     end
831 end
832
833 def runProfiler
834     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
835         skip
836         return
837     end
838
839     profilerOutput = uniqueFilename(".json")
840     if $canRunDisplayProfilerOutput
841         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
842     else
843         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
844         run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
845     end
846 end
847
848 def runExceptionFuzz
849     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
850     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
851 end
852
853 def runExecutableAllocationFuzz(name, *options)
854     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
855     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
856 end
857
858 def runTypeProfiler
859     if !$jitTests
860         return
861     end
862
863     return if !$isFTLPlatform
864
865     run("ftl-no-cjit-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
866     run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
867 end
868
869 def runControlFlowProfiler
870     if !$jitTests
871         return
872     end
873
874     return if !$isFTLPlatform
875
876     run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
877 end
878
879 def runTest262(mode, exception, includeFiles, flags)
880     failsWithException = exception != "NoException"
881     isStrict = false
882     isModule = false
883     isAsync = false
884
885     flags.each {
886         | flag |
887         case flag
888         when :strict
889             isStrict = true
890         when :module
891             isModule = true
892         when :async
893             isAsync = true
894         else
895             raise "Invalid flag for runTest262, #{flag}"
896         end
897     }
898
899     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
900
901     args = [pathToVM.to_s] + BASE_OPTIONS
902     args << "--exception=" + exception if failsWithException
903     args << "--test262-async" if isAsync
904     args += includeFiles
905
906     case mode
907     when :normal
908         errorHandler = simpleErrorHandler
909         outputHandler = silentOutputHandler
910     when :fail
911         errorHandler = expectedFailErrorHandler
912         outputHandler = noisyOutputHandler
913     when :failDueToOutdatedOrBadTest
914         errorHandler = expectedFailErrorHandler
915         outputHandler = noisyOutputHandler
916     else
917         raise "Invalid mode: #{mode}"
918     end
919
920     if isStrict
921         kind = "default-strict"
922         args << "--strict-file=#{$benchmark}"
923     else
924         kind = "default"
925         if isModule
926             args << "--module-file=#{$benchmark}"
927         else
928             args << $benchmark.to_s
929         end
930     end
931
932     addRunCommand(kind, args, outputHandler, errorHandler)
933 end
934
935 def prepareTest262Fixture
936     # This function is used to add the files used by Test262 modules tests.
937     prepareExtraRelativeFiles([""], $collection)
938 end
939
940 def runES6(mode)
941     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
942     case mode
943     when :normal
944         errorHandler = simpleErrorHandler
945     when :fail
946         errorHandler = expectedFailErrorHandler
947     when :failDueToOutdatedOrBadTest
948         errorHandler = expectedFailErrorHandler
949     else
950         raise "Invalid mode: #{mode}"
951     end
952     addRunCommand("default", args, noisyOutputHandler, errorHandler)
953 end
954
955 def runModules
956     run("default-modules", "-m")
957
958     if !$jitTests
959         return
960     end
961
962     run("no-llint-modules", "-m", "--useLLInt=false")
963     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
964     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
965     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
966
967     return if !$isFTLPlatform
968
969     run("default-ftl-modules", "-m", *FTL_OPTIONS)
970     run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
971     run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
972     run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
973     run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
974     run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
975 end
976
977 def runWebAssembly
978     return if !$jitTests
979     return if !$isFTLPlatform
980     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
981     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
982     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
983     run("default-wasm", "-m", *FTL_OPTIONS)
984     if $mode != "quick"
985         run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
986         run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
987         run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
988         run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
989         run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
990     end
991 end
992
993 def runWebAssemblyEmscripten(mode)
994     case mode
995     when :skip
996         return
997     end
998     return if !$jitTests
999     return if !$isFTLPlatform
1000     wasm = $benchmark.to_s.sub! '.js', '.wasm'
1001     prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
1002     run("default-wasm", *FTL_OPTIONS)
1003     if $mode != "quick"
1004         run("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1005         run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1006         run("wasm-no-call-ic", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1007         run("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1008     end
1009 end
1010
1011 def runWebAssemblySpecTest(mode)
1012     case mode
1013     when :skip
1014         return
1015     end
1016     return if !$jitTests
1017     return if !$isFTLPlatform
1018     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1019
1020     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1021     prepareExtraRelativeFiles(modules.map { |f| "../../" + f }, $collection)
1022
1023     harness = Dir[WASMTESTS_PATH + "spec-harness/" + "*.js"].map { |f| File.basename(f) }
1024     prepareExtraRelativeFiles(harness.map { |f| "../../spec-harness/" + f }, $collection)
1025
1026     runWithOutputHandler("default-wasm", noisyOutputHandler, "../spec-harness.js", *FTL_OPTIONS)
1027     if $mode != "quick"
1028       runWithOutputHandler("wasm-no-cjit-yes-tls-context", noisyOutputHandler, "../spec-harness.js",  "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1029       runWithOutputHandler("wasm-eager-jettison", noisyOutputHandler, "../spec-harness.js", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1030       runWithOutputHandler("wasm-no-call-ic", noisyOutputHandler, "../spec-harness.js", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1031       runWithOutputHandler("wasm-no-tls-context", noisyOutputHandler, "../spec-harness.js", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1032     end
1033 end
1034
1035 def runWebAssemblyLowExecutableMemory(*optionalTestSpecificOptions)
1036     return if !$jitTests
1037     return if !$isFTLPlatform
1038     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1039     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1040     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1041     # Only let WebAssembly get executable memory.
1042     run("default-wasm", "--useConcurrentGC=0" , "--useConcurrentJIT=0", "--jitMemoryReservationSize=15000", "--useBaselineJIT=0", "--useDFGJIT=0", "--useFTLJIT=0", "-m")
1043 end
1044
1045 def runChakra(mode, exception, baselineFile, extraFiles)
1046     raise unless $benchmark.to_s =~ /\.js$/
1047     failsWithException = exception != "NoException"
1048     testName = $~.pre_match
1049
1050     prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
1051     prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)
1052
1053     args = [pathToVM.to_s] + BASE_OPTIONS
1054     args += FTL_OPTIONS if $isFTLPlatform
1055     args += EAGER_OPTIONS
1056     args << "--exception=" + exception if failsWithException
1057     args << "--dumpException" if failsWithException
1058     args += ["jsc-lib.js"]
1059
1060     case mode
1061     when :baseline
1062         prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
1063         errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
1064         outputHandler = noisyOutputHandler
1065     when :pass
1066         errorHandler = chakraPassFailErrorHandler
1067         outputHandler = noisyOutputHandler
1068     when :skipDueToOutdatedOrBadTest
1069         return
1070     when :skip
1071         return
1072     else
1073         raise "Invalid mode: #{mode}"
1074     end
1075
1076     kind = "default"
1077     args << $benchmark.to_s
1078
1079     addRunCommand(kind, args, outputHandler, errorHandler)
1080 end
1081
1082 def runLayoutTest(kind, *options)
1083     raise unless $benchmark.to_s =~ /\.js$/
1084     testName = $~.pre_match
1085     if kind
1086         kind = "layout-" + kind
1087     else
1088         kind = "layout"
1089     end
1090
1091     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1092     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1093
1094     args = [pathToVM.to_s] + BASE_OPTIONS + options +
1095         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1096          $benchmark.to_s,
1097          (Pathname.new("resources") + "standalone-post.js").to_s]
1098     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1099 end
1100
1101 def runLayoutTestNoFTL
1102     runLayoutTest("no-ftl")
1103 end
1104
1105 def runLayoutTestNoLLInt
1106     runLayoutTest("no-llint", "--useLLInt=false")
1107 end
1108
1109 def runLayoutTestNoCJIT
1110     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1111 end
1112
1113 def runLayoutTestDFGEagerNoCJIT
1114     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1115 end
1116
1117 def runLayoutTestDefault
1118     runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
1119 end
1120
1121 def runLayoutTestFTLNoCJIT
1122     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1123 end
1124
1125 def runLayoutTestFTLEagerNoCJIT
1126     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1127 end
1128
1129 def runLayoutTestFTLEagerNoCJITB3O1
1130     runLayoutTest("ftl-eager-no-cjit-b3o1", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS))
1131 end
1132
1133 def noFTLRunLayoutTest
1134     if !$jitTests
1135         return
1136     end
1137
1138     runLayoutTestNoLLInt
1139     runLayoutTestNoCJIT
1140     runLayoutTestDFGEagerNoCJIT
1141 end
1142
1143 def defaultQuickRunLayoutTest
1144     runLayoutTestDefault
1145     if $jitTests
1146         if $isFTLPlatform
1147             runLayoutTestNoFTL
1148             runLayoutTestFTLNoCJIT
1149             runLayoutTestFTLEagerNoCJIT
1150         else
1151             noFTLRunLayoutTest
1152         end
1153     end
1154 end
1155
1156 def defaultRunLayoutTest
1157     if $mode == "quick"
1158         defaultQuickRunLayoutTest
1159     else
1160         runLayoutTestDefault
1161         if $jitTests
1162             noFTLRunLayoutTest
1163
1164             return if !$isFTLPlatform
1165
1166             runLayoutTestNoFTL
1167             runLayoutTestFTLNoCJIT
1168             runLayoutTestFTLEagerNoCJIT
1169         end
1170     end
1171 end
1172
1173 def noEagerNoNoLLIntTestsRunLayoutTest
1174     runLayoutTestDefault
1175     if $jitTests
1176         runLayoutTestNoCJIT
1177
1178         return if !$isFTLPlatform
1179
1180         runLayoutTestNoFTL
1181         runLayoutTestFTLNoCJIT
1182     end
1183 end
1184
1185 def noNoLLIntRunLayoutTest
1186     runLayoutTestDefault
1187     if $jitTests
1188         runLayoutTestNoCJIT
1189         runLayoutTestDFGEagerNoCJIT
1190
1191         return if !$isFTLPlatform
1192
1193         runLayoutTestNoFTL
1194         runLayoutTestFTLNoCJIT
1195         runLayoutTestFTLEagerNoCJIT
1196     end
1197 end
1198
1199 def prepareExtraRelativeFiles(extraFiles, destination)
1200     Dir.chdir($outputDir) {
1201         extraFiles.each {
1202             | file |
1203             dest = destination + file
1204             FileUtils.mkdir_p(dest.dirname)
1205             FileUtils.cp $extraFilesBaseDir + file, dest
1206         }
1207     }
1208 end
1209
1210 def baseDirForCollection(collectionName)
1211     Pathname(".tests") + collectionName
1212 end
1213
1214 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1215     raise unless absoluteBase.absolute?
1216     Dir.chdir($outputDir) {
1217         collectionBaseDir = baseDirForCollection($collectionName)
1218         extraFiles.each {
1219             | file |
1220             destination = collectionBaseDir + file
1221             FileUtils.mkdir_p destination.dirname unless destination.directory?
1222             FileUtils.cp absoluteBase + file, destination
1223         }
1224     }
1225 end
1226
1227 def runMozillaTest(kind, mode, extraFiles, *options)
1228     if kind
1229         kind = "mozilla-" + kind
1230     else
1231         kind = "mozilla"
1232     end
1233     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1234     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1235     case mode
1236     when :normal
1237         errorHandler = mozillaErrorHandler
1238     when :negative
1239         errorHandler = mozillaExit3ErrorHandler
1240     when :fail
1241         errorHandler = mozillaFailErrorHandler
1242     when :failDueToOutdatedOrBadTest
1243         errorHandler = mozillaFailErrorHandler
1244     when :skip
1245         return
1246     else
1247         raise "Invalid mode: #{mode}"
1248     end
1249     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1250 end
1251
1252 def runMozillaTestDefault(mode, *extraFiles)
1253     runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
1254 end
1255
1256 def runMozillaTestNoFTL(mode, *extraFiles)
1257     runMozillaTest("no-ftl", mode, extraFiles)
1258 end
1259
1260 def runMozillaTestLLInt(mode, *extraFiles)
1261     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1262 end
1263
1264 def runMozillaTestBaselineJIT(mode, *extraFiles)
1265     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1266 end
1267
1268 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1269     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1270 end
1271
1272 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1273     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1274 end
1275
1276 def defaultQuickRunMozillaTest(mode, *extraFiles)
1277     if $jitTests
1278         runMozillaTestDefault(mode, *extraFiles)
1279         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1280     else
1281         runMozillaTestNoFTL(mode, *extraFiles)
1282         if $jitTests
1283             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1284         end
1285     end
1286 end
1287
1288 def defaultRunMozillaTest(mode, *extraFiles)
1289     if $mode == "quick"
1290         defaultQuickRunMozillaTest(mode, *extraFiles)
1291     else
1292         runMozillaTestNoFTL(mode, *extraFiles)
1293         if $jitTests
1294             runMozillaTestLLInt(mode, *extraFiles)
1295             runMozillaTestBaselineJIT(mode, *extraFiles)
1296             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1297             runMozillaTestDefault(mode, *extraFiles)
1298             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
1299         end
1300     end
1301 end
1302
1303 def runNoisyTest(kind, *options)
1304     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
1305 end
1306
1307 def runNoisyTestDefault
1308     runNoisyTest("default", *FTL_OPTIONS)
1309 end
1310
1311 def runNoisyTestNoFTL
1312     runNoisyTest("no-ftl")
1313 end
1314
1315 def runNoisyTestNoCJIT
1316     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1317 end
1318
1319 def runNoisyTestNoCJITB3O1
1320     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS))
1321 end
1322
1323 def runNoisyTestEagerNoCJIT
1324     runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1325 end
1326
1327 def defaultRunNoisyTest
1328     runNoisyTestDefault
1329     if $jitTests and $isFTLPlatform
1330         runNoisyTestNoFTL
1331         runNoisyTestNoCJIT
1332         runNoisyTestNoCJITB3O1
1333         runNoisyTestEagerNoCJIT
1334     end
1335 end
1336
1337 def skip
1338     $didAddRunCommand = true
1339     puts "Skipping #{$collectionName}/#{$benchmark}"
1340 end
1341
1342 def allJSFiles(path)
1343     if path.file?
1344         [path]
1345     else
1346         result = []
1347         Dir.foreach(path) {
1348             | filename |
1349             next unless filename =~ /\.js$/
1350             next unless (path + filename).file?
1351             result << path + filename
1352         }
1353         result
1354     end
1355 end
1356
1357 def uniqueifyName(names, name)
1358     result = name.to_s
1359     toAdd = 1
1360     while names[result]
1361         result = "#{name}-#{toAdd}"
1362         toAdd += 1
1363     end
1364     names[result] = true
1365     result
1366 end
1367
1368 def simplifyCollectionName(collectionPath)
1369     outerDir = collectionPath.dirname
1370     name = collectionPath.basename
1371     lastName = name
1372     if collectionPath.directory?
1373         while lastName.to_s =~ /test/
1374             lastName = outerDir.basename
1375             name = lastName + name
1376             outerDir = outerDir.dirname
1377         end
1378     end
1379     uniqueifyName($collectionNames, name)
1380 end
1381
1382 def prepareCollection(name)
1383     FileUtils.mkdir_p $outputDir + name
1384
1385     absoluteCollection = $collection.realpath
1386
1387     Dir.chdir($outputDir) {
1388         bundleDir = baseDirForCollection(name)
1389
1390         # Create the proper directory structures.
1391         FileUtils.mkdir_p bundleDir
1392         if bundleDir.basename == $collection.basename
1393             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1394             $collection = bundleDir
1395         else
1396             FileUtils.cp_r absoluteCollection, bundleDir
1397             $collection = bundleDir + $collection.basename
1398         end
1399
1400         $extraFilesBaseDir = absoluteCollection
1401     }
1402 end
1403
1404 $collectionNames = {}
1405
1406 def handleCollectionFile(collection)
1407     collectionName = simplifyCollectionName(collection)
1408    
1409     paths = {}
1410     subCollections = []
1411     YAML::load(IO::read(collection)).each {
1412         | entry |
1413         if entry["collection"]
1414             subCollections << entry["collection"]
1415             next
1416         end
1417         
1418         if Pathname.new(entry["path"]).absolute?
1419             raise "Absolute path: " + entry["path"] + " in #{collection}"
1420         end
1421         
1422         if paths[entry["path"]]
1423             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1424         end
1425         
1426         subCollection = collection.dirname + entry["path"]
1427         
1428         if subCollection.file?
1429             subCollectionName = Pathname.new(entry["path"]).dirname
1430         else
1431             subCollectionName = entry["path"]
1432         end
1433         
1434         $collection = subCollection
1435         $collectionName = Pathname.new(collectionName)
1436         Pathname.new(subCollectionName).each_filename {
1437             | filename |
1438             next if filename =~ /^\./
1439             $collectionName += filename
1440         }
1441         $collectionName = $collectionName.to_s
1442         
1443         prepareCollection($collectionName)
1444       
1445         Dir.chdir($outputDir) {
1446             pathsToSearch = [$collection]
1447             if entry["tests"]
1448                 if entry["tests"].is_a? Array
1449                     pathsToSearch = entry["tests"].map {
1450                         | testName |
1451                         pathsToSearch[0] + testName
1452                     }
1453                 else
1454                     pathsToSearch[0] += entry["tests"]
1455                 end
1456             end
1457             pathsToSearch.each {
1458                 | pathToSearch |
1459                 allJSFiles(pathToSearch).each {
1460                     | path |
1461                     
1462                     $benchmark = path.basename
1463                     $benchmarkDirectory = path.dirname
1464                     
1465                     $runCommandOptions = {}
1466                     eval entry["cmd"]
1467                 }
1468             }
1469         }
1470     }
1471     
1472     subCollections.each {
1473         | subCollection |
1474         handleCollection(collection.dirname + subCollection)
1475     }
1476 end
1477
1478 def handleCollectionDirectory(collection)
1479     collectionName = simplifyCollectionName(collection)
1480     
1481     $collection = collection
1482     $collectionName = collectionName
1483     prepareCollection(collectionName)
1484    
1485     Dir.chdir($outputDir) {
1486         $benchmarkDirectory = $collection
1487         allJSFiles($collection).each {
1488             | path |
1489             
1490             $benchmark = path.basename
1491             
1492             $runCommandOptions = {}
1493             defaultRun unless parseRunCommands
1494         }
1495     }
1496 end
1497
1498 def handleCollection(collection)
1499     collection = Pathname.new(collection)
1500     
1501     if collection.file?
1502         handleCollectionFile(collection)
1503     else
1504         handleCollectionDirectory(collection)
1505     end
1506 end
1507
1508 def appendFailure(plan)
1509     File.open($outputDir + "failed", "a") {
1510         | outp |
1511         outp.puts plan.name
1512     }
1513     $numFailures += 1
1514 end
1515
1516 def appendPass(plan)
1517     File.open($outputDir + "passed", "a") {
1518         | outp |
1519         outp.puts plan.name
1520     }
1521     $numPasses += 1
1522 end
1523
1524 def appendResult(plan, didPass)
1525     File.open($outputDir + "results", "a") {
1526         | outp |
1527         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1528     }
1529 end
1530
1531 def prepareBundle
1532     raise if $bundle
1533
1534     if $doNotMessWithVMPath
1535         if !$remote and !$tarball
1536             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1537             $jscPath = Pathname.new($jscPath).realpath
1538         else
1539             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1540         end
1541     else
1542         originalJSCPath = $jscPath
1543         vmDir = $outputDir + ".vm"
1544         FileUtils.mkdir_p vmDir
1545         
1546         frameworkPath = frameworkFromJSCPath($jscPath)
1547         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1548         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1549         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1550
1551         if frameworkPath
1552             source = frameworkPath
1553             destination = Pathname.new(".vm")
1554         else
1555             source = originalJSCPath
1556             destination = $jscPath
1557
1558             Dir.chdir($outputDir) {
1559                 FileUtils.mkdir_p $jscPath.dirname
1560             }
1561         end
1562
1563         Dir.chdir($outputDir) {
1564             if $copyVM
1565                 FileUtils.cp_r source, destination
1566             else
1567                 begin 
1568                     FileUtils.ln_s source, destination
1569                 rescue Exception
1570                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1571                     FileUtils.cp_r source, destination
1572                 end
1573             end
1574
1575             if $remote and $hostOS == "linux"
1576                 begin
1577                     dependencies = `ldd #{source}`
1578                     dependencies.split(/\n/).each {
1579                         | dependency |
1580                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1581                     }
1582                 rescue
1583                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1584                 end
1585             end
1586         }
1587     end
1588     
1589     Dir.chdir($outputDir) {
1590         FileUtils.cp_r HELPERS_PATH, ".helpers"
1591     }
1592
1593     ARGV.each {
1594         | collection |
1595         handleCollection(collection)
1596     }
1597
1598     puts
1599 end
1600
1601 def cleanOldResults
1602     raise unless $bundle
1603
1604     eachResultFile($outputDir) {
1605         | path |
1606         FileUtils.rm_f path
1607     }
1608 end
1609
1610 def cleanEmptyResultFiles
1611     eachResultFile($outputDir) {
1612         | path |
1613         next unless path.basename.to_s =~ /\.out$/
1614         next unless FileTest.size(path) == 0
1615         FileUtils.rm_f path
1616     }
1617 end
1618
1619 def eachResultFile(startingDir, &block)
1620     dirsToClean = [startingDir]
1621     until dirsToClean.empty?
1622         nextDir = dirsToClean.pop
1623         Dir.foreach(nextDir) {
1624             | entry |
1625             next if entry =~ /^\./
1626             path = nextDir + entry
1627             if path.directory?
1628                 dirsToClean.push(path)
1629             else
1630                 block.call(path)
1631             end
1632         }
1633     end
1634 end
1635
1636 def prepareTestRunner
1637     raise if $bundle
1638
1639     $runlist.each_with_index {
1640         | plan, index |
1641         plan.index = index
1642     }
1643
1644     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1645     toDelete = []
1646     Dir.foreach($runnerDir) {
1647         | filename |
1648         if filename =~ /^test_/
1649             toDelete << filename
1650         end
1651     }
1652     
1653     toDelete.each {
1654         | filename |
1655         File.unlink($runnerDir + filename)
1656     }
1657
1658     $runlist.each {
1659         | plan |
1660         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1661     }
1662
1663     case $testRunnerType
1664     when :make
1665         prepareMakeTestRunner
1666     when :shell
1667         prepareShellTestRunner
1668     when :ruby
1669         prepareRubyTestRunner
1670     else
1671         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1672     end
1673 end
1674
1675 def cleanRunnerDirectory
1676     raise unless $bundle
1677     Dir.foreach($runnerDir) {
1678         | filename |
1679         next unless filename =~ /^test_fail/
1680         FileUtils.rm_f $runnerDir + filename
1681     }
1682 end
1683
1684 def sshRead(cmd)
1685     raise unless $remote
1686
1687     result = ""
1688     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1689       | inp |
1690       inp.each_line {
1691         | line |
1692         result += line
1693       }
1694     }
1695     raise "#{$?}" unless $?.success?
1696     result
1697 end
1698
1699 def runCommandOnTester(cmd)
1700     if $remote
1701         result = sshRead(cmd)
1702     else
1703         result = `#{cmd}`
1704     end
1705 end
1706
1707 def numberOfProcessors
1708     if $hostOS == "windows"
1709         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1710     else
1711         begin
1712             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1713         rescue
1714             numProcessors = 0
1715         end
1716
1717         if numProcessors == 0
1718             begin
1719                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1720             rescue
1721                 numProcessors == 0
1722             end
1723         end
1724     end
1725
1726     if numProcessors == 0
1727         numProcessors = 1
1728     end
1729     return numProcessors
1730 end
1731
1732 def runAndMonitorTestRunnerCommand(*cmd)
1733     numberOfTests = 0
1734     Dir.chdir($runnerDir) {
1735         # -1 for the runscript, and -2 for '..' and '.'
1736         numberOfTests = Dir.entries(".").count - 3
1737     }
1738     unless $progressMeter
1739         mysys(cmd.join(' '))
1740     else
1741        running = {}
1742        didRun = {}
1743        didFail = {}
1744        blankLine = true
1745        prevStringLength = 0
1746        IO.popen(cmd.join(' '), mode="r") {
1747            | inp |
1748            inp.each_line {
1749                | line |
1750                line = line.scrub.chomp
1751                if line =~ /^Running /
1752                    running[$~.post_match] = true
1753                elsif line =~ /^PASS: /
1754                    didRun[$~.post_match] = true
1755                elsif line =~ /^FAIL: /
1756                    didRun[$~.post_match] = true
1757                    didFail[$~.post_match] = true
1758                else
1759                    unless blankLine
1760                        print("\r" + " " * prevStringLength + "\r")
1761                    end
1762                    puts line
1763                    blankLine = true
1764                end
1765
1766                def lpad(str, chars)
1767                    str = str.to_s
1768                    if str.length > chars
1769                        str
1770                    else
1771                       "%#{chars}s"%(str)
1772                    end
1773                end
1774
1775                string  = ""
1776                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1777                unless didFail.empty?
1778                    string += " (failed #{didFail.size})"
1779                end
1780                string += " "
1781                (running.size - didRun.size).times {
1782                    string += "."
1783                }
1784                if string.length < prevStringLength
1785                    print string
1786                    print(" " * (prevStringLength - string.length))
1787                end
1788                print string
1789                prevStringLength = string.length
1790                blankLine = false
1791                $stdout.flush
1792            }
1793        }
1794        puts
1795        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1796     end
1797 end
1798
1799 def runTestRunner
1800     if $remote
1801         if !$remoteDirectory
1802             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1803         end
1804         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1805         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1806         remoteScript = "\""
1807         remoteScript += "cd #{$remoteDirectory} && "
1808         remoteScript += "rm -rf #{$outputDir.basename} && "
1809         remoteScript += "tar xzf #{$tarFileName} && "
1810         remoteScript += "cd #{$outputDir.basename}/.runner && "
1811         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1812         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1813         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
1814         $envVars.each { |var| remoteScript += "export " << var << "\n" }
1815         remoteScript += "#{testRunnerCommand}\""
1816         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1817     else
1818         Dir.chdir($runnerDir) {
1819             runAndMonitorTestRunnerCommand(testRunnerCommand)
1820         }
1821     end
1822 end
1823
1824 def detectFailures
1825     raise if $bundle
1826
1827     failures = []
1828     
1829     if $remote
1830         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1831         output.split(/\n/).each {
1832             | line |
1833             next unless line =~ /test_fail_/
1834             failures << $~.post_match.to_i
1835         }
1836     else
1837         Dir.foreach($runnerDir) {
1838             | filename |
1839             next unless filename =~ /test_fail_/
1840             failures << $~.post_match.to_i
1841         }
1842     end
1843
1844     failureSet = {}
1845
1846     failures.each {
1847         | failure | 
1848         appendFailure($runlist[failure])
1849         failureSet[failure] = true
1850     }
1851
1852     familyMap = {}
1853     $runlist.each_with_index {
1854         | plan, index |
1855         unless familyMap[plan.family]
1856             familyMap[plan.family] = []
1857         end
1858         if failureSet[index]
1859             appendResult(plan, false)
1860             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
1861             next
1862         else
1863             appendResult(plan, true)
1864             familyMap[plan.family] << {:result => "PASS", :plan => plan};
1865         end
1866         appendPass(plan)
1867     }
1868
1869     File.open($outputDir + "resultsByFamily", "w") {
1870         | outp |
1871         first = true
1872         familyMap.keys.sort.each {
1873             | familyName |
1874             if first
1875                 first = false
1876             else
1877                 outp.puts
1878             end
1879             
1880             outp.print "#{familyName}:"
1881
1882             numPassed = 0
1883             familyMap[familyName].each {
1884                 | entry |
1885                 if entry[:result] == "PASS"
1886                     numPassed += 1
1887                 end
1888             }
1889
1890             if numPassed == familyMap[familyName].size
1891                 outp.puts " PASSED"
1892             elsif numPassed == 0
1893                 outp.puts " FAILED"
1894             else
1895                 outp.puts
1896                 familyMap[familyName].each {
1897                     | entry |
1898                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
1899                 }
1900             end
1901         }
1902     }
1903 end
1904
1905 def compressBundle
1906     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
1907     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1908     raise unless system(cmd)
1909 end
1910
1911 def clean(file)
1912     FileUtils.rm_rf file unless $bundle
1913 end
1914
1915 clean($outputDir + "failed")
1916 clean($outputDir + "passed")
1917 clean($outputDir + "results")
1918 clean($outputDir + "resultsByFamily")
1919 clean($outputDir + ".vm")
1920 clean($outputDir + ".helpers")
1921 clean($outputDir + ".runner")
1922 clean($outputDir + ".tests")
1923 clean($outputDir + "_payload")
1924
1925 Dir.mkdir($outputDir) unless $outputDir.directory?
1926
1927 $outputDir = $outputDir.realpath
1928 $runnerDir = $outputDir + ".runner"
1929
1930 if !$numChildProcesses
1931     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
1932         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
1933     else
1934         $numChildProcesses = numberOfProcessors
1935     end
1936 end
1937
1938 if ENV["JSCTEST_timeout"]
1939     # In the worst case, the processors just interfere with each other.
1940     # Increase the timeout proportionally to the number of processors.
1941     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
1942 end
1943
1944 def runBundle
1945     raise unless $bundle
1946
1947     cleanRunnerDirectory
1948     cleanOldResults
1949     runTestRunner
1950     cleanEmptyResultFiles
1951 end
1952
1953 def runNormal
1954     raise if $bundle or $tarball
1955
1956     prepareBundle
1957     prepareTestRunner
1958     runTestRunner
1959     cleanEmptyResultFiles
1960     detectFailures
1961 end
1962
1963 def runTarball
1964     raise unless $tarball
1965
1966     prepareBundle 
1967     prepareTestRunner
1968     compressBundle
1969 end
1970
1971 def runRemote
1972     raise unless $remote
1973
1974     prepareBundle
1975     prepareTestRunner
1976     compressBundle
1977     runTestRunner
1978     detectFailures
1979 end
1980
1981 puts
1982 if $bundle
1983     runBundle
1984 elsif $remote
1985     runRemote
1986 elsif $tarball
1987     runTarball
1988 else
1989     runNormal
1990 end