[Win][JSCOnly] Support running JSC tests for win-cairo from windows command prompt
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2016 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
47 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
48 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
49 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
50 WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
51 CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
52 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
53 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
54
55 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
56
57 begin
58     require 'shellwords'
59 rescue Exception => e
60     $stderr.puts "Warning: did not find shellwords, not running any tests."
61     exit 0
62 end
63
64 $canRunDisplayProfilerOutput = false
65
66 begin
67     require 'rubygems'
68     require 'json'
69     require 'highline'
70     $canRunDisplayProfilerOutput = true
71 rescue Exception => e
72     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
73     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
74     $stderr.puts "Error: #{e.inspect}"
75 end
76
77 def printCommandArray(*cmd)
78     begin
79         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
80     rescue
81         commandArray = cmd.join(' ')
82     end
83     $stderr.puts ">> #{commandArray}"
84 end
85
86 def mysys(*cmd)
87     printCommandArray(*cmd) if $verbosity >= 1
88     raise "Command failed: #{$?.inspect}" unless system(*cmd)
89 end
90
91 def escapeAll(array)
92     array.map {
93         | v |
94         raise "Detected a non-string in #{inspect}" unless v.is_a? String
95         Shellwords.shellescape(v)
96     }.join(' ')
97 end
98
99
100 $jscPath = nil
101 $doNotMessWithVMPath = false
102 $jitTests = true
103 $memoryLimited = false
104 $outputDir = Pathname.new("results")
105 $verbosity = 0
106 $bundle = nil
107 $tarball = false
108 $tarFileName = "payload.tar.gz"
109 $copyVM = false
110 $testRunnerType = nil
111 $testWriter = "default"
112 $remoteUser = nil
113 $remoteHost = nil
114 $remotePort = nil
115 $remoteDirectory = nil
116 $architecture = nil
117 $hostOS = nil
118 $filter = nil
119 $envVars = []
120 $quickMode = false
121 $buildType = "release"
122 $forceCollectContinuously = false
123
124 def usage
125     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
126     puts
127     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
128     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
129     puts "                            --jsc specifies an already present JavaScriptCore to test."
130     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
131     puts "                            Skip tests tagged with //@skip if $memoryLimited"
132     puts "--no-jit                    Do not run JIT specific tests."
133     puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
134     puts "                            platform."
135     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
136     puts "--verbose            (-v)   Print more things while running."
137     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
138     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
139     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
140     puts "                            e.g. x86, x86_64, arm."
141     puts "--os                        Specify os instead of determining from JavaScriptCore build."
142     puts "                            e.g. darwin, linux & windows."
143     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
144     puts "                            In general the shell runner is slower than the make runner."
145     puts "--make-runner               Uses the faster make-based runner."
146     puts "--ruby-runner               Uses the ruby runner for machines without unix shell or make."
147     puts "--test-writer [writer]      Specifies the test script format."
148     puts "                            default is to use shell scripts to run the tests"
149     puts "                            \"ruby\" to use ruby scripts for systems without a unix shell."
150     puts "--remote                    Specify a remote host on which to run tests from command line argument."
151     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
152     puts "--child-processes    (-c)   Specify the number of child processes."
153     puts "--filter                    Only run tests whose name matches the given regular expression."
154     puts "--help               (-h)   Print this message."
155     puts "--env-vars                  Add a list of environment variables to set before running jsc."
156     puts "                            Each environment variable should be separated by a space."
157     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
158     puts "                            it will override the default value."
159     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
160     exit 1
161 end
162
163 jscArg = nil
164
165 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
166                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
167                ['--no-copy', GetoptLong::NO_ARGUMENT],
168                ['--memory-limited', GetoptLong::NO_ARGUMENT],
169                ['--no-jit', GetoptLong::NO_ARGUMENT],
170                ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
171                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
172                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
173                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
174                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
175                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
176                ['--os', GetoptLong::REQUIRED_ARGUMENT],
177                ['--shell-runner', GetoptLong::NO_ARGUMENT],
178                ['--make-runner', GetoptLong::NO_ARGUMENT],
179                ['--ruby-runner', GetoptLong::NO_ARGUMENT],
180                ['--test-writer', GetoptLong::REQUIRED_ARGUMENT],
181                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
182                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
183                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
184                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
185                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
186                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
187                ['--debug', GetoptLong::NO_ARGUMENT],
188                ['--release', GetoptLong::NO_ARGUMENT],
189                ['--quick', '-q', GetoptLong::NO_ARGUMENT]).each {
190     | opt, arg |
191     case opt
192     when '--help'
193         usage
194     when '--jsc'
195         jscArg = arg
196     when '--no-copy'
197         $doNotMessWithVMPath = true
198     when '--output-dir'
199         $outputDir = Pathname.new(arg)
200     when '--memory-limited'
201         $memoryLimited = true
202     when '--no-jit'
203         $jitTests = false
204     when '--force-collectContinuously'
205         $forceCollectContinuously = true;
206     when '--verbose'
207         $verbosity += 1
208     when '--run-bundle'
209         $bundle = Pathname.new(arg)
210     when '--tarball'
211         $tarball = true
212         $copyVM = true
213         $tarFileName = arg unless arg == ''
214     when '--force-vm-copy'
215         $copyVM = true
216     when '--shell-runner'
217         $testRunnerType = :shell
218     when '--make-runner'
219         $testRunnerType = :make
220     when '--ruby-runner'
221         $testRunnerType = :ruby
222     when '--test-writer'
223         $testWriter = arg
224     when '--remote'
225         $copyVM = true
226         $tarball = true
227         $remote = true
228         uri = URI("ssh://" + arg)
229         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
230     when '--remote-config-file'
231         $remoteConfigFile = arg
232     when '--child-processes'
233         $numChildProcesses = arg.to_i
234     when '--filter'
235         $filter = Regexp.new(arg)
236     when '--arch'
237         $architecture = arg
238     when '--os'
239         $hostOS = arg
240     when '--env-vars'
241         $envVars = arg.gsub(/\s+/, ' ').split(' ')
242     when '--quick'
243         $quickMode = true
244     when '--debug'
245         $buildType = "debug"
246     when '--release'
247         $buildType = "release"
248     end
249 }
250
251 if $remoteConfigFile
252     file = File.read($remoteConfigFile)
253     config = JSON.parse(file)
254
255     if !$remote and config['remote']
256         $copyVM = true
257         $tarball = true
258         $remote = true
259         uri = URI("ssh://" + config['remote'])
260         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
261     end
262
263     if config['remoteDirectory']
264         $remoteDirectory = config['remoteDirectory']
265     end
266 end
267
268 unless jscArg
269     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
270     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
271     command += ($buildType == "release") ? " --release" : " --debug"
272     command += " --executablePath"
273
274     output = `#{command}`.split("\n")
275     if !output.length
276         $stderr.puts "Error: must specify --jsc <path>"
277         exit 1
278     end
279
280     output = output[0]
281     jscArg = Pathname.new(output).join("jsc")
282     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
283     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
284     if !File.file?(jscArg)
285         $stderr.puts "Error: must specify --jsc <path>"
286         exit 1
287     end
288
289     puts "Using the following jsc path: #{jscArg}"
290 end
291
292 if $doNotMessWithVMPath
293     $jscPath = Pathname.new(jscArg)
294 else
295     $jscPath = Pathname.new(jscArg).realpath
296 end
297
298 $progressMeter = ($verbosity == 0 and $stdout.tty?)
299
300 if $bundle
301     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
302     $outputDir = $bundle
303 end
304
305 # Try to determine architecture. Return nil on failure.
306 def machOArchitectureCode
307     begin 
308         otoolLines = `otool -afh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
309         otoolLines.each_with_index {
310             | value, index |
311             if value =~ /magic/ and value =~ /cputype/
312                 return otoolLines[index + 1].split[1].to_i
313             end
314         }
315     rescue
316         $stderr.puts "Warning: unable to execute otool."
317     end
318     $stderr.puts "Warning: unable to determine architecture."
319     nil
320 end
321
322 def determineArchitectureFromMachOBinary
323     code = machOArchitectureCode
324     return nil unless code
325     is64BitFlag = 0x01000000
326     case code
327     when 7
328         "x86"
329     when 7 | is64BitFlag
330         "x86-64"
331     when 12
332         "arm"
333     when 12 | is64BitFlag
334         "arm64"
335     else
336         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
337         nil
338     end
339 end
340
341 def determineArchitectureFromELFBinary
342     f = File.open($jscPath.to_s)
343     data = f.read(19)
344
345     if !(data[0,4] == "\x7F\x45\x4C\x46")
346         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
347         return nil
348     end
349
350     code = data[18].ord
351     case code
352     when 3
353         "x86"
354     when 8
355         "mips"
356     when 62
357         "x86-64"
358     when 40
359         "arm"
360     when 183
361         "arm64"
362     else
363         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
364         nil
365     end
366 end
367
368 def determineArchitectureFromPEBinary
369     f = File.open($jscPath.to_s)
370     data = f.read(1024)
371
372     if !(data[0, 2] == "MZ")
373         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
374         return nil
375     end
376
377     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
378
379     if !(data[peHeaderAddr, 4] == "PE\0\0")
380         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
381         return nil
382     end
383
384     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
385
386     case machine
387     when 0x014c
388         "x86"
389     when 0x8664
390         "x86-64"
391     else
392         $stderr.puts "Warning: unsupported machine type: #{machine}"
393         nil
394     end
395 end
396
397 def determineArchitecture
398     case $hostOS
399     when "darwin"
400         determineArchitectureFromMachOBinary
401     when "linux"
402         determineArchitectureFromELFBinary
403     when "windows"
404         determineArchitectureFromPEBinary
405     else
406         $stderr.puts "Warning: unable to determine architecture on this platform."
407         nil
408     end
409 end
410
411 def determineOS
412     case RbConfig::CONFIG["host_os"]
413     when /darwin/i
414         "darwin"
415     when /linux/i
416         "linux"
417     when /mswin|mingw|cygwin/
418         "windows"
419     else
420         $stderr.puts "Warning: unable to determine host operating system"
421         nil
422     end
423 end
424
425 $hostOS = determineOS unless $hostOS
426 $architecture = determineArchitecture unless $architecture
427 $isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $architecture == "mips" || $hostOS == "windows")
428
429 if !$testRunnerType
430     if $remote and $hostOS == "darwin"
431         $testRunnerType = :shell
432     else
433         $testRunnerType = :make
434     end
435 end
436
437 if $testWriter
438     if /[^-a-zA-Z0-9_]/.match($testWriter)
439         raise "Invalid test writer #{$testWriter} given"
440     end
441 end
442
443 $numFailures = 0
444 $numPasses = 0
445
446 # We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
447 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--maxPerThreadStackUsage=1572864"]
448 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
449 # NOTE: Tests rely on this using scribbleFreeCells.
450 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
451 B3O1_OPTIONS = ["--defaultB3OptLevel=1"]
452 FTL_OPTIONS = ["--useFTLJIT=true"]
453
454 require_relative "webkitruby/jsc-stress-test-writer-#{$testWriter}"
455
456 def shouldCollectContinuously?
457     $buildType == "release" or $forceCollectContinuously
458 end
459
460 COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false"] : []
461
462 $runlist = []
463
464 def frameworkFromJSCPath(jscPath)
465     parentDirectory = jscPath.dirname
466     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
467         parentDirectory.dirname
468     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
469         jscPath.dirname + "JavaScriptCore.framework"
470     else
471         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
472         nil
473     end
474 end
475
476 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
477     dir = Pathname.new(".")
478     $benchmarkDirectory.each_filename {
479         | pathComponent |
480         dir += ".."
481     }
482     dir + resourcePath
483 end
484
485 def pathToVM
486     pathToBundleResourceFromBenchmarkDirectory($jscPath)
487 end
488
489 def pathToHelpers
490     pathToBundleResourceFromBenchmarkDirectory(".helpers")
491 end
492
493 $runCommandOptions = {}
494
495 $uniqueFilenameCounter = 0
496 def uniqueFilename(extension)
497     payloadDir = $outputDir + "_payload"
498     Dir.mkdir payloadDir unless payloadDir.directory?
499     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
500     $uniqueFilenameCounter += 1
501     result
502 end
503
504 def baseOutputName(kind)
505     "#{$collectionName}/#{$benchmark}.#{kind}"
506 end
507
508 def addRunCommand(kind, command, outputHandler, errorHandler)
509     $didAddRunCommand = true
510     name = baseOutputName(kind)
511     if $filter and name !~ $filter
512         return
513     end
514     plan = Plan.new(
515         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
516         errorHandler)
517     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
518         $runlist.unshift plan
519     else
520         $runlist << plan
521     end
522 end
523
524 # Returns true if there were run commands found in the file ($benchmarkDirectory +
525 # $benchmark), in which case those run commands have already been executed. Otherwise
526 # returns false, in which case you're supposed to add your own run commands.
527 def parseRunCommands
528     oldDidAddRunCommand = $didAddRunCommand
529     $didAddRunCommand = false
530
531     Dir.chdir($outputDir) {
532         File.open($benchmarkDirectory + $benchmark) {
533             | inp |
534             inp.each_line {
535                 | line |
536                 begin
537                     doesMatch = line =~ /^\/\/@/
538                 rescue Exception => e
539                     # Apparently this happens in the case of some UTF8 stuff in some files, where
540                     # Ruby tries to be strict and throw exceptions.
541                     next
542                 end
543                 next unless doesMatch
544                 eval $~.post_match
545             }
546         }
547     }
548
549     result = $didAddRunCommand
550     $didAddRunCommand = result or oldDidAddRunCommand
551     result
552 end
553
554 def slow!
555     $runCommandOptions[:isSlow] = true
556 end
557
558 def runWithOutputHandler(kind, outputHandler, *options)
559     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
560 end
561
562 def run(kind, *options)
563     runWithOutputHandler(kind, silentOutputHandler, *options)
564 end
565
566 def runNoFTL(*optionalTestSpecificOptions)
567     run("no-ftl", *optionalTestSpecificOptions)
568 end
569
570 def runWithRAMSize(size, *optionalTestSpecificOptions)
571     run("ram-size-#{size}", "--forceRAMSize=#{size}", *optionalTestSpecificOptions)
572 end
573
574 def runOneLargeHeap(*optionalTestSpecificOptions)
575     if $memoryLimited
576         $didAddRunCommand = true
577         puts "Skipping #{$collectionName}/#{$benchmark}"
578     else
579         run("default", *optionalTestSpecificOptions)
580     end
581 end
582
583 def runNoJIT(*optionalTestSpecificOptions)
584     run("no-jit", "--useJIT=false", *optionalTestSpecificOptions)
585 end
586
587 def runNoLLInt(*optionalTestSpecificOptions)
588     if $jitTests
589         run("no-llint", "--useLLInt=false", *optionalTestSpecificOptions)
590     end
591 end
592
593 # NOTE: Tests rely on this using scribbleFreeCells.
594 def runNoCJITValidate(*optionalTestSpecificOptions)
595     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
596 end
597
598 def runNoCJITValidatePhases(*optionalTestSpecificOptions)
599     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
600 end
601
602 def runNoCJITCollectContinuously(*optionalTestSpecificOptions)
603     run("no-cjit-collect-continuously", *(NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
604 end
605
606 def runDefault(*optionalTestSpecificOptions)
607     run("default", *(FTL_OPTIONS + optionalTestSpecificOptions))
608 end
609
610 def runFTLNoCJIT(*optionalTestSpecificOptions)
611     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
612 end
613
614 def runFTLNoCJITB3O1(*optionalTestSpecificOptions)
615     run("ftl-no-cjit-b3o1", "--useArrayAllocationProfiling=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
616 end
617
618 def runFTLNoCJITValidate(*optionalTestSpecificOptions)
619     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
620 end
621
622 def runFTLNoCJITNoPutStackValidate(*optionalTestSpecificOptions)
623     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
624 end
625
626 def runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
627     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
628 end
629
630 def runFTLNoCJITOSRValidation(*optionalTestSpecificOptions)
631     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
632 end
633
634 def runDFGEager(*optionalTestSpecificOptions)
635     run("dfg-eager", *(EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
636 end
637
638 def runDFGEagerNoCJITValidate(*optionalTestSpecificOptions)
639     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
640 end
641
642 def runFTLEager(*optionalTestSpecificOptions)
643     run("ftl-eager", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
644 end
645
646 def runFTLEagerWatchdog(*optionalTestSpecificOptions)
647     timeout = rand(100)
648     run("ftl-eager-watchdog-#{timeout}", "--watchdog=#{timeout}", "--watchdog-exception-ok", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
649 end
650
651 def runFTLEagerNoCJITValidate(*optionalTestSpecificOptions)
652     run("ftl-eager-no-cjit", "--validateGraph=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
653 end
654
655 def runFTLEagerNoCJITB3O1(*optionalTestSpecificOptions)
656     run("ftl-eager-no-cjit-b3o1", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
657 end
658
659 def runFTLEagerNoCJITOSRValidation(*optionalTestSpecificOptions)
660     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
661 end
662
663 def runNoCJITNoASO(*optionalTestSpecificOptions)
664     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
665 end
666
667 def runNoCJITNoAccessInlining(*optionalTestSpecificOptions)
668     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
669 end
670
671 def runFTLNoCJITNoAccessInlining(*optionalTestSpecificOptions)
672     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
673 end
674
675 def runFTLNoCJITSmallPool(*optionalTestSpecificOptions)
676     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
677 end
678
679 def runNoCJIT(*optionalTestSpecificOptions)
680     run("no-cjit", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
681 end
682
683 def runDFGMaximalFlushPhase(*optionalTestSpecificOptions)
684     run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
685 end
686
687 def runShadowChicken(*optionalTestSpecificOptions)
688     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true", *optionalTestSpecificOptions)
689 end
690
691 def defaultRun
692     if $quickMode
693         defaultQuickRun
694     else
695         runDefault
696         if $jitTests
697             runNoLLInt
698             runNoCJITValidatePhases
699             runNoCJITCollectContinuously if shouldCollectContinuously?
700             runDFGEager
701             runDFGEagerNoCJITValidate
702             runDFGMaximalFlushPhase
703
704             return if !$isFTLPlatform
705
706             runNoFTL
707             runFTLNoCJITValidate
708             runFTLNoCJITB3O1
709             runFTLNoCJITNoPutStackValidate
710             runFTLNoCJITNoInlineValidate
711             runFTLEager
712             runFTLEagerNoCJITValidate
713             runFTLEagerNoCJITB3O1
714             runFTLNoCJITSmallPool
715         end
716     end
717 end
718
719 def defaultNoNoLLIntRun
720     if $quickMode
721         defaultQuickRun
722     else
723         runDefault
724         if $jitTests
725             runNoCJITValidatePhases
726             runNoCJITCollectContinuously if shouldCollectContinuously?
727             runDFGEager
728             runDFGEagerNoCJITValidate
729             runDFGMaximalFlushPhase
730
731             return if !$isFTLPlatform
732
733             runNoFTL
734             runFTLNoCJITValidate
735             runFTLNoCJITB3O1
736             runFTLNoCJITNoPutStackValidate
737             runFTLNoCJITNoInlineValidate
738             runFTLEager
739             runFTLEagerNoCJITValidate
740             runFTLNoCJITSmallPool
741         end
742     end
743 end
744
745 def defaultQuickRun
746     runDefault
747     if $jitTests
748         runNoCJITValidate
749
750         return if $isFTLPlatform
751
752         runNoFTL
753         runFTLNoCJITValidate
754     end
755 end
756
757 def defaultSpotCheckNoMaximalFlush
758     defaultQuickRun
759     runNoCJITNoAccessInlining
760
761     return if !$isFTLPlatform
762
763     runFTLNoCJITOSRValidation
764     runFTLNoCJITNoAccessInlining
765     runFTLNoCJITB3O1
766 end
767
768 def defaultSpotCheck
769     defaultSpotCheckNoMaximalFlush
770     runDFGMaximalFlushPhase
771 end
772
773 # This is expected to not do eager runs because eager runs can have a lot of recompilations
774 # for reasons that don't arise in the real world. It's used for tests that assert convergence
775 # by counting recompilations.
776 def defaultNoEagerRun
777     runDefault
778     if $jitTests
779         runNoLLInt
780         runNoCJITValidatePhases
781         runNoCJITCollectContinuously if shouldCollectContinuously?
782
783         return if !$isFTLPlatform
784
785         runNoFTL
786         runFTLNoCJITValidate
787         runFTLNoCJITNoInlineValidate
788         runFTLNoCJITB3O1
789     end
790 end
791
792 def defaultNoSamplingProfilerRun
793     runDefault
794     if $jitTests
795         runNoLLInt
796         runNoCJITValidatePhases
797         runNoCJITCollectContinuously if shouldCollectContinuously?
798         runDFGEager
799         runDFGEagerNoCJITValidate
800         runDFGMaximalFlushPhase
801
802         return if !$isFTLPlatform
803
804         runNoFTL
805         runFTLNoCJITNoPutStackValidate
806         runFTLNoCJITNoInlineValidate
807         runFTLEager
808         runFTLEagerNoCJITValidate
809         runFTLNoCJITSmallPool
810     end
811 end
812
813 def runProfiler
814     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
815         skip
816         return
817     end
818
819     profilerOutput = uniqueFilename(".json")
820     if $canRunDisplayProfilerOutput
821         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
822     else
823         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
824         run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
825     end
826 end
827
828 def runExceptionFuzz
829     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
830     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
831 end
832
833 def runExecutableAllocationFuzz(name, *options)
834     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
835     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
836 end
837
838 def runTypeProfiler
839     if !$jitTests
840         return
841     end
842
843     return if !$isFTLPlatform
844
845     run("ftl-no-cjit-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
846     run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
847 end
848
849 def runControlFlowProfiler
850     if !$jitTests
851         return
852     end
853
854     return if !$isFTLPlatform
855
856     run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
857 end
858
859 def runTest262(mode, exception, includeFiles, flags)
860     failsWithException = exception != "NoException"
861     isStrict = false
862     isModule = false
863     isAsync = false
864
865     flags.each {
866         | flag |
867         case flag
868         when :strict
869             isStrict = true
870         when :module
871             isModule = true
872         when :async
873             isAsync = true
874         else
875             raise "Invalid flag for runTest262, #{flag}"
876         end
877     }
878
879     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
880
881     args = [pathToVM.to_s] + BASE_OPTIONS
882     args << "--exception=" + exception if failsWithException
883     args << "--test262-async" if isAsync
884     args += includeFiles
885
886     case mode
887     when :normal
888         errorHandler = simpleErrorHandler
889         outputHandler = silentOutputHandler
890     when :fail
891         errorHandler = expectedFailErrorHandler
892         outputHandler = noisyOutputHandler
893     when :failDueToOutdatedOrBadTest
894         errorHandler = expectedFailErrorHandler
895         outputHandler = noisyOutputHandler
896     else
897         raise "Invalid mode: #{mode}"
898     end
899
900     if isStrict
901         kind = "default-strict"
902         args << "--strict-file=#{$benchmark}"
903     else
904         kind = "default"
905         if isModule
906             args << "--module-file=#{$benchmark}"
907         else
908             args << $benchmark.to_s
909         end
910     end
911
912     addRunCommand(kind, args, outputHandler, errorHandler)
913 end
914
915 def prepareTest262Fixture
916     # This function is used to add the files used by Test262 modules tests.
917     prepareExtraRelativeFiles([""], $collection)
918 end
919
920 def runES6(mode)
921     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
922     case mode
923     when :normal
924         errorHandler = simpleErrorHandler
925     when :fail
926         errorHandler = expectedFailErrorHandler
927     when :failDueToOutdatedOrBadTest
928         errorHandler = expectedFailErrorHandler
929     else
930         raise "Invalid mode: #{mode}"
931     end
932     addRunCommand("default", args, noisyOutputHandler, errorHandler)
933 end
934
935 def runModules
936     run("default-modules", "-m")
937
938     if !$jitTests
939         return
940     end
941
942     run("no-llint-modules", "-m", "--useLLInt=false")
943     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
944     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
945     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
946
947     return if !$isFTLPlatform
948
949     run("default-ftl-modules", "-m", *FTL_OPTIONS)
950     run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
951     run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
952     run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
953     run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
954     run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
955 end
956
957 def runWebAssembly
958     return if !$jitTests
959     return if !$isFTLPlatform
960     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
961     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
962     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
963     run("default-wasm", "-m", *FTL_OPTIONS)
964     if !$quickMode
965         run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
966         run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
967         run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
968         run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
969         run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
970     end
971 end
972
973 def runWebAssemblyEmscripten(mode)
974     case mode
975     when :skip
976         return
977     end
978     return if !$jitTests
979     return if !$isFTLPlatform
980     wasm = $benchmark.to_s.sub! '.js', '.wasm'
981     prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
982     run("default-wasm", *FTL_OPTIONS)
983     if !$quickMode
984         run("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
985         run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
986         run("wasm-no-call-ic", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
987         run("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
988     end
989 end
990
991 def runWebAssemblySpecTest(mode)
992     case mode
993     when :skip
994         return
995     end
996     return if !$jitTests
997     return if !$isFTLPlatform
998     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
999
1000     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1001     prepareExtraRelativeFiles(modules.map { |f| "../../" + f }, $collection)
1002
1003     harness = Dir[WASMTESTS_PATH + "spec-harness/" + "*.js"].map { |f| File.basename(f) }
1004     prepareExtraRelativeFiles(harness.map { |f| "../../spec-harness/" + f }, $collection)
1005
1006     runWithOutputHandler("default-wasm", noisyOutputHandler, "../spec-harness.js", *FTL_OPTIONS)
1007     if !$quickMode
1008       runWithOutputHandler("wasm-no-cjit-yes-tls-context", noisyOutputHandler, "../spec-harness.js",  "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1009       runWithOutputHandler("wasm-eager-jettison", noisyOutputHandler, "../spec-harness.js", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1010       runWithOutputHandler("wasm-no-call-ic", noisyOutputHandler, "../spec-harness.js", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1011       runWithOutputHandler("wasm-no-tls-context", noisyOutputHandler, "../spec-harness.js", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1012     end
1013 end
1014
1015 def runWebAssemblyLowExecutableMemory(*optionalTestSpecificOptions)
1016     return if !$jitTests
1017     return if !$isFTLPlatform
1018     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1019     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1020     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1021     # Only let WebAssembly get executable memory.
1022     run("default-wasm", "--useConcurrentGC=0" , "--useConcurrentJIT=0", "--jitMemoryReservationSize=15000", "--useBaselineJIT=0", "--useDFGJIT=0", "--useFTLJIT=0", "-m")
1023 end
1024
1025 def runChakra(mode, exception, baselineFile, extraFiles)
1026     raise unless $benchmark.to_s =~ /\.js$/
1027     failsWithException = exception != "NoException"
1028     testName = $~.pre_match
1029
1030     prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
1031     prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)
1032
1033     args = [pathToVM.to_s] + BASE_OPTIONS
1034     args += FTL_OPTIONS if $isFTLPlatform
1035     args += EAGER_OPTIONS
1036     args << "--exception=" + exception if failsWithException
1037     args << "--dumpException" if failsWithException
1038     args += ["jsc-lib.js"]
1039
1040     case mode
1041     when :baseline
1042         prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
1043         errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
1044         outputHandler = noisyOutputHandler
1045     when :pass
1046         errorHandler = chakraPassFailErrorHandler
1047         outputHandler = noisyOutputHandler
1048     when :skipDueToOutdatedOrBadTest
1049         return
1050     when :skip
1051         return
1052     else
1053         raise "Invalid mode: #{mode}"
1054     end
1055
1056     kind = "default"
1057     args << $benchmark.to_s
1058
1059     addRunCommand(kind, args, outputHandler, errorHandler)
1060 end
1061
1062 def runLayoutTest(kind, *options)
1063     raise unless $benchmark.to_s =~ /\.js$/
1064     testName = $~.pre_match
1065     if kind
1066         kind = "layout-" + kind
1067     else
1068         kind = "layout"
1069     end
1070
1071     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1072     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1073
1074     args = [pathToVM.to_s] + BASE_OPTIONS + options +
1075         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1076          $benchmark.to_s,
1077          (Pathname.new("resources") + "standalone-post.js").to_s]
1078     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1079 end
1080
1081 def runLayoutTestNoFTL
1082     runLayoutTest("no-ftl")
1083 end
1084
1085 def runLayoutTestNoLLInt
1086     runLayoutTest("no-llint", "--useLLInt=false")
1087 end
1088
1089 def runLayoutTestNoCJIT
1090     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1091 end
1092
1093 def runLayoutTestDFGEagerNoCJIT
1094     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1095 end
1096
1097 def runLayoutTestDefault
1098     runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
1099 end
1100
1101 def runLayoutTestFTLNoCJIT
1102     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1103 end
1104
1105 def runLayoutTestFTLEagerNoCJIT
1106     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1107 end
1108
1109 def runLayoutTestFTLEagerNoCJITB3O1
1110     runLayoutTest("ftl-eager-no-cjit-b3o1", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS))
1111 end
1112
1113 def noFTLRunLayoutTest
1114     if !$jitTests
1115         return
1116     end
1117
1118     runLayoutTestNoLLInt
1119     runLayoutTestNoCJIT
1120     runLayoutTestDFGEagerNoCJIT
1121 end
1122
1123 def defaultQuickRunLayoutTest
1124     runLayoutTestDefault
1125     if $jitTests
1126         if $isFTLPlatform
1127             runLayoutTestNoFTL
1128             runLayoutTestFTLNoCJIT
1129             runLayoutTestFTLEagerNoCJIT
1130         else
1131             noFTLRunLayoutTest
1132         end
1133     end
1134 end
1135
1136 def defaultRunLayoutTest
1137     if $quickMode
1138         defaultQuickRunLayoutTest
1139     else
1140         runLayoutTestDefault
1141         if $jitTests
1142             noFTLRunLayoutTest
1143
1144             return if !$isFTLPlatform
1145
1146             runLayoutTestNoFTL
1147             runLayoutTestFTLNoCJIT
1148             runLayoutTestFTLEagerNoCJIT
1149         end
1150     end
1151 end
1152
1153 def noEagerNoNoLLIntTestsRunLayoutTest
1154     runLayoutTestDefault
1155     if $jitTests
1156         runLayoutTestNoCJIT
1157
1158         return if !$isFTLPlatform
1159
1160         runLayoutTestNoFTL
1161         runLayoutTestFTLNoCJIT
1162     end
1163 end
1164
1165 def noNoLLIntRunLayoutTest
1166     runLayoutTestDefault
1167     if $jitTests
1168         runLayoutTestNoCJIT
1169         runLayoutTestDFGEagerNoCJIT
1170
1171         return if !$isFTLPlatform
1172
1173         runLayoutTestNoFTL
1174         runLayoutTestFTLNoCJIT
1175         runLayoutTestFTLEagerNoCJIT
1176     end
1177 end
1178
1179 def prepareExtraRelativeFiles(extraFiles, destination)
1180     Dir.chdir($outputDir) {
1181         extraFiles.each {
1182             | file |
1183             dest = destination + file
1184             FileUtils.mkdir_p(dest.dirname)
1185             FileUtils.cp $extraFilesBaseDir + file, dest
1186         }
1187     }
1188 end
1189
1190 def baseDirForCollection(collectionName)
1191     Pathname(".tests") + collectionName
1192 end
1193
1194 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1195     raise unless absoluteBase.absolute?
1196     Dir.chdir($outputDir) {
1197         collectionBaseDir = baseDirForCollection($collectionName)
1198         extraFiles.each {
1199             | file |
1200             destination = collectionBaseDir + file
1201             FileUtils.mkdir_p destination.dirname unless destination.directory?
1202             FileUtils.cp absoluteBase + file, destination
1203         }
1204     }
1205 end
1206
1207 def runMozillaTest(kind, mode, extraFiles, *options)
1208     if kind
1209         kind = "mozilla-" + kind
1210     else
1211         kind = "mozilla"
1212     end
1213     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1214     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1215     case mode
1216     when :normal
1217         errorHandler = mozillaErrorHandler
1218     when :negative
1219         errorHandler = mozillaExit3ErrorHandler
1220     when :fail
1221         errorHandler = mozillaFailErrorHandler
1222     when :skip
1223         return
1224     else
1225         raise "Invalid mode: #{mode}"
1226     end
1227     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1228 end
1229
1230 def runMozillaTestDefault(mode, *extraFiles)
1231     runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
1232 end
1233
1234 def runMozillaTestNoFTL(mode, *extraFiles)
1235     runMozillaTest("no-ftl", mode, extraFiles)
1236 end
1237
1238 def runMozillaTestLLInt(mode, *extraFiles)
1239     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1240 end
1241
1242 def runMozillaTestBaselineJIT(mode, *extraFiles)
1243     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1244 end
1245
1246 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1247     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1248 end
1249
1250 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1251     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1252 end
1253
1254 def defaultQuickRunMozillaTest(mode, *extraFiles)
1255     if $jitTests
1256         runMozillaTestDefault(mode, *extraFiles)
1257         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1258     else
1259         runMozillaTestNoFTL(mode, *extraFiles)
1260         if $jitTests
1261             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1262         end
1263     end
1264 end
1265
1266 def defaultRunMozillaTest(mode, *extraFiles)
1267     if $quickMode
1268         defaultQuickRunMozillaTest(mode, *extraFiles)
1269     else
1270         runMozillaTestNoFTL(mode, *extraFiles)
1271         if $jitTests
1272             runMozillaTestLLInt(mode, *extraFiles)
1273             runMozillaTestBaselineJIT(mode, *extraFiles)
1274             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1275             runMozillaTestDefault(mode, *extraFiles)
1276             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
1277         end
1278     end
1279 end
1280
1281 def runNoisyTest(kind, *options)
1282     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
1283 end
1284
1285 def runNoisyTestDefault
1286     runNoisyTest("default", *FTL_OPTIONS)
1287 end
1288
1289 def runNoisyTestNoFTL
1290     runNoisyTest("no-ftl")
1291 end
1292
1293 def runNoisyTestNoCJIT
1294     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1295 end
1296
1297 def runNoisyTestNoCJITB3O1
1298     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS))
1299 end
1300
1301 def runNoisyTestEagerNoCJIT
1302     runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1303 end
1304
1305 def defaultRunNoisyTest
1306     runNoisyTestDefault
1307     if $jitTests and $isFTLPlatform
1308         runNoisyTestNoFTL
1309         runNoisyTestNoCJIT
1310         runNoisyTestNoCJITB3O1
1311         runNoisyTestEagerNoCJIT
1312     end
1313 end
1314
1315 def skip
1316     $didAddRunCommand = true
1317     puts "Skipping #{$collectionName}/#{$benchmark}"
1318 end
1319
1320 def allJSFiles(path)
1321     if path.file?
1322         [path]
1323     else
1324         result = []
1325         Dir.foreach(path) {
1326             | filename |
1327             next unless filename =~ /\.js$/
1328             next unless (path + filename).file?
1329             result << path + filename
1330         }
1331         result
1332     end
1333 end
1334
1335 def uniqueifyName(names, name)
1336     result = name.to_s
1337     toAdd = 1
1338     while names[result]
1339         result = "#{name}-#{toAdd}"
1340         toAdd += 1
1341     end
1342     names[result] = true
1343     result
1344 end
1345
1346 def simplifyCollectionName(collectionPath)
1347     outerDir = collectionPath.dirname
1348     name = collectionPath.basename
1349     lastName = name
1350     if collectionPath.directory?
1351         while lastName.to_s =~ /test/
1352             lastName = outerDir.basename
1353             name = lastName + name
1354             outerDir = outerDir.dirname
1355         end
1356     end
1357     uniqueifyName($collectionNames, name)
1358 end
1359
1360 def prepareCollection(name)
1361     FileUtils.mkdir_p $outputDir + name
1362
1363     absoluteCollection = $collection.realpath
1364
1365     Dir.chdir($outputDir) {
1366         bundleDir = baseDirForCollection(name)
1367
1368         # Create the proper directory structures.
1369         FileUtils.mkdir_p bundleDir
1370         if bundleDir.basename == $collection.basename
1371             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1372             $collection = bundleDir
1373         else
1374             FileUtils.cp_r absoluteCollection, bundleDir
1375             $collection = bundleDir + $collection.basename
1376         end
1377
1378         $extraFilesBaseDir = absoluteCollection
1379     }
1380 end
1381
1382 $collectionNames = {}
1383
1384 def handleCollectionFile(collection)
1385     collectionName = simplifyCollectionName(collection)
1386    
1387     paths = {}
1388     subCollections = []
1389     YAML::load(IO::read(collection)).each {
1390         | entry |
1391         if entry["collection"]
1392             subCollections << entry["collection"]
1393             next
1394         end
1395         
1396         if Pathname.new(entry["path"]).absolute?
1397             raise "Absolute path: " + entry["path"] + " in #{collection}"
1398         end
1399         
1400         if paths[entry["path"]]
1401             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1402         end
1403         
1404         subCollection = collection.dirname + entry["path"]
1405         
1406         if subCollection.file?
1407             subCollectionName = Pathname.new(entry["path"]).dirname
1408         else
1409             subCollectionName = entry["path"]
1410         end
1411         
1412         $collection = subCollection
1413         $collectionName = Pathname.new(collectionName)
1414         Pathname.new(subCollectionName).each_filename {
1415             | filename |
1416             next if filename =~ /^\./
1417             $collectionName += filename
1418         }
1419         $collectionName = $collectionName.to_s
1420         
1421         prepareCollection($collectionName)
1422       
1423         Dir.chdir($outputDir) {
1424             pathsToSearch = [$collection]
1425             if entry["tests"]
1426                 if entry["tests"].is_a? Array
1427                     pathsToSearch = entry["tests"].map {
1428                         | testName |
1429                         pathsToSearch[0] + testName
1430                     }
1431                 else
1432                     pathsToSearch[0] += entry["tests"]
1433                 end
1434             end
1435             pathsToSearch.each {
1436                 | pathToSearch |
1437                 allJSFiles(pathToSearch).each {
1438                     | path |
1439                     
1440                     $benchmark = path.basename
1441                     $benchmarkDirectory = path.dirname
1442                     
1443                     $runCommandOptions = {}
1444                     eval entry["cmd"]
1445                 }
1446             }
1447         }
1448     }
1449     
1450     subCollections.each {
1451         | subCollection |
1452         handleCollection(collection.dirname + subCollection)
1453     }
1454 end
1455
1456 def handleCollectionDirectory(collection)
1457     collectionName = simplifyCollectionName(collection)
1458     
1459     $collection = collection
1460     $collectionName = collectionName
1461     prepareCollection(collectionName)
1462    
1463     Dir.chdir($outputDir) {
1464         $benchmarkDirectory = $collection
1465         allJSFiles($collection).each {
1466             | path |
1467             
1468             $benchmark = path.basename
1469             
1470             $runCommandOptions = {}
1471             defaultRun unless parseRunCommands
1472         }
1473     }
1474 end
1475
1476 def handleCollection(collection)
1477     collection = Pathname.new(collection)
1478     
1479     if collection.file?
1480         handleCollectionFile(collection)
1481     else
1482         handleCollectionDirectory(collection)
1483     end
1484 end
1485
1486 def appendFailure(plan)
1487     File.open($outputDir + "failed", "a") {
1488         | outp |
1489         outp.puts plan.name
1490     }
1491     $numFailures += 1
1492 end
1493
1494 def appendPass(plan)
1495     File.open($outputDir + "passed", "a") {
1496         | outp |
1497         outp.puts plan.name
1498     }
1499     $numPasses += 1
1500 end
1501
1502 def appendResult(plan, didPass)
1503     File.open($outputDir + "results", "a") {
1504         | outp |
1505         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1506     }
1507 end
1508
1509 def prepareBundle
1510     raise if $bundle
1511
1512     if $doNotMessWithVMPath
1513         if !$remote and !$tarball
1514             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1515             $jscPath = Pathname.new($jscPath).realpath
1516         else
1517             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1518         end
1519     else
1520         originalJSCPath = $jscPath
1521         vmDir = $outputDir + ".vm"
1522         FileUtils.mkdir_p vmDir
1523         
1524         frameworkPath = frameworkFromJSCPath($jscPath)
1525         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1526         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1527         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1528
1529         if frameworkPath
1530             source = frameworkPath
1531             destination = Pathname.new(".vm")
1532         else
1533             source = originalJSCPath
1534             destination = $jscPath
1535
1536             Dir.chdir($outputDir) {
1537                 FileUtils.mkdir_p $jscPath.dirname
1538             }
1539         end
1540
1541         Dir.chdir($outputDir) {
1542             if $copyVM
1543                 FileUtils.cp_r source, destination
1544             else
1545                 begin 
1546                     FileUtils.ln_s source, destination
1547                 rescue Exception
1548                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1549                     FileUtils.cp_r source, destination
1550                 end
1551             end
1552
1553             if $remote and $hostOS == "linux"
1554                 begin
1555                     dependencies = `ldd #{source}`
1556                     dependencies.split(/\n/).each {
1557                         | dependency |
1558                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1559                     }
1560                 rescue
1561                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1562                 end
1563             end
1564         }
1565     end
1566     
1567     Dir.chdir($outputDir) {
1568         FileUtils.cp_r HELPERS_PATH, ".helpers"
1569     }
1570
1571     ARGV.each {
1572         | collection |
1573         handleCollection(collection)
1574     }
1575
1576     puts
1577 end
1578
1579 def cleanOldResults
1580     raise unless $bundle
1581
1582     eachResultFile($outputDir) {
1583         | path |
1584         FileUtils.rm_f path
1585     }
1586 end
1587
1588 def cleanEmptyResultFiles
1589     eachResultFile($outputDir) {
1590         | path |
1591         next unless path.basename.to_s =~ /\.out$/
1592         next unless FileTest.size(path) == 0
1593         FileUtils.rm_f path
1594     }
1595 end
1596
1597 def eachResultFile(startingDir, &block)
1598     dirsToClean = [startingDir]
1599     until dirsToClean.empty?
1600         nextDir = dirsToClean.pop
1601         Dir.foreach(nextDir) {
1602             | entry |
1603             next if entry =~ /^\./
1604             path = nextDir + entry
1605             if path.directory?
1606                 dirsToClean.push(path)
1607             else
1608                 block.call(path)
1609             end
1610         }
1611     end
1612 end
1613
1614 def prepareTestRunner
1615     raise if $bundle
1616
1617     $runlist.each_with_index {
1618         | plan, index |
1619         plan.index = index
1620     }
1621
1622     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1623     toDelete = []
1624     Dir.foreach($runnerDir) {
1625         | filename |
1626         if filename =~ /^test_/
1627             toDelete << filename
1628         end
1629     }
1630     
1631     toDelete.each {
1632         | filename |
1633         File.unlink($runnerDir + filename)
1634     }
1635
1636     $runlist.each {
1637         | plan |
1638         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1639     }
1640
1641     case $testRunnerType
1642     when :make
1643         prepareMakeTestRunner
1644     when :shell
1645         prepareShellTestRunner
1646     when :ruby
1647         prepareRubyTestRunner
1648     else
1649         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1650     end
1651 end
1652
1653 def cleanRunnerDirectory
1654     raise unless $bundle
1655     Dir.foreach($runnerDir) {
1656         | filename |
1657         next unless filename =~ /^test_fail/
1658         FileUtils.rm_f $runnerDir + filename
1659     }
1660 end
1661
1662 def sshRead(cmd)
1663     raise unless $remote
1664
1665     result = ""
1666     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1667       | inp |
1668       inp.each_line {
1669         | line |
1670         result += line
1671       }
1672     }
1673     raise "#{$?}" unless $?.success?
1674     result
1675 end
1676
1677 def runCommandOnTester(cmd)
1678     if $remote
1679         result = sshRead(cmd)
1680     else
1681         result = `#{cmd}`
1682     end
1683 end
1684
1685 def numberOfProcessors
1686     if $hostOS == "windows"
1687         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1688     else
1689         begin
1690             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1691         rescue
1692             numProcessors = 0
1693         end
1694
1695         if numProcessors == 0
1696             begin
1697                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1698             rescue
1699                 numProcessors == 0
1700             end
1701         end
1702     end
1703
1704     if numProcessors == 0
1705         numProcessors = 1
1706     end
1707     return numProcessors
1708 end
1709
1710 def runAndMonitorTestRunnerCommand(*cmd)
1711     numberOfTests = 0
1712     Dir.chdir($runnerDir) {
1713         # -1 for the runscript, and -2 for '..' and '.'
1714         numberOfTests = Dir.entries(".").count - 3
1715     }
1716     unless $progressMeter
1717         mysys(cmd.join(' '))
1718     else
1719        running = {}
1720        didRun = {}
1721        didFail = {}
1722        blankLine = true
1723        prevStringLength = 0
1724        IO.popen(cmd.join(' '), mode="r") {
1725            | inp |
1726            inp.each_line {
1727                | line |
1728                line = line.scrub.chomp
1729                if line =~ /^Running /
1730                    running[$~.post_match] = true
1731                elsif line =~ /^PASS: /
1732                    didRun[$~.post_match] = true
1733                elsif line =~ /^FAIL: /
1734                    didRun[$~.post_match] = true
1735                    didFail[$~.post_match] = true
1736                else
1737                    unless blankLine
1738                        print("\r" + " " * prevStringLength + "\r")
1739                    end
1740                    puts line
1741                    blankLine = true
1742                end
1743
1744                def lpad(str, chars)
1745                    str = str.to_s
1746                    if str.length > chars
1747                        str
1748                    else
1749                       "%#{chars}s"%(str)
1750                    end
1751                end
1752
1753                string  = ""
1754                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1755                unless didFail.empty?
1756                    string += " (failed #{didFail.size})"
1757                end
1758                string += " "
1759                (running.size - didRun.size).times {
1760                    string += "."
1761                }
1762                if string.length < prevStringLength
1763                    print string
1764                    print(" " * (prevStringLength - string.length))
1765                end
1766                print string
1767                prevStringLength = string.length
1768                blankLine = false
1769                $stdout.flush
1770            }
1771        }
1772        puts
1773        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1774     end
1775 end
1776
1777 def runTestRunner
1778     if $remote
1779         if !$remoteDirectory
1780             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1781         end
1782         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1783         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1784         remoteScript = "\""
1785         remoteScript += "cd #{$remoteDirectory} && "
1786         remoteScript += "rm -rf #{$outputDir.basename} && "
1787         remoteScript += "tar xzf #{$tarFileName} && "
1788         remoteScript += "cd #{$outputDir.basename}/.runner && "
1789         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1790         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1791         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
1792         $envVars.each { |var| remoteScript += "export " << var << "\n" }
1793         remoteScript += "#{testRunnerCommand}\""
1794         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1795     else
1796         Dir.chdir($runnerDir) {
1797             runAndMonitorTestRunnerCommand(testRunnerCommand)
1798         }
1799     end
1800 end
1801
1802 def detectFailures
1803     raise if $bundle
1804
1805     failures = []
1806     
1807     if $remote
1808         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1809         output.split(/\n/).each {
1810             | line |
1811             next unless line =~ /test_fail_/
1812             failures << $~.post_match.to_i
1813         }
1814     else
1815         Dir.foreach($runnerDir) {
1816             | filename |
1817             next unless filename =~ /test_fail_/
1818             failures << $~.post_match.to_i
1819         }
1820     end
1821
1822     failureSet = {}
1823
1824     failures.each {
1825         | failure | 
1826         appendFailure($runlist[failure])
1827         failureSet[failure] = true
1828     }
1829
1830     familyMap = {}
1831     $runlist.each_with_index {
1832         | plan, index |
1833         unless familyMap[plan.family]
1834             familyMap[plan.family] = []
1835         end
1836         if failureSet[index]
1837             appendResult(plan, false)
1838             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
1839             next
1840         else
1841             appendResult(plan, true)
1842             familyMap[plan.family] << {:result => "PASS", :plan => plan};
1843         end
1844         appendPass(plan)
1845     }
1846
1847     File.open($outputDir + "resultsByFamily", "w") {
1848         | outp |
1849         first = true
1850         familyMap.keys.sort.each {
1851             | familyName |
1852             if first
1853                 first = false
1854             else
1855                 outp.puts
1856             end
1857             
1858             outp.print "#{familyName}:"
1859
1860             numPassed = 0
1861             familyMap[familyName].each {
1862                 | entry |
1863                 if entry[:result] == "PASS"
1864                     numPassed += 1
1865                 end
1866             }
1867
1868             if numPassed == familyMap[familyName].size
1869                 outp.puts " PASSED"
1870             elsif numPassed == 0
1871                 outp.puts " FAILED"
1872             else
1873                 outp.puts
1874                 familyMap[familyName].each {
1875                     | entry |
1876                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
1877                 }
1878             end
1879         }
1880     }
1881 end
1882
1883 def compressBundle
1884     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
1885     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1886     raise unless system(cmd)
1887 end
1888
1889 def clean(file)
1890     FileUtils.rm_rf file unless $bundle
1891 end
1892
1893 clean($outputDir + "failed")
1894 clean($outputDir + "passed")
1895 clean($outputDir + "results")
1896 clean($outputDir + "resultsByFamily")
1897 clean($outputDir + ".vm")
1898 clean($outputDir + ".helpers")
1899 clean($outputDir + ".runner")
1900 clean($outputDir + ".tests")
1901 clean($outputDir + "_payload")
1902
1903 Dir.mkdir($outputDir) unless $outputDir.directory?
1904
1905 $outputDir = $outputDir.realpath
1906 $runnerDir = $outputDir + ".runner"
1907
1908 if !$numChildProcesses
1909     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
1910         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
1911     else
1912         $numChildProcesses = numberOfProcessors
1913     end
1914 end
1915
1916 if ENV["JSCTEST_timeout"]
1917     # In the worst case, the processors just interfere with each other.
1918     # Increase the timeout proportionally to the number of processors.
1919     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
1920 end
1921
1922 def runBundle
1923     raise unless $bundle
1924
1925     cleanRunnerDirectory
1926     cleanOldResults
1927     runTestRunner
1928     cleanEmptyResultFiles
1929 end
1930
1931 def runNormal
1932     raise if $bundle or $tarball
1933
1934     prepareBundle
1935     prepareTestRunner
1936     runTestRunner
1937     cleanEmptyResultFiles
1938     detectFailures
1939 end
1940
1941 def runTarball
1942     raise unless $tarball
1943
1944     prepareBundle 
1945     prepareTestRunner
1946     compressBundle
1947 end
1948
1949 def runRemote
1950     raise unless $remote
1951
1952     prepareBundle
1953     prepareTestRunner
1954     compressBundle
1955     runTestRunner
1956     detectFailures
1957 end
1958
1959 puts
1960 if $bundle
1961     runBundle
1962 elsif $remote
1963     runRemote
1964 elsif $tarball
1965     runTarball
1966 else
1967     runNormal
1968 end