Unreviewed, rolling out r221868.
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2016 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
47 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
48 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
49 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
50 WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
51 CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
52 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
53 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
54
55 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
56
57 begin
58     require 'shellwords'
59 rescue Exception => e
60     $stderr.puts "Warning: did not find shellwords, not running any tests."
61     exit 0
62 end
63
64 $canRunDisplayProfilerOutput = false
65
66 begin
67     require 'rubygems'
68     require 'json'
69     require 'highline'
70     $canRunDisplayProfilerOutput = true
71 rescue Exception => e
72     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
73     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
74     $stderr.puts "Error: #{e.inspect}"
75 end
76
77 def printCommandArray(*cmd)
78     begin
79         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
80     rescue
81         commandArray = cmd.join(' ')
82     end
83     $stderr.puts ">> #{commandArray}"
84 end
85
86 def mysys(*cmd)
87     printCommandArray(*cmd) if $verbosity >= 1
88     raise "Command failed: #{$?.inspect}" unless system(*cmd)
89 end
90
91 def escapeAll(array)
92     array.map {
93         | v |
94         raise "Detected a non-string in #{inspect}" unless v.is_a? String
95         Shellwords.shellescape(v)
96     }.join(' ')
97 end
98
99
100 $jscPath = nil
101 $doNotMessWithVMPath = false
102 $jitTests = true
103 $memoryLimited = false
104 $outputDir = Pathname.new("results")
105 $verbosity = 0
106 $bundle = nil
107 $tarball = false
108 $tarFileName = "payload.tar.gz"
109 $copyVM = false
110 $testRunnerType = nil
111 $testWriter = "default"
112 $remoteUser = nil
113 $remoteHost = nil
114 $remotePort = nil
115 $remoteDirectory = nil
116 $architecture = nil
117 $hostOS = nil
118 $filter = nil
119 $envVars = []
120 $quickMode = false
121 $buildType = "release"
122 $forceCollectContinuously = false
123
124 def usage
125     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
126     puts
127     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
128     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
129     puts "                            --jsc specifies an already present JavaScriptCore to test."
130     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
131     puts "                            Skip tests tagged with //@skip if $memoryLimited"
132     puts "--no-jit                    Do not run JIT specific tests."
133     puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
134     puts "                            platform."
135     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
136     puts "--verbose            (-v)   Print more things while running."
137     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
138     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
139     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
140     puts "                            e.g. x86, x86_64, arm."
141     puts "--os                        Specify os instead of determining from JavaScriptCore build."
142     puts "                            e.g. darwin, linux & windows."
143     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
144     puts "                            In general the shell runner is slower than the make runner."
145     puts "--make-runner               Uses the faster make-based runner."
146     puts "--ruby-runner               Uses the ruby runner for machines without unix shell or make."
147     puts "--test-writer [writer]      Specifies the test script format."
148     puts "                            default is to use shell scripts to run the tests"
149     puts "                            \"ruby\" to use ruby scripts for systems without a unix shell."
150     puts "--remote                    Specify a remote host on which to run tests from command line argument."
151     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
152     puts "--child-processes    (-c)   Specify the number of child processes."
153     puts "--filter                    Only run tests whose name matches the given regular expression."
154     puts "--help               (-h)   Print this message."
155     puts "--env-vars                  Add a list of environment variables to set before running jsc."
156     puts "                            Each environment variable should be separated by a space."
157     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
158     puts "                            it will override the default value."
159     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
160     exit 1
161 end
162
163 jscArg = nil
164
165 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
166                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
167                ['--no-copy', GetoptLong::NO_ARGUMENT],
168                ['--memory-limited', GetoptLong::NO_ARGUMENT],
169                ['--no-jit', GetoptLong::NO_ARGUMENT],
170                ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
171                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
172                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
173                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
174                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
175                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
176                ['--os', GetoptLong::REQUIRED_ARGUMENT],
177                ['--shell-runner', GetoptLong::NO_ARGUMENT],
178                ['--make-runner', GetoptLong::NO_ARGUMENT],
179                ['--ruby-runner', GetoptLong::NO_ARGUMENT],
180                ['--test-writer', GetoptLong::REQUIRED_ARGUMENT],
181                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
182                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
183                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
184                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
185                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
186                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
187                ['--debug', GetoptLong::NO_ARGUMENT],
188                ['--release', GetoptLong::NO_ARGUMENT],
189                ['--quick', '-q', GetoptLong::NO_ARGUMENT]).each {
190     | opt, arg |
191     case opt
192     when '--help'
193         usage
194     when '--jsc'
195         jscArg = arg
196     when '--no-copy'
197         $doNotMessWithVMPath = true
198     when '--output-dir'
199         $outputDir = Pathname.new(arg)
200     when '--memory-limited'
201         $memoryLimited = true
202     when '--no-jit'
203         $jitTests = false
204     when '--force-collectContinuously'
205         $forceCollectContinuously = true;
206     when '--verbose'
207         $verbosity += 1
208     when '--run-bundle'
209         $bundle = Pathname.new(arg)
210     when '--tarball'
211         $tarball = true
212         $copyVM = true
213         $tarFileName = arg unless arg == ''
214     when '--force-vm-copy'
215         $copyVM = true
216     when '--shell-runner'
217         $testRunnerType = :shell
218     when '--make-runner'
219         $testRunnerType = :make
220     when '--ruby-runner'
221         $testRunnerType = :ruby
222     when '--test-writer'
223         $testWriter = arg
224     when '--remote'
225         $copyVM = true
226         $tarball = true
227         $remote = true
228         uri = URI("ssh://" + arg)
229         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
230     when '--remote-config-file'
231         $remoteConfigFile = arg
232     when '--child-processes'
233         $numChildProcesses = arg.to_i
234     when '--filter'
235         $filter = Regexp.new(arg)
236     when '--arch'
237         $architecture = arg
238     when '--os'
239         $hostOS = arg
240     when '--env-vars'
241         $envVars = arg.gsub(/\s+/, ' ').split(' ')
242     when '--quick'
243         $quickMode = true
244     when '--debug'
245         $buildType = "debug"
246     when '--release'
247         $buildType = "release"
248     end
249 }
250
251 if $remoteConfigFile
252     file = File.read($remoteConfigFile)
253     config = JSON.parse(file)
254
255     if !$remote and config['remote']
256         $copyVM = true
257         $tarball = true
258         $remote = true
259         uri = URI("ssh://" + config['remote'])
260         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
261     end
262
263     if config['remoteDirectory']
264         $remoteDirectory = config['remoteDirectory']
265     end
266 end
267
268 unless jscArg
269     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
270     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
271     command += ($buildType == "release") ? " --release" : " --debug"
272     command += " --executablePath"
273
274     output = `#{command}`.split("\n")
275     if !output.length
276         $stderr.puts "Error: must specify --jsc <path>"
277         exit 1
278     end
279
280     output = output[0]
281     jscArg = Pathname.new(output).join("jsc")
282     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
283     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
284     if !File.file?(jscArg)
285         $stderr.puts "Error: must specify --jsc <path>"
286         exit 1
287     end
288
289     puts "Using the following jsc path: #{jscArg}"
290 end
291
292 if $doNotMessWithVMPath
293     $jscPath = Pathname.new(jscArg)
294 else
295     $jscPath = Pathname.new(jscArg).realpath
296 end
297
298 $progressMeter = ($verbosity == 0 and $stdout.tty?)
299
300 if $bundle
301     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
302     $outputDir = $bundle
303 end
304
305 # Try to determine architecture. Return nil on failure.
306 def machOArchitectureCode
307     begin 
308         otoolLines = `otool -afh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
309         otoolLines.each_with_index {
310             | value, index |
311             if value =~ /magic/ and value =~ /cputype/
312                 return otoolLines[index + 1].split[1].to_i
313             end
314         }
315     rescue
316         $stderr.puts "Warning: unable to execute otool."
317     end
318     $stderr.puts "Warning: unable to determine architecture."
319     nil
320 end
321
322 def determineArchitectureFromMachOBinary
323     code = machOArchitectureCode
324     return nil unless code
325     is64BitFlag = 0x01000000
326     case code
327     when 7
328         "x86"
329     when 7 | is64BitFlag
330         "x86-64"
331     when 12
332         "arm"
333     when 12 | is64BitFlag
334         "arm64"
335     else
336         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
337         nil
338     end
339 end
340
341 def determineArchitectureFromELFBinary
342     f = File.open($jscPath.to_s)
343     data = f.read(19)
344
345     if !(data[0,4] == "\x7F\x45\x4C\x46")
346         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
347         return nil
348     end
349
350     code = data[18].ord
351     case code
352     when 3
353         "x86"
354     when 8
355         "mips"
356     when 62
357         "x86-64"
358     when 40
359         "arm"
360     when 183
361         "arm64"
362     else
363         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
364         nil
365     end
366 end
367
368 def determineArchitectureFromPEBinary
369     f = File.open($jscPath.to_s)
370     data = f.read(1024)
371
372     if !(data[0, 2] == "MZ")
373         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
374         return nil
375     end
376
377     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
378
379     if !(data[peHeaderAddr, 4] == "PE\0\0")
380         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
381         return nil
382     end
383
384     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
385
386     case machine
387     when 0x014c
388         "x86"
389     when 0x8664
390         "x86-64"
391     else
392         $stderr.puts "Warning: unsupported machine type: #{machine}"
393         nil
394     end
395 end
396
397 def determineArchitecture
398     case $hostOS
399     when "darwin"
400         determineArchitectureFromMachOBinary
401     when "linux"
402         determineArchitectureFromELFBinary
403     when "windows"
404         determineArchitectureFromPEBinary
405     else
406         $stderr.puts "Warning: unable to determine architecture on this platform."
407         nil
408     end
409 end
410
411 def determineOS
412     case RbConfig::CONFIG["host_os"]
413     when /darwin/i
414         "darwin"
415     when /linux/i
416         "linux"
417     when /mswin|mingw|cygwin/
418         "windows"
419     else
420         $stderr.puts "Warning: unable to determine host operating system"
421         nil
422     end
423 end
424
425 $hostOS = determineOS unless $hostOS
426 $architecture = determineArchitecture unless $architecture
427 $isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $architecture == "mips" || $hostOS == "windows")
428
429 if !$testRunnerType
430     if $remote and $hostOS == "darwin"
431         $testRunnerType = :shell
432     else
433         $testRunnerType = :make
434     end
435 end
436
437 if $testWriter
438     if /[^-a-zA-Z0-9_]/.match($testWriter)
439         raise "Invalid test writer #{$testWriter} given"
440     end
441 end
442
443 $numFailures = 0
444 $numPasses = 0
445
446 # We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
447 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--maxPerThreadStackUsage=1572864"]
448 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
449 # NOTE: Tests rely on this using scribbleFreeCells.
450 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
451 B3O1_OPTIONS = ["--defaultB3OptLevel=1"]
452 FTL_OPTIONS = ["--useFTLJIT=true"]
453
454 require_relative "webkitruby/jsc-stress-test-writer-#{$testWriter}"
455
456 def shouldCollectContinuously?
457     $buildType == "release" or $forceCollectContinuously
458 end
459
460 COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false"] : []
461
462 $runlist = []
463
464 def frameworkFromJSCPath(jscPath)
465     parentDirectory = jscPath.dirname
466     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
467         parentDirectory.dirname
468     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
469         jscPath.dirname + "JavaScriptCore.framework"
470     else
471         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
472         nil
473     end
474 end
475
476 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
477     dir = Pathname.new(".")
478     $benchmarkDirectory.each_filename {
479         | pathComponent |
480         dir += ".."
481     }
482     dir + resourcePath
483 end
484
485 def pathToVM
486     pathToBundleResourceFromBenchmarkDirectory($jscPath)
487 end
488
489 def pathToHelpers
490     pathToBundleResourceFromBenchmarkDirectory(".helpers")
491 end
492
493 $runCommandOptions = {}
494
495 $uniqueFilenameCounter = 0
496 def uniqueFilename(extension)
497     payloadDir = $outputDir + "_payload"
498     Dir.mkdir payloadDir unless payloadDir.directory?
499     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
500     $uniqueFilenameCounter += 1
501     result
502 end
503
504 def baseOutputName(kind)
505     "#{$collectionName}/#{$benchmark}.#{kind}"
506 end
507
508 def addRunCommand(kind, command, outputHandler, errorHandler)
509     $didAddRunCommand = true
510     name = baseOutputName(kind)
511     if $filter and name !~ $filter
512         return
513     end
514     plan = Plan.new(
515         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
516         errorHandler)
517     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
518         $runlist.unshift plan
519     else
520         $runlist << plan
521     end
522 end
523
524 # Returns true if there were run commands found in the file ($benchmarkDirectory +
525 # $benchmark), in which case those run commands have already been executed. Otherwise
526 # returns false, in which case you're supposed to add your own run commands.
527 def parseRunCommands
528     oldDidAddRunCommand = $didAddRunCommand
529     $didAddRunCommand = false
530
531     Dir.chdir($outputDir) {
532         File.open($benchmarkDirectory + $benchmark) {
533             | inp |
534             inp.each_line {
535                 | line |
536                 begin
537                     doesMatch = line =~ /^\/\/@/
538                 rescue Exception => e
539                     # Apparently this happens in the case of some UTF8 stuff in some files, where
540                     # Ruby tries to be strict and throw exceptions.
541                     next
542                 end
543                 next unless doesMatch
544                 eval $~.post_match
545             }
546         }
547     }
548
549     result = $didAddRunCommand
550     $didAddRunCommand = result or oldDidAddRunCommand
551     result
552 end
553
554 def slow!
555     $runCommandOptions[:isSlow] = true
556 end
557
558 def runWithOutputHandler(kind, outputHandler, *options)
559     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
560 end
561
562 def run(kind, *options)
563     runWithOutputHandler(kind, silentOutputHandler, *options)
564 end
565
566 def runNoFTL(*optionalTestSpecificOptions)
567     run("no-ftl", *optionalTestSpecificOptions)
568 end
569
570 def runWithRAMSize(size, *optionalTestSpecificOptions)
571     run("ram-size-#{size}", "--forceRAMSize=#{size}", *optionalTestSpecificOptions)
572 end
573
574 def runOneLargeHeap(*optionalTestSpecificOptions)
575     if $memoryLimited
576         $didAddRunCommand = true
577         puts "Skipping #{$collectionName}/#{$benchmark}"
578     else
579         run("default", *optionalTestSpecificOptions)
580     end
581 end
582
583 def runNoJIT(*optionalTestSpecificOptions)
584     run("no-jit", "--useJIT=false", *optionalTestSpecificOptions)
585 end
586
587 def runNoLLInt(*optionalTestSpecificOptions)
588     if $jitTests
589         run("no-llint", "--useLLInt=false", *optionalTestSpecificOptions)
590     end
591 end
592
593 # NOTE: Tests rely on this using scribbleFreeCells.
594 def runNoCJITValidate(*optionalTestSpecificOptions)
595     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
596 end
597
598 def runNoCJITValidatePhases(*optionalTestSpecificOptions)
599     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
600 end
601
602 def runNoCJITCollectContinuously(*optionalTestSpecificOptions)
603     run("no-cjit-collect-continuously", *(NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
604 end
605
606 def runDefault(*optionalTestSpecificOptions)
607     run("default", *(FTL_OPTIONS + optionalTestSpecificOptions))
608 end
609
610 def runFTLNoCJIT(*optionalTestSpecificOptions)
611     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
612 end
613
614 def runFTLNoCJITB3O1(*optionalTestSpecificOptions)
615     run("ftl-no-cjit-b3o1", "--useArrayAllocationProfiling=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
616 end
617
618 def runFTLNoCJITValidate(*optionalTestSpecificOptions)
619     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
620 end
621
622 def runFTLNoCJITNoPutStackValidate(*optionalTestSpecificOptions)
623     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
624 end
625
626 def runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
627     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
628 end
629
630 def runFTLNoCJITOSRValidation(*optionalTestSpecificOptions)
631     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
632 end
633
634 def runDFGEager(*optionalTestSpecificOptions)
635     run("dfg-eager", *(EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
636 end
637
638 def runDFGEagerNoCJITValidate(*optionalTestSpecificOptions)
639     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
640 end
641
642 def runFTLEager(*optionalTestSpecificOptions)
643     run("ftl-eager", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
644 end
645
646 def runFTLEagerWatchdog(*optionalTestSpecificOptions)
647     timeout = rand(100)
648     run("ftl-eager-watchdog-#{timeout}", "--watchdog=#{timeout}", "--watchdog-exception-ok", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
649 end
650
651 def runFTLEagerNoCJITValidate(*optionalTestSpecificOptions)
652     run("ftl-eager-no-cjit", "--validateGraph=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
653 end
654
655 def runFTLEagerNoCJITB3O1(*optionalTestSpecificOptions)
656     run("ftl-eager-no-cjit-b3o1", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
657 end
658
659 def runFTLEagerNoCJITOSRValidation(*optionalTestSpecificOptions)
660     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
661 end
662
663 def runNoCJITNoASO(*optionalTestSpecificOptions)
664     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
665 end
666
667 def runNoCJITNoAccessInlining(*optionalTestSpecificOptions)
668     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
669 end
670
671 def runFTLNoCJITNoAccessInlining(*optionalTestSpecificOptions)
672     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
673 end
674
675 def runFTLNoCJITSmallPool(*optionalTestSpecificOptions)
676     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
677 end
678
679 def runNoCJIT(*optionalTestSpecificOptions)
680     run("no-cjit", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
681 end
682
683 def runDFGMaximalFlushPhase(*optionalTestSpecificOptions)
684     run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
685 end
686
687 def runShadowChicken(*optionalTestSpecificOptions)
688     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true", *optionalTestSpecificOptions)
689 end
690
691 def defaultRun
692     if $quickMode
693         defaultQuickRun
694     else
695         runDefault
696         if $jitTests
697             runNoLLInt
698             runNoCJITValidatePhases
699             runNoCJITCollectContinuously if shouldCollectContinuously?
700             runDFGEager
701             runDFGEagerNoCJITValidate
702             runDFGMaximalFlushPhase
703
704             return if !$isFTLPlatform
705
706             runNoFTL
707             runFTLNoCJITValidate
708             runFTLNoCJITB3O1
709             runFTLNoCJITNoPutStackValidate
710             runFTLNoCJITNoInlineValidate
711             runFTLEager
712             runFTLEagerNoCJITValidate
713             runFTLEagerNoCJITB3O1
714             runFTLNoCJITSmallPool
715         end
716     end
717 end
718
719 def defaultNoNoLLIntRun
720     if $quickMode
721         defaultQuickRun
722     else
723         runDefault
724         if $jitTests
725             runNoCJITValidatePhases
726             runNoCJITCollectContinuously if shouldCollectContinuously?
727             runDFGEager
728             runDFGEagerNoCJITValidate
729             runDFGMaximalFlushPhase
730
731             return if !$isFTLPlatform
732
733             runNoFTL
734             runFTLNoCJITValidate
735             runFTLNoCJITB3O1
736             runFTLNoCJITNoPutStackValidate
737             runFTLNoCJITNoInlineValidate
738             runFTLEager
739             runFTLEagerNoCJITValidate
740             runFTLNoCJITSmallPool
741         end
742     end
743 end
744
745 def defaultQuickRun
746     runDefault
747     if $jitTests
748         runNoCJITValidate
749
750         return if $isFTLPlatform
751
752         runNoFTL
753         runFTLNoCJITValidate
754     end
755 end
756
757 def defaultSpotCheckNoMaximalFlush
758     defaultQuickRun
759     runNoCJITNoAccessInlining
760
761     return if !$isFTLPlatform
762
763     runFTLNoCJITOSRValidation
764     runFTLNoCJITNoAccessInlining
765     runFTLNoCJITB3O1
766 end
767
768 def defaultSpotCheck
769     defaultSpotCheckNoMaximalFlush
770     runDFGMaximalFlushPhase
771 end
772
773 # This is expected to not do eager runs because eager runs can have a lot of recompilations
774 # for reasons that don't arise in the real world. It's used for tests that assert convergence
775 # by counting recompilations.
776 def defaultNoEagerRun
777     runDefault
778     if $jitTests
779         runNoLLInt
780         runNoCJITValidatePhases
781         runNoCJITCollectContinuously if shouldCollectContinuously?
782
783         return if !$isFTLPlatform
784
785         runNoFTL
786         runFTLNoCJITValidate
787         runFTLNoCJITNoInlineValidate
788         runFTLNoCJITB3O1
789     end
790 end
791
792 def defaultNoSamplingProfilerRun
793     runDefault
794     if $jitTests
795         runNoLLInt
796         runNoCJITValidatePhases
797         runNoCJITCollectContinuously if shouldCollectContinuously?
798         runDFGEager
799         runDFGEagerNoCJITValidate
800         runDFGMaximalFlushPhase
801
802         return if !$isFTLPlatform
803
804         runNoFTL
805         runFTLNoCJITNoPutStackValidate
806         runFTLNoCJITNoInlineValidate
807         runFTLEager
808         runFTLEagerNoCJITValidate
809         runFTLNoCJITSmallPool
810     end
811 end
812
813 def runProfiler
814     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
815         skip
816         return
817     end
818
819     profilerOutput = uniqueFilename(".json")
820     if $canRunDisplayProfilerOutput
821         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
822     else
823         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
824         run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
825     end
826 end
827
828 def runExceptionFuzz
829     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
830     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
831 end
832
833 def runExecutableAllocationFuzz(name, *options)
834     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
835     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
836 end
837
838 def runTypeProfiler
839     if !$jitTests
840         return
841     end
842
843     return if !$isFTLPlatform
844
845     run("ftl-no-cjit-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
846     run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
847 end
848
849 def runControlFlowProfiler
850     if !$jitTests
851         return
852     end
853
854     return if !$isFTLPlatform
855
856     run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
857 end
858
859 def runTest262(mode, exception, includeFiles, flags)
860     failsWithException = exception != "NoException"
861     isStrict = false
862     isModule = false
863     isAsync = false
864
865     flags.each {
866         | flag |
867         case flag
868         when :strict
869             isStrict = true
870         when :module
871             isModule = true
872         when :async
873             isAsync = true
874         else
875             raise "Invalid flag for runTest262, #{flag}"
876         end
877     }
878
879     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
880
881     args = [pathToVM.to_s] + BASE_OPTIONS
882     args << "--exception=" + exception if failsWithException
883     args << "--test262-async" if isAsync
884     args += includeFiles
885
886     case mode
887     when :normal
888         errorHandler = simpleErrorHandler
889         outputHandler = silentOutputHandler
890     when :fail
891         errorHandler = expectedFailErrorHandler
892         outputHandler = noisyOutputHandler
893     when :failDueToOutdatedOrBadTest
894         errorHandler = expectedFailErrorHandler
895         outputHandler = noisyOutputHandler
896     else
897         raise "Invalid mode: #{mode}"
898     end
899
900     if isStrict
901         kind = "default-strict"
902         args << "--strict-file=#{$benchmark}"
903     else
904         kind = "default"
905         if isModule
906             args << "--module-file=#{$benchmark}"
907         else
908             args << $benchmark.to_s
909         end
910     end
911
912     addRunCommand(kind, args, outputHandler, errorHandler)
913 end
914
915 def prepareTest262Fixture
916     # This function is used to add the files used by Test262 modules tests.
917     prepareExtraRelativeFiles([""], $collection)
918 end
919
920 def runES6(mode)
921     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
922     case mode
923     when :normal
924         errorHandler = simpleErrorHandler
925     when :fail
926         errorHandler = expectedFailErrorHandler
927     when :failDueToOutdatedOrBadTest
928         errorHandler = expectedFailErrorHandler
929     else
930         raise "Invalid mode: #{mode}"
931     end
932     addRunCommand("default", args, noisyOutputHandler, errorHandler)
933 end
934
935 def runModules
936     run("default-modules", "-m")
937
938     if !$jitTests
939         return
940     end
941
942     run("no-llint-modules", "-m", "--useLLInt=false")
943     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
944     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
945     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
946
947     return if !$isFTLPlatform
948
949     run("default-ftl-modules", "-m", *FTL_OPTIONS)
950     run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
951     run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
952     run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
953     run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
954     run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
955 end
956
957 def runWebAssembly
958     return if !$jitTests
959     return if !$isFTLPlatform
960     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
961     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
962     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
963     run("default-wasm", "-m", *FTL_OPTIONS)
964     if !$quickMode
965         run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
966         run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
967         run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
968         run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
969         run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
970     end
971 end
972
973 def runWebAssemblyEmscripten(mode)
974     case mode
975     when :skip
976         return
977     end
978     return if !$jitTests
979     return if !$isFTLPlatform
980     wasm = $benchmark.to_s.sub! '.js', '.wasm'
981     prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
982     run("default-wasm", *FTL_OPTIONS)
983     if !$quickMode
984         run("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
985         run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
986         run("wasm-no-call-ic", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
987         run("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
988     end
989 end
990
991 def runWebAssemblySpecTest(mode)
992     case mode
993     when :skip
994         return
995     end
996     return if !$jitTests
997     return if !$isFTLPlatform
998     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
999
1000     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1001     prepareExtraRelativeFiles(modules.map { |f| "../../" + f }, $collection)
1002
1003     harness = Dir[WASMTESTS_PATH + "spec-harness/" + "*.js"].map { |f| File.basename(f) }
1004     prepareExtraRelativeFiles(harness.map { |f| "../../spec-harness/" + f }, $collection)
1005
1006     runWithOutputHandler("default-wasm", noisyOutputHandler, "../spec-harness.js", *FTL_OPTIONS)
1007     if !$quickMode
1008       runWithOutputHandler("wasm-no-cjit-yes-tls-context", noisyOutputHandler, "../spec-harness.js",  "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1009       runWithOutputHandler("wasm-eager-jettison", noisyOutputHandler, "../spec-harness.js", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1010       runWithOutputHandler("wasm-no-call-ic", noisyOutputHandler, "../spec-harness.js", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1011       runWithOutputHandler("wasm-no-tls-context", noisyOutputHandler, "../spec-harness.js", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1012     end
1013 end
1014
1015 def runWebAssemblyLowExecutableMemory(*optionalTestSpecificOptions)
1016     return if !$jitTests
1017     return if !$isFTLPlatform
1018     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1019     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1020     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1021     # Only let WebAssembly get executable memory.
1022     run("default-wasm", "--useConcurrentGC=0" , "--useConcurrentJIT=0", "--jitMemoryReservationSize=15000", "--useBaselineJIT=0", "--useDFGJIT=0", "--useFTLJIT=0", "-m")
1023 end
1024
1025 def runChakra(mode, exception, baselineFile, extraFiles)
1026     raise unless $benchmark.to_s =~ /\.js$/
1027     failsWithException = exception != "NoException"
1028     testName = $~.pre_match
1029
1030     prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
1031     prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)
1032
1033     args = [pathToVM.to_s] + BASE_OPTIONS
1034     args += FTL_OPTIONS if $isFTLPlatform
1035     args += EAGER_OPTIONS
1036     args << "--exception=" + exception if failsWithException
1037     args << "--dumpException" if failsWithException
1038     args += ["jsc-lib.js"]
1039
1040     case mode
1041     when :baseline
1042         prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
1043         errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
1044         outputHandler = noisyOutputHandler
1045     when :pass
1046         errorHandler = chakraPassFailErrorHandler
1047         outputHandler = noisyOutputHandler
1048     when :skipDueToOutdatedOrBadTest
1049         return
1050     when :skip
1051         return
1052     else
1053         raise "Invalid mode: #{mode}"
1054     end
1055
1056     kind = "default"
1057     args << $benchmark.to_s
1058
1059     addRunCommand(kind, args, outputHandler, errorHandler)
1060 end
1061
1062 def runLayoutTest(kind, *options)
1063     raise unless $benchmark.to_s =~ /\.js$/
1064     testName = $~.pre_match
1065     if kind
1066         kind = "layout-" + kind
1067     else
1068         kind = "layout"
1069     end
1070
1071     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1072     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1073
1074     args = [pathToVM.to_s] + BASE_OPTIONS + options +
1075         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1076          $benchmark.to_s,
1077          (Pathname.new("resources") + "standalone-post.js").to_s]
1078     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1079 end
1080
1081 def runLayoutTestNoFTL
1082     runLayoutTest("no-ftl")
1083 end
1084
1085 def runLayoutTestNoLLInt
1086     runLayoutTest("no-llint", "--useLLInt=false")
1087 end
1088
1089 def runLayoutTestNoCJIT
1090     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1091 end
1092
1093 def runLayoutTestDFGEagerNoCJIT
1094     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1095 end
1096
1097 def runLayoutTestDefault
1098     runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
1099 end
1100
1101 def runLayoutTestFTLNoCJIT
1102     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1103 end
1104
1105 def runLayoutTestFTLEagerNoCJIT
1106     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1107 end
1108
1109 def runLayoutTestFTLEagerNoCJITB3O1
1110     runLayoutTest("ftl-eager-no-cjit-b3o1", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS))
1111 end
1112
1113 def noFTLRunLayoutTest
1114     if !$jitTests
1115         return
1116     end
1117
1118     runLayoutTestNoLLInt
1119     runLayoutTestNoCJIT
1120     runLayoutTestDFGEagerNoCJIT
1121 end
1122
1123 def defaultQuickRunLayoutTest
1124     runLayoutTestDefault
1125     if $jitTests
1126         if $isFTLPlatform
1127             runLayoutTestNoFTL
1128             runLayoutTestFTLNoCJIT
1129             runLayoutTestFTLEagerNoCJIT
1130         else
1131             noFTLRunLayoutTest
1132         end
1133     end
1134 end
1135
1136 def defaultRunLayoutTest
1137     if $quickMode
1138         defaultQuickRunLayoutTest
1139     else
1140         runLayoutTestDefault
1141         if $jitTests
1142             noFTLRunLayoutTest
1143
1144             return if !$isFTLPlatform
1145
1146             runLayoutTestNoFTL
1147             runLayoutTestFTLNoCJIT
1148             runLayoutTestFTLEagerNoCJIT
1149         end
1150     end
1151 end
1152
1153 def noEagerNoNoLLIntTestsRunLayoutTest
1154     runLayoutTestDefault
1155     if $jitTests
1156         runLayoutTestNoCJIT
1157
1158         return if !$isFTLPlatform
1159
1160         runLayoutTestNoFTL
1161         runLayoutTestFTLNoCJIT
1162     end
1163 end
1164
1165 def noNoLLIntRunLayoutTest
1166     runLayoutTestDefault
1167     if $jitTests
1168         runLayoutTestNoCJIT
1169         runLayoutTestDFGEagerNoCJIT
1170
1171         return if !$isFTLPlatform
1172
1173         runLayoutTestNoFTL
1174         runLayoutTestFTLNoCJIT
1175         runLayoutTestFTLEagerNoCJIT
1176     end
1177 end
1178
1179 def prepareExtraRelativeFiles(extraFiles, destination)
1180     Dir.chdir($outputDir) {
1181         extraFiles.each {
1182             | file |
1183             dest = destination + file
1184             FileUtils.mkdir_p(dest.dirname)
1185             FileUtils.cp $extraFilesBaseDir + file, dest
1186         }
1187     }
1188 end
1189
1190 def baseDirForCollection(collectionName)
1191     Pathname(".tests") + collectionName
1192 end
1193
1194 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1195     raise unless absoluteBase.absolute?
1196     Dir.chdir($outputDir) {
1197         collectionBaseDir = baseDirForCollection($collectionName)
1198         extraFiles.each {
1199             | file |
1200             destination = collectionBaseDir + file
1201             FileUtils.mkdir_p destination.dirname unless destination.directory?
1202             FileUtils.cp absoluteBase + file, destination
1203         }
1204     }
1205 end
1206
1207 def runMozillaTest(kind, mode, extraFiles, *options)
1208     if kind
1209         kind = "mozilla-" + kind
1210     else
1211         kind = "mozilla"
1212     end
1213     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1214     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1215     case mode
1216     when :normal
1217         errorHandler = mozillaErrorHandler
1218     when :negative
1219         errorHandler = mozillaExit3ErrorHandler
1220     when :fail
1221         errorHandler = mozillaFailErrorHandler
1222     when :failDueToOutdatedOrBadTest
1223         errorHandler = mozillaFailErrorHandler
1224     when :skip
1225         return
1226     else
1227         raise "Invalid mode: #{mode}"
1228     end
1229     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1230 end
1231
1232 def runMozillaTestDefault(mode, *extraFiles)
1233     runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
1234 end
1235
1236 def runMozillaTestNoFTL(mode, *extraFiles)
1237     runMozillaTest("no-ftl", mode, extraFiles)
1238 end
1239
1240 def runMozillaTestLLInt(mode, *extraFiles)
1241     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1242 end
1243
1244 def runMozillaTestBaselineJIT(mode, *extraFiles)
1245     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1246 end
1247
1248 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1249     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1250 end
1251
1252 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1253     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1254 end
1255
1256 def defaultQuickRunMozillaTest(mode, *extraFiles)
1257     if $jitTests
1258         runMozillaTestDefault(mode, *extraFiles)
1259         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1260     else
1261         runMozillaTestNoFTL(mode, *extraFiles)
1262         if $jitTests
1263             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1264         end
1265     end
1266 end
1267
1268 def defaultRunMozillaTest(mode, *extraFiles)
1269     if $quickMode
1270         defaultQuickRunMozillaTest(mode, *extraFiles)
1271     else
1272         runMozillaTestNoFTL(mode, *extraFiles)
1273         if $jitTests
1274             runMozillaTestLLInt(mode, *extraFiles)
1275             runMozillaTestBaselineJIT(mode, *extraFiles)
1276             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1277             runMozillaTestDefault(mode, *extraFiles)
1278             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
1279         end
1280     end
1281 end
1282
1283 def runNoisyTest(kind, *options)
1284     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
1285 end
1286
1287 def runNoisyTestDefault
1288     runNoisyTest("default", *FTL_OPTIONS)
1289 end
1290
1291 def runNoisyTestNoFTL
1292     runNoisyTest("no-ftl")
1293 end
1294
1295 def runNoisyTestNoCJIT
1296     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1297 end
1298
1299 def runNoisyTestNoCJITB3O1
1300     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS))
1301 end
1302
1303 def runNoisyTestEagerNoCJIT
1304     runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1305 end
1306
1307 def defaultRunNoisyTest
1308     runNoisyTestDefault
1309     if $jitTests and $isFTLPlatform
1310         runNoisyTestNoFTL
1311         runNoisyTestNoCJIT
1312         runNoisyTestNoCJITB3O1
1313         runNoisyTestEagerNoCJIT
1314     end
1315 end
1316
1317 def skip
1318     $didAddRunCommand = true
1319     puts "Skipping #{$collectionName}/#{$benchmark}"
1320 end
1321
1322 def allJSFiles(path)
1323     if path.file?
1324         [path]
1325     else
1326         result = []
1327         Dir.foreach(path) {
1328             | filename |
1329             next unless filename =~ /\.js$/
1330             next unless (path + filename).file?
1331             result << path + filename
1332         }
1333         result
1334     end
1335 end
1336
1337 def uniqueifyName(names, name)
1338     result = name.to_s
1339     toAdd = 1
1340     while names[result]
1341         result = "#{name}-#{toAdd}"
1342         toAdd += 1
1343     end
1344     names[result] = true
1345     result
1346 end
1347
1348 def simplifyCollectionName(collectionPath)
1349     outerDir = collectionPath.dirname
1350     name = collectionPath.basename
1351     lastName = name
1352     if collectionPath.directory?
1353         while lastName.to_s =~ /test/
1354             lastName = outerDir.basename
1355             name = lastName + name
1356             outerDir = outerDir.dirname
1357         end
1358     end
1359     uniqueifyName($collectionNames, name)
1360 end
1361
1362 def prepareCollection(name)
1363     FileUtils.mkdir_p $outputDir + name
1364
1365     absoluteCollection = $collection.realpath
1366
1367     Dir.chdir($outputDir) {
1368         bundleDir = baseDirForCollection(name)
1369
1370         # Create the proper directory structures.
1371         FileUtils.mkdir_p bundleDir
1372         if bundleDir.basename == $collection.basename
1373             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1374             $collection = bundleDir
1375         else
1376             FileUtils.cp_r absoluteCollection, bundleDir
1377             $collection = bundleDir + $collection.basename
1378         end
1379
1380         $extraFilesBaseDir = absoluteCollection
1381     }
1382 end
1383
1384 $collectionNames = {}
1385
1386 def handleCollectionFile(collection)
1387     collectionName = simplifyCollectionName(collection)
1388    
1389     paths = {}
1390     subCollections = []
1391     YAML::load(IO::read(collection)).each {
1392         | entry |
1393         if entry["collection"]
1394             subCollections << entry["collection"]
1395             next
1396         end
1397         
1398         if Pathname.new(entry["path"]).absolute?
1399             raise "Absolute path: " + entry["path"] + " in #{collection}"
1400         end
1401         
1402         if paths[entry["path"]]
1403             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1404         end
1405         
1406         subCollection = collection.dirname + entry["path"]
1407         
1408         if subCollection.file?
1409             subCollectionName = Pathname.new(entry["path"]).dirname
1410         else
1411             subCollectionName = entry["path"]
1412         end
1413         
1414         $collection = subCollection
1415         $collectionName = Pathname.new(collectionName)
1416         Pathname.new(subCollectionName).each_filename {
1417             | filename |
1418             next if filename =~ /^\./
1419             $collectionName += filename
1420         }
1421         $collectionName = $collectionName.to_s
1422         
1423         prepareCollection($collectionName)
1424       
1425         Dir.chdir($outputDir) {
1426             pathsToSearch = [$collection]
1427             if entry["tests"]
1428                 if entry["tests"].is_a? Array
1429                     pathsToSearch = entry["tests"].map {
1430                         | testName |
1431                         pathsToSearch[0] + testName
1432                     }
1433                 else
1434                     pathsToSearch[0] += entry["tests"]
1435                 end
1436             end
1437             pathsToSearch.each {
1438                 | pathToSearch |
1439                 allJSFiles(pathToSearch).each {
1440                     | path |
1441                     
1442                     $benchmark = path.basename
1443                     $benchmarkDirectory = path.dirname
1444                     
1445                     $runCommandOptions = {}
1446                     eval entry["cmd"]
1447                 }
1448             }
1449         }
1450     }
1451     
1452     subCollections.each {
1453         | subCollection |
1454         handleCollection(collection.dirname + subCollection)
1455     }
1456 end
1457
1458 def handleCollectionDirectory(collection)
1459     collectionName = simplifyCollectionName(collection)
1460     
1461     $collection = collection
1462     $collectionName = collectionName
1463     prepareCollection(collectionName)
1464    
1465     Dir.chdir($outputDir) {
1466         $benchmarkDirectory = $collection
1467         allJSFiles($collection).each {
1468             | path |
1469             
1470             $benchmark = path.basename
1471             
1472             $runCommandOptions = {}
1473             defaultRun unless parseRunCommands
1474         }
1475     }
1476 end
1477
1478 def handleCollection(collection)
1479     collection = Pathname.new(collection)
1480     
1481     if collection.file?
1482         handleCollectionFile(collection)
1483     else
1484         handleCollectionDirectory(collection)
1485     end
1486 end
1487
1488 def appendFailure(plan)
1489     File.open($outputDir + "failed", "a") {
1490         | outp |
1491         outp.puts plan.name
1492     }
1493     $numFailures += 1
1494 end
1495
1496 def appendPass(plan)
1497     File.open($outputDir + "passed", "a") {
1498         | outp |
1499         outp.puts plan.name
1500     }
1501     $numPasses += 1
1502 end
1503
1504 def appendResult(plan, didPass)
1505     File.open($outputDir + "results", "a") {
1506         | outp |
1507         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1508     }
1509 end
1510
1511 def prepareBundle
1512     raise if $bundle
1513
1514     if $doNotMessWithVMPath
1515         if !$remote and !$tarball
1516             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1517             $jscPath = Pathname.new($jscPath).realpath
1518         else
1519             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1520         end
1521     else
1522         originalJSCPath = $jscPath
1523         vmDir = $outputDir + ".vm"
1524         FileUtils.mkdir_p vmDir
1525         
1526         frameworkPath = frameworkFromJSCPath($jscPath)
1527         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1528         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1529         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1530
1531         if frameworkPath
1532             source = frameworkPath
1533             destination = Pathname.new(".vm")
1534         else
1535             source = originalJSCPath
1536             destination = $jscPath
1537
1538             Dir.chdir($outputDir) {
1539                 FileUtils.mkdir_p $jscPath.dirname
1540             }
1541         end
1542
1543         Dir.chdir($outputDir) {
1544             if $copyVM
1545                 FileUtils.cp_r source, destination
1546             else
1547                 begin 
1548                     FileUtils.ln_s source, destination
1549                 rescue Exception
1550                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1551                     FileUtils.cp_r source, destination
1552                 end
1553             end
1554
1555             if $remote and $hostOS == "linux"
1556                 begin
1557                     dependencies = `ldd #{source}`
1558                     dependencies.split(/\n/).each {
1559                         | dependency |
1560                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1561                     }
1562                 rescue
1563                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1564                 end
1565             end
1566         }
1567     end
1568     
1569     Dir.chdir($outputDir) {
1570         FileUtils.cp_r HELPERS_PATH, ".helpers"
1571     }
1572
1573     ARGV.each {
1574         | collection |
1575         handleCollection(collection)
1576     }
1577
1578     puts
1579 end
1580
1581 def cleanOldResults
1582     raise unless $bundle
1583
1584     eachResultFile($outputDir) {
1585         | path |
1586         FileUtils.rm_f path
1587     }
1588 end
1589
1590 def cleanEmptyResultFiles
1591     eachResultFile($outputDir) {
1592         | path |
1593         next unless path.basename.to_s =~ /\.out$/
1594         next unless FileTest.size(path) == 0
1595         FileUtils.rm_f path
1596     }
1597 end
1598
1599 def eachResultFile(startingDir, &block)
1600     dirsToClean = [startingDir]
1601     until dirsToClean.empty?
1602         nextDir = dirsToClean.pop
1603         Dir.foreach(nextDir) {
1604             | entry |
1605             next if entry =~ /^\./
1606             path = nextDir + entry
1607             if path.directory?
1608                 dirsToClean.push(path)
1609             else
1610                 block.call(path)
1611             end
1612         }
1613     end
1614 end
1615
1616 def prepareTestRunner
1617     raise if $bundle
1618
1619     $runlist.each_with_index {
1620         | plan, index |
1621         plan.index = index
1622     }
1623
1624     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1625     toDelete = []
1626     Dir.foreach($runnerDir) {
1627         | filename |
1628         if filename =~ /^test_/
1629             toDelete << filename
1630         end
1631     }
1632     
1633     toDelete.each {
1634         | filename |
1635         File.unlink($runnerDir + filename)
1636     }
1637
1638     $runlist.each {
1639         | plan |
1640         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1641     }
1642
1643     case $testRunnerType
1644     when :make
1645         prepareMakeTestRunner
1646     when :shell
1647         prepareShellTestRunner
1648     when :ruby
1649         prepareRubyTestRunner
1650     else
1651         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1652     end
1653 end
1654
1655 def cleanRunnerDirectory
1656     raise unless $bundle
1657     Dir.foreach($runnerDir) {
1658         | filename |
1659         next unless filename =~ /^test_fail/
1660         FileUtils.rm_f $runnerDir + filename
1661     }
1662 end
1663
1664 def sshRead(cmd)
1665     raise unless $remote
1666
1667     result = ""
1668     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1669       | inp |
1670       inp.each_line {
1671         | line |
1672         result += line
1673       }
1674     }
1675     raise "#{$?}" unless $?.success?
1676     result
1677 end
1678
1679 def runCommandOnTester(cmd)
1680     if $remote
1681         result = sshRead(cmd)
1682     else
1683         result = `#{cmd}`
1684     end
1685 end
1686
1687 def numberOfProcessors
1688     if $hostOS == "windows"
1689         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1690     else
1691         begin
1692             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1693         rescue
1694             numProcessors = 0
1695         end
1696
1697         if numProcessors == 0
1698             begin
1699                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1700             rescue
1701                 numProcessors == 0
1702             end
1703         end
1704     end
1705
1706     if numProcessors == 0
1707         numProcessors = 1
1708     end
1709     return numProcessors
1710 end
1711
1712 def runAndMonitorTestRunnerCommand(*cmd)
1713     numberOfTests = 0
1714     Dir.chdir($runnerDir) {
1715         # -1 for the runscript, and -2 for '..' and '.'
1716         numberOfTests = Dir.entries(".").count - 3
1717     }
1718     unless $progressMeter
1719         mysys(cmd.join(' '))
1720     else
1721        running = {}
1722        didRun = {}
1723        didFail = {}
1724        blankLine = true
1725        prevStringLength = 0
1726        IO.popen(cmd.join(' '), mode="r") {
1727            | inp |
1728            inp.each_line {
1729                | line |
1730                line = line.scrub.chomp
1731                if line =~ /^Running /
1732                    running[$~.post_match] = true
1733                elsif line =~ /^PASS: /
1734                    didRun[$~.post_match] = true
1735                elsif line =~ /^FAIL: /
1736                    didRun[$~.post_match] = true
1737                    didFail[$~.post_match] = true
1738                else
1739                    unless blankLine
1740                        print("\r" + " " * prevStringLength + "\r")
1741                    end
1742                    puts line
1743                    blankLine = true
1744                end
1745
1746                def lpad(str, chars)
1747                    str = str.to_s
1748                    if str.length > chars
1749                        str
1750                    else
1751                       "%#{chars}s"%(str)
1752                    end
1753                end
1754
1755                string  = ""
1756                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1757                unless didFail.empty?
1758                    string += " (failed #{didFail.size})"
1759                end
1760                string += " "
1761                (running.size - didRun.size).times {
1762                    string += "."
1763                }
1764                if string.length < prevStringLength
1765                    print string
1766                    print(" " * (prevStringLength - string.length))
1767                end
1768                print string
1769                prevStringLength = string.length
1770                blankLine = false
1771                $stdout.flush
1772            }
1773        }
1774        puts
1775        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1776     end
1777 end
1778
1779 def runTestRunner
1780     if $remote
1781         if !$remoteDirectory
1782             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1783         end
1784         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1785         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1786         remoteScript = "\""
1787         remoteScript += "cd #{$remoteDirectory} && "
1788         remoteScript += "rm -rf #{$outputDir.basename} && "
1789         remoteScript += "tar xzf #{$tarFileName} && "
1790         remoteScript += "cd #{$outputDir.basename}/.runner && "
1791         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1792         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1793         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
1794         $envVars.each { |var| remoteScript += "export " << var << "\n" }
1795         remoteScript += "#{testRunnerCommand}\""
1796         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1797     else
1798         Dir.chdir($runnerDir) {
1799             runAndMonitorTestRunnerCommand(testRunnerCommand)
1800         }
1801     end
1802 end
1803
1804 def detectFailures
1805     raise if $bundle
1806
1807     failures = []
1808     
1809     if $remote
1810         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1811         output.split(/\n/).each {
1812             | line |
1813             next unless line =~ /test_fail_/
1814             failures << $~.post_match.to_i
1815         }
1816     else
1817         Dir.foreach($runnerDir) {
1818             | filename |
1819             next unless filename =~ /test_fail_/
1820             failures << $~.post_match.to_i
1821         }
1822     end
1823
1824     failureSet = {}
1825
1826     failures.each {
1827         | failure | 
1828         appendFailure($runlist[failure])
1829         failureSet[failure] = true
1830     }
1831
1832     familyMap = {}
1833     $runlist.each_with_index {
1834         | plan, index |
1835         unless familyMap[plan.family]
1836             familyMap[plan.family] = []
1837         end
1838         if failureSet[index]
1839             appendResult(plan, false)
1840             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
1841             next
1842         else
1843             appendResult(plan, true)
1844             familyMap[plan.family] << {:result => "PASS", :plan => plan};
1845         end
1846         appendPass(plan)
1847     }
1848
1849     File.open($outputDir + "resultsByFamily", "w") {
1850         | outp |
1851         first = true
1852         familyMap.keys.sort.each {
1853             | familyName |
1854             if first
1855                 first = false
1856             else
1857                 outp.puts
1858             end
1859             
1860             outp.print "#{familyName}:"
1861
1862             numPassed = 0
1863             familyMap[familyName].each {
1864                 | entry |
1865                 if entry[:result] == "PASS"
1866                     numPassed += 1
1867                 end
1868             }
1869
1870             if numPassed == familyMap[familyName].size
1871                 outp.puts " PASSED"
1872             elsif numPassed == 0
1873                 outp.puts " FAILED"
1874             else
1875                 outp.puts
1876                 familyMap[familyName].each {
1877                     | entry |
1878                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
1879                 }
1880             end
1881         }
1882     }
1883 end
1884
1885 def compressBundle
1886     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
1887     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1888     raise unless system(cmd)
1889 end
1890
1891 def clean(file)
1892     FileUtils.rm_rf file unless $bundle
1893 end
1894
1895 clean($outputDir + "failed")
1896 clean($outputDir + "passed")
1897 clean($outputDir + "results")
1898 clean($outputDir + "resultsByFamily")
1899 clean($outputDir + ".vm")
1900 clean($outputDir + ".helpers")
1901 clean($outputDir + ".runner")
1902 clean($outputDir + ".tests")
1903 clean($outputDir + "_payload")
1904
1905 Dir.mkdir($outputDir) unless $outputDir.directory?
1906
1907 $outputDir = $outputDir.realpath
1908 $runnerDir = $outputDir + ".runner"
1909
1910 if !$numChildProcesses
1911     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
1912         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
1913     else
1914         $numChildProcesses = numberOfProcessors
1915     end
1916 end
1917
1918 if ENV["JSCTEST_timeout"]
1919     # In the worst case, the processors just interfere with each other.
1920     # Increase the timeout proportionally to the number of processors.
1921     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
1922 end
1923
1924 def runBundle
1925     raise unless $bundle
1926
1927     cleanRunnerDirectory
1928     cleanOldResults
1929     runTestRunner
1930     cleanEmptyResultFiles
1931 end
1932
1933 def runNormal
1934     raise if $bundle or $tarball
1935
1936     prepareBundle
1937     prepareTestRunner
1938     runTestRunner
1939     cleanEmptyResultFiles
1940     detectFailures
1941 end
1942
1943 def runTarball
1944     raise unless $tarball
1945
1946     prepareBundle 
1947     prepareTestRunner
1948     compressBundle
1949 end
1950
1951 def runRemote
1952     raise unless $remote
1953
1954     prepareBundle
1955     prepareTestRunner
1956     compressBundle
1957     runTestRunner
1958     detectFailures
1959 end
1960
1961 puts
1962 if $bundle
1963     runBundle
1964 elsif $remote
1965     runRemote
1966 elsif $tarball
1967     runTarball
1968 else
1969     runNormal
1970 end