459254514e393b39d34035b73bf840b176747e1f
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2016 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
47 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
48 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
49 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
50 WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
51 CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
52 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
53 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
54
55 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
56
57 begin
58     require 'shellwords'
59 rescue Exception => e
60     $stderr.puts "Warning: did not find shellwords, not running any tests."
61     exit 0
62 end
63
64 $canRunDisplayProfilerOutput = false
65
66 begin
67     require 'rubygems'
68     require 'json'
69     require 'highline'
70     $canRunDisplayProfilerOutput = true
71 rescue Exception => e
72     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
73     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
74     $stderr.puts "Error: #{e.inspect}"
75 end
76
77 def printCommandArray(*cmd)
78     begin
79         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
80     rescue
81         commandArray = cmd.join(' ')
82     end
83     $stderr.puts ">> #{commandArray}"
84 end
85
86 def mysys(*cmd)
87     printCommandArray(*cmd) if $verbosity >= 1
88     raise "Command failed: #{$?.inspect}" unless system(*cmd)
89 end
90
91 def escapeAll(array)
92     array.map {
93         | v |
94         raise "Detected a non-string in #{inspect}" unless v.is_a? String
95         Shellwords.shellescape(v)
96     }.join(' ')
97 end
98
99
100 $jscPath = nil
101 $doNotMessWithVMPath = false
102 $jitTests = true
103 $memoryLimited = false
104 $outputDir = Pathname.new("results")
105 $verbosity = 0
106 $bundle = nil
107 $tarball = false
108 $tarFileName = "payload.tar.gz"
109 $copyVM = false
110 $testRunnerType = nil
111 $testWriter = "default"
112 $remoteUser = nil
113 $remoteHost = nil
114 $remotePort = nil
115 $remoteDirectory = nil
116 $architecture = nil
117 $hostOS = nil
118 $filter = nil
119 $envVars = []
120 $mode = "full"
121 $buildType = "release"
122 $forceCollectContinuously = false
123
124 def usage
125     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
126     puts
127     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
128     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
129     puts "                            --jsc specifies an already present JavaScriptCore to test."
130     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
131     puts "                            Skip tests tagged with //@skip if $memoryLimited"
132     puts "--no-jit                    Do not run JIT specific tests."
133     puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
134     puts "                            platform."
135     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
136     puts "--verbose            (-v)   Print more things while running."
137     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
138     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
139     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
140     puts "                            e.g. x86, x86_64, arm."
141     puts "--os                        Specify os instead of determining from JavaScriptCore build."
142     puts "                            e.g. darwin, linux & windows."
143     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
144     puts "                            In general the shell runner is slower than the make runner."
145     puts "--make-runner               Uses the faster make-based runner."
146     puts "--ruby-runner               Uses the ruby runner for machines without unix shell or make."
147     puts "--test-writer [writer]      Specifies the test script format."
148     puts "                            default is to use shell scripts to run the tests"
149     puts "                            \"ruby\" to use ruby scripts for systems without a unix shell."
150     puts "--remote                    Specify a remote host on which to run tests from command line argument."
151     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
152     puts "--child-processes    (-c)   Specify the number of child processes."
153     puts "--filter                    Only run tests whose name matches the given regular expression."
154     puts "--help               (-h)   Print this message."
155     puts "--env-vars                  Add a list of environment variables to set before running jsc."
156     puts "                            Each environment variable should be separated by a space."
157     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
158     puts "                            it will override the default value."
159     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
160     puts "--basic                     Run with default and these additional modes: no-llint,"
161     puts "                            no-cjit-validate-phases, no-cjit-collect-continuously, dfg-eager"
162     puts "                            and for FTL platforms: no-ftl, ftl-eager-no-cjit and"
163     puts "                            ftl-no-cjit-small-pool."
164     exit 1
165 end
166
167 jscArg = nil
168
169 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
170                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
171                ['--no-copy', GetoptLong::NO_ARGUMENT],
172                ['--memory-limited', GetoptLong::NO_ARGUMENT],
173                ['--no-jit', GetoptLong::NO_ARGUMENT],
174                ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
175                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
176                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
177                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
178                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
179                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
180                ['--os', GetoptLong::REQUIRED_ARGUMENT],
181                ['--shell-runner', GetoptLong::NO_ARGUMENT],
182                ['--make-runner', GetoptLong::NO_ARGUMENT],
183                ['--ruby-runner', GetoptLong::NO_ARGUMENT],
184                ['--test-writer', GetoptLong::REQUIRED_ARGUMENT],
185                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
186                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
187                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
188                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
189                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
190                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
191                ['--debug', GetoptLong::NO_ARGUMENT],
192                ['--release', GetoptLong::NO_ARGUMENT],
193                ['--quick', '-q', GetoptLong::NO_ARGUMENT],
194                ['--basic', GetoptLong::NO_ARGUMENT]).each {
195     | opt, arg |
196     case opt
197     when '--help'
198         usage
199     when '--jsc'
200         jscArg = arg
201     when '--no-copy'
202         $doNotMessWithVMPath = true
203     when '--output-dir'
204         $outputDir = Pathname.new(arg)
205     when '--memory-limited'
206         $memoryLimited = true
207     when '--no-jit'
208         $jitTests = false
209     when '--force-collectContinuously'
210         $forceCollectContinuously = true;
211     when '--verbose'
212         $verbosity += 1
213     when '--run-bundle'
214         $bundle = Pathname.new(arg)
215     when '--tarball'
216         $tarball = true
217         $copyVM = true
218         $tarFileName = arg unless arg == ''
219     when '--force-vm-copy'
220         $copyVM = true
221     when '--shell-runner'
222         $testRunnerType = :shell
223     when '--make-runner'
224         $testRunnerType = :make
225     when '--ruby-runner'
226         $testRunnerType = :ruby
227     when '--test-writer'
228         $testWriter = arg
229     when '--remote'
230         $copyVM = true
231         $tarball = true
232         $remote = true
233         uri = URI("ssh://" + arg)
234         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
235     when '--remote-config-file'
236         $remoteConfigFile = arg
237     when '--child-processes'
238         $numChildProcesses = arg.to_i
239     when '--filter'
240         $filter = Regexp.new(arg)
241     when '--arch'
242         $architecture = arg
243     when '--os'
244         $hostOS = arg
245     when '--env-vars'
246         $envVars = arg.gsub(/\s+/, ' ').split(' ')
247     when '--quick'
248         $mode = "quick"
249     when '--basic'
250         $mode = "basic"
251     when '--debug'
252         $buildType = "debug"
253     when '--release'
254         $buildType = "release"
255     end
256 }
257
258 if $remoteConfigFile
259     file = File.read($remoteConfigFile)
260     config = JSON.parse(file)
261
262     if !$remote and config['remote']
263         $copyVM = true
264         $tarball = true
265         $remote = true
266         uri = URI("ssh://" + config['remote'])
267         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
268     end
269
270     if config['remoteDirectory']
271         $remoteDirectory = config['remoteDirectory']
272     end
273 end
274
275 unless jscArg
276     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
277     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
278     command += ($buildType == "release") ? " --release" : " --debug"
279     command += " --executablePath"
280
281     output = `#{command}`.split("\n")
282     if !output.length
283         $stderr.puts "Error: must specify --jsc <path>"
284         exit 1
285     end
286
287     output = output[0]
288     jscArg = Pathname.new(output).join("jsc")
289     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
290     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
291     if !File.file?(jscArg)
292         $stderr.puts "Error: must specify --jsc <path>"
293         exit 1
294     end
295
296     puts "Using the following jsc path: #{jscArg}"
297 end
298
299 if $doNotMessWithVMPath
300     $jscPath = Pathname.new(jscArg)
301 else
302     $jscPath = Pathname.new(jscArg).realpath
303 end
304
305 $progressMeter = ($verbosity == 0 and $stdout.tty?)
306
307 if $bundle
308     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
309     $outputDir = $bundle
310 end
311
312 # Try to determine architecture. Return nil on failure.
313 def machOArchitectureCode
314     begin 
315         otoolLines = `otool -afh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
316         otoolLines.each_with_index {
317             | value, index |
318             if value =~ /magic/ and value =~ /cputype/
319                 return otoolLines[index + 1].split[1].to_i
320             end
321         }
322     rescue
323         $stderr.puts "Warning: unable to execute otool."
324     end
325     $stderr.puts "Warning: unable to determine architecture."
326     nil
327 end
328
329 def determineArchitectureFromMachOBinary
330     code = machOArchitectureCode
331     return nil unless code
332     is64BitFlag = 0x01000000
333     case code
334     when 7
335         "x86"
336     when 7 | is64BitFlag
337         "x86-64"
338     when 12
339         "arm"
340     when 12 | is64BitFlag
341         "arm64"
342     else
343         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
344         nil
345     end
346 end
347
348 def determineArchitectureFromELFBinary
349     f = File.open($jscPath.to_s)
350     data = f.read(19)
351
352     if !(data[0,4] == "\x7F\x45\x4C\x46")
353         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
354         return nil
355     end
356
357     code = data[18].ord
358     case code
359     when 3
360         "x86"
361     when 8
362         "mips"
363     when 62
364         "x86-64"
365     when 40
366         "arm"
367     when 183
368         "arm64"
369     else
370         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
371         nil
372     end
373 end
374
375 def determineArchitectureFromPEBinary
376     f = File.open($jscPath.to_s)
377     data = f.read(1024)
378
379     if !(data[0, 2] == "MZ")
380         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
381         return nil
382     end
383
384     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
385
386     if !(data[peHeaderAddr, 4] == "PE\0\0")
387         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
388         return nil
389     end
390
391     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
392
393     case machine
394     when 0x014c
395         "x86"
396     when 0x8664
397         "x86-64"
398     else
399         $stderr.puts "Warning: unsupported machine type: #{machine}"
400         nil
401     end
402 end
403
404 def determineArchitecture
405     case $hostOS
406     when "darwin"
407         determineArchitectureFromMachOBinary
408     when "linux"
409         determineArchitectureFromELFBinary
410     when "windows"
411         determineArchitectureFromPEBinary
412     else
413         $stderr.puts "Warning: unable to determine architecture on this platform."
414         nil
415     end
416 end
417
418 def determineOS
419     case RbConfig::CONFIG["host_os"]
420     when /darwin/i
421         "darwin"
422     when /linux/i
423         "linux"
424     when /mswin|mingw|cygwin/
425         "windows"
426     else
427         $stderr.puts "Warning: unable to determine host operating system"
428         nil
429     end
430 end
431
432 $hostOS = determineOS unless $hostOS
433 $architecture = determineArchitecture unless $architecture
434 $isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $architecture == "mips" || $hostOS == "windows")
435
436 def isFTLEnabled
437     $jitTests && $isFTLPlatform
438 end
439
440 if !$testRunnerType
441     if $remote and $hostOS == "darwin"
442         $testRunnerType = :shell
443     else
444         $testRunnerType = :make
445     end
446 end
447
448 if $testWriter
449     if /[^-a-zA-Z0-9_]/.match($testWriter)
450         raise "Invalid test writer #{$testWriter} given"
451     end
452 end
453
454 $numFailures = 0
455 $numPasses = 0
456
457 # We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
458 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--validateExceptionChecks=true", "--useDollarVM=true", "--maxPerThreadStackUsage=1572864"]
459 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
460 # NOTE: Tests rely on this using scribbleFreeCells.
461 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
462 B3O1_OPTIONS = ["--defaultB3OptLevel=1"]
463 FTL_OPTIONS = ["--useFTLJIT=true"]
464 PROBE_OSR_EXIT_OPTION = ["--useProbeOSRExit=true"]
465
466 require_relative "webkitruby/jsc-stress-test-writer-#{$testWriter}"
467
468 def shouldCollectContinuously?
469     $buildType == "release" or $forceCollectContinuously
470 end
471
472 COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false"] : []
473
474 $runlist = []
475
476 def frameworkFromJSCPath(jscPath)
477     parentDirectory = jscPath.dirname
478     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
479         parentDirectory.dirname
480     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
481         jscPath.dirname + "JavaScriptCore.framework"
482     else
483         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
484         nil
485     end
486 end
487
488 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
489     dir = Pathname.new(".")
490     $benchmarkDirectory.each_filename {
491         | pathComponent |
492         dir += ".."
493     }
494     dir + resourcePath
495 end
496
497 def pathToVM
498     pathToBundleResourceFromBenchmarkDirectory($jscPath)
499 end
500
501 def pathToHelpers
502     pathToBundleResourceFromBenchmarkDirectory(".helpers")
503 end
504
505 $runCommandOptions = {}
506
507 $uniqueFilenameCounter = 0
508 def uniqueFilename(extension)
509     payloadDir = $outputDir + "_payload"
510     Dir.mkdir payloadDir unless payloadDir.directory?
511     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
512     $uniqueFilenameCounter += 1
513     result
514 end
515
516 def baseOutputName(kind)
517     "#{$collectionName}/#{$benchmark}.#{kind}"
518 end
519
520 def addRunCommand(kind, command, outputHandler, errorHandler, *additionalEnv)
521     $didAddRunCommand = true
522     name = baseOutputName(kind)
523     if $filter and name !~ $filter
524         return
525     end
526     plan = Plan.new(
527         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
528         errorHandler)
529     plan.additionalEnv.push(*additionalEnv)
530     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
531         $runlist.unshift plan
532     else
533         $runlist << plan
534     end
535 end
536
537 # Returns true if there were run commands found in the file ($benchmarkDirectory +
538 # $benchmark), in which case those run commands have already been executed. Otherwise
539 # returns false, in which case you're supposed to add your own run commands.
540 def parseRunCommands
541     oldDidAddRunCommand = $didAddRunCommand
542     $didAddRunCommand = false
543     $skipped = false
544
545     Dir.chdir($outputDir) {
546         File.open($benchmarkDirectory + $benchmark) {
547             | inp |
548             inp.each_line {
549                 | line |
550                 begin
551                     doesMatch = line =~ /^\/\/@/
552                 rescue Exception => e
553                     # Apparently this happens in the case of some UTF8 stuff in some files, where
554                     # Ruby tries to be strict and throw exceptions.
555                     next
556                 end
557                 next unless doesMatch
558                 eval $~.post_match
559                 if $skipped
560                     break
561                 end
562             }
563         }
564     }
565
566     result = $didAddRunCommand
567     $didAddRunCommand = result or oldDidAddRunCommand
568     result
569 end
570
571 def slow!
572     $runCommandOptions[:isSlow] = true
573 end
574
575 def runWithOutputHandler(kind, outputHandler, *options)
576     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
577 end
578
579 def run(kind, *options)
580     runWithOutputHandler(kind, silentOutputHandler, *options)
581 end
582
583 def runNoFTL(*optionalTestSpecificOptions)
584     run("no-ftl", *optionalTestSpecificOptions)
585 end
586
587 def runWithRAMSize(size, *optionalTestSpecificOptions)
588     run("ram-size-#{size}", "--forceRAMSize=#{size}", *optionalTestSpecificOptions)
589 end
590
591 def runOneLargeHeap(*optionalTestSpecificOptions)
592     if $memoryLimited
593         $didAddRunCommand = true
594         puts "Skipping #{$collectionName}/#{$benchmark}"
595     else
596         run("default", *optionalTestSpecificOptions)
597     end
598 end
599
600 def runNoJIT(*optionalTestSpecificOptions)
601     run("no-jit", "--useJIT=false", *optionalTestSpecificOptions)
602 end
603
604 def runNoLLInt(*optionalTestSpecificOptions)
605     if $jitTests
606         run("no-llint", "--useLLInt=false", *optionalTestSpecificOptions)
607     end
608 end
609
610 # NOTE: Tests rely on this using scribbleFreeCells.
611 def runNoCJITValidate(*optionalTestSpecificOptions)
612     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
613 end
614
615 def runNoCJITValidatePhases(*optionalTestSpecificOptions)
616     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
617 end
618
619 def runNoCJITCollectContinuously(*optionalTestSpecificOptions)
620     run("no-cjit-collect-continuously", *(NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
621 end
622
623 def runDefault(*optionalTestSpecificOptions)
624     run("default", *(FTL_OPTIONS + optionalTestSpecificOptions))
625 end
626
627 def runBigIntEnabled(*optionalTestSpecificOptions)
628     run("big-int-enabled", "--useBigInt=true" , *(FTL_OPTIONS + optionalTestSpecificOptions))
629 end
630
631 def runFTLNoCJIT(*optionalTestSpecificOptions)
632     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
633 end
634
635 def runFTLNoCJITB3O1(*optionalTestSpecificOptions)
636     run("ftl-no-cjit-b3o1", "--useArrayAllocationProfiling=false", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
637 end
638
639 def runFTLNoCJITValidate(*optionalTestSpecificOptions)
640     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + PROBE_OSR_EXIT_OPTION + optionalTestSpecificOptions))
641 end
642
643 def runFTLNoCJITNoPutStackValidate(*optionalTestSpecificOptions)
644     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
645 end
646
647 def runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
648     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
649 end
650
651 def runFTLNoCJITOSRValidation(*optionalTestSpecificOptions)
652     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
653 end
654
655 def runDFGEager(*optionalTestSpecificOptions)
656     run("dfg-eager", *(EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + PROBE_OSR_EXIT_OPTION + optionalTestSpecificOptions))
657 end
658
659 def runDFGEagerNoCJITValidate(*optionalTestSpecificOptions)
660     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
661 end
662
663 def runFTLEager(*optionalTestSpecificOptions)
664     run("ftl-eager", "--airForceBriggsAllocator=true", "--forcePolyProto=true", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
665 end
666
667 def runFTLEagerWatchdog(*optionalTestSpecificOptions)
668     timeout = rand(100)
669     run("ftl-eager-watchdog-#{timeout}", "--watchdog=#{timeout}", "--watchdog-exception-ok", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
670 end
671
672 def runFTLEagerNoCJITValidate(*optionalTestSpecificOptions)
673     run("ftl-eager-no-cjit", "--validateGraph=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
674 end
675
676 def runFTLEagerNoCJITB3O1(*optionalTestSpecificOptions)
677     run("ftl-eager-no-cjit-b3o1", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
678 end
679
680 def runFTLEagerNoCJITOSRValidation(*optionalTestSpecificOptions)
681     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
682 end
683
684 def runNoCJITNoASO(*optionalTestSpecificOptions)
685     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
686 end
687
688 def runNoCJITNoAccessInlining(*optionalTestSpecificOptions)
689     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
690 end
691
692 def runFTLNoCJITNoAccessInlining(*optionalTestSpecificOptions)
693     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
694 end
695
696 def runFTLNoCJITSmallPool(*optionalTestSpecificOptions)
697     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
698 end
699
700 def runNoCJIT(*optionalTestSpecificOptions)
701     run("no-cjit", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
702 end
703
704 def runDFGMaximalFlushPhase(*optionalTestSpecificOptions)
705     run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
706 end
707
708 def runShadowChicken(*optionalTestSpecificOptions)
709     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true", *optionalTestSpecificOptions)
710 end
711
712 def defaultRun
713     if $mode == "quick"
714         defaultQuickRun
715     else
716         runDefault
717         if $jitTests
718             runNoLLInt
719             runNoCJITValidatePhases
720             runNoCJITCollectContinuously if shouldCollectContinuously?
721             runDFGEager
722             if $mode != "basic"
723                 runDFGEagerNoCJITValidate
724                 runDFGMaximalFlushPhase
725             end
726
727             return if !$isFTLPlatform
728
729             runNoFTL
730             runFTLEager
731             runFTLEagerNoCJITValidate
732             runFTLNoCJITSmallPool
733
734             return if $mode == "basic"
735
736             runFTLNoCJITValidate
737             runFTLNoCJITB3O1
738             runFTLNoCJITNoPutStackValidate
739             runFTLNoCJITNoInlineValidate
740             runFTLEagerNoCJITB3O1
741         end
742     end
743 end
744
745 def defaultNoNoLLIntRun
746     if $mode == "quick"
747         defaultQuickRun
748     else
749         runDefault
750         if $jitTests
751             runNoCJITValidatePhases
752             runNoCJITCollectContinuously if shouldCollectContinuously?
753             runDFGEager
754             if $mode != "basic"
755                 runDFGEagerNoCJITValidate
756                 runDFGMaximalFlushPhase
757             end
758
759             return if !$isFTLPlatform
760
761             runNoFTL
762             runFTLNoCJITValidate
763             runFTLNoCJITSmallPool
764
765             return if $mode == "basic"
766
767             runFTLNoCJITB3O1
768             runFTLNoCJITNoPutStackValidate
769             runFTLNoCJITNoInlineValidate
770             runFTLEager
771             runFTLEagerNoCJITValidate
772         end
773     end
774 end
775
776 def defaultQuickRun
777     runDefault
778     if $jitTests
779         runNoCJITValidate
780
781         return if !$isFTLPlatform
782
783         runNoFTL
784         runFTLNoCJITValidate
785     end
786 end
787
788 def defaultSpotCheckNoMaximalFlush
789     defaultQuickRun
790     runNoCJITNoAccessInlining
791
792     return if !$isFTLPlatform
793
794     runFTLNoCJITOSRValidation
795     runFTLNoCJITNoAccessInlining
796     runFTLNoCJITB3O1
797 end
798
799 def defaultSpotCheck
800     defaultSpotCheckNoMaximalFlush
801     runDFGMaximalFlushPhase
802 end
803
804 # This is expected to not do eager runs because eager runs can have a lot of recompilations
805 # for reasons that don't arise in the real world. It's used for tests that assert convergence
806 # by counting recompilations.
807 def defaultNoEagerRun
808     runDefault
809     if $jitTests
810         runNoLLInt
811         runNoCJITValidatePhases
812         runNoCJITCollectContinuously if shouldCollectContinuously?
813
814         return if !$isFTLPlatform
815
816         runNoFTL
817         runFTLNoCJITValidate
818
819         return if $mode == "basic"
820
821         runFTLNoCJITNoInlineValidate
822         runFTLNoCJITB3O1
823     end
824 end
825
826 def defaultNoSamplingProfilerRun
827     runDefault
828     if $jitTests
829         runNoLLInt
830         runNoCJITValidatePhases
831         runNoCJITCollectContinuously if shouldCollectContinuously?
832         runDFGEager
833         runDFGEagerNoCJITValidate
834         runDFGMaximalFlushPhase
835
836         return if !$isFTLPlatform
837
838         runNoFTL
839         runFTLNoCJITNoPutStackValidate
840         runFTLNoCJITNoInlineValidate
841         runFTLEager
842         runFTLEagerNoCJITValidate
843         runFTLNoCJITSmallPool
844     end
845 end
846
847 def runProfiler
848     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
849         skip
850         return
851     end
852
853     profilerOutput = uniqueFilename(".json")
854     if $canRunDisplayProfilerOutput
855         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
856     else
857         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
858         run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
859     end
860 end
861
862 def runExceptionFuzz
863     subCommand = escapeAll([pathToVM.to_s, "--useDollarVM=true", $benchmark.to_s])
864     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
865 end
866
867 def runExecutableAllocationFuzz(name, *options)
868     subCommand = escapeAll([pathToVM.to_s, "--useDollarVM=true", $benchmark.to_s] + options)
869     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
870 end
871
872 def runTypeProfiler
873     if !$jitTests
874         return
875     end
876
877     run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
878     run("ftl-no-cjit-type-profiler-force-poly-proto", "--useTypeProfiler=true", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
879
880     return if !$isFTLPlatform
881
882     run("ftl-type-profiler-ftl-eager", "--useTypeProfiler=true", *(FTL_OPTIONS + EAGER_OPTIONS))
883 end
884
885 def runControlFlowProfiler
886     if !$jitTests
887         return
888     end
889
890     return if !$isFTLPlatform
891
892     run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
893 end
894
895 def runTest262(mode, exception, includeFiles, flags)
896     failsWithException = exception != "NoException"
897     isStrict = false
898     isModule = false
899     isAsync = false
900
901     flags.each {
902         | flag |
903         case flag
904         when :strict
905             isStrict = true
906         when :module
907             isModule = true
908         when :async
909             isAsync = true
910         else
911             raise "Invalid flag for runTest262, #{flag}"
912         end
913     }
914
915     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
916
917     args = [pathToVM.to_s] + BASE_OPTIONS
918     args << "--exception=" + exception if failsWithException
919     args << "--test262-async" if isAsync
920     args += includeFiles
921
922     case mode
923     when :normal
924         errorHandler = simpleErrorHandler
925         outputHandler = silentOutputHandler
926     when :fail
927         errorHandler = expectedFailErrorHandler
928         outputHandler = noisyOutputHandler
929     when :failDueToOutdatedOrBadTest
930         errorHandler = expectedFailErrorHandler
931         outputHandler = noisyOutputHandler
932     when :skip
933         return
934     else
935         raise "Invalid mode: #{mode}"
936     end
937
938     if isStrict
939         kind = "default-strict"
940         args << "--strict-file=#{$benchmark}"
941     else
942         kind = "default"
943         if isModule
944             args << "--module-file=#{$benchmark}"
945         else
946             args << $benchmark.to_s
947         end
948     end
949
950     addRunCommand(kind, args, outputHandler, errorHandler)
951 end
952
953 def prepareTest262Fixture
954     # This function is used to add the files used by Test262 modules tests.
955     prepareExtraRelativeFiles([""], $collection)
956 end
957
958 def runES6(mode)
959     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
960     case mode
961     when :normal
962         errorHandler = simpleErrorHandler
963     when :fail
964         errorHandler = expectedFailErrorHandler
965     when :failDueToOutdatedOrBadTest
966         errorHandler = expectedFailErrorHandler
967     when :skip
968         return
969     else
970         raise "Invalid mode: #{mode}"
971     end
972     addRunCommand("default", args, noisyOutputHandler, errorHandler)
973 end
974
975 def runModules
976     run("default-modules", "-m")
977
978     if !$jitTests
979         return
980     end
981
982     run("no-llint-modules", "-m", "--useLLInt=false")
983     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
984     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
985     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
986
987     return if !$isFTLPlatform
988
989     run("default-ftl-modules", "-m", *FTL_OPTIONS)
990     run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
991     run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
992     run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
993     run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
994     run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
995 end
996
997 def runWebAssembly
998     return if !$jitTests
999     return if !$isFTLPlatform
1000     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1001     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1002     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1003     run("default-wasm", "-m", *FTL_OPTIONS)
1004     if $mode != "quick"
1005         run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1006         run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1007         run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1008         run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1009         run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
1010     end
1011 end
1012
1013 def runWebAssemblyEmscripten(mode)
1014     case mode
1015     when :skip
1016         return
1017     end
1018     return if !$jitTests
1019     return if !$isFTLPlatform
1020     wasm = $benchmark.to_s.sub! '.js', '.wasm'
1021     prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
1022     run("default-wasm", *FTL_OPTIONS)
1023     if $mode != "quick"
1024         run("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1025         run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1026         run("wasm-no-call-ic", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1027         run("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1028     end
1029 end
1030
1031 def runWebAssemblySpecTest(mode)
1032     case mode
1033     when :skip
1034         return
1035     end
1036     return if !$jitTests
1037     return if !$isFTLPlatform
1038     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1039
1040     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1041     prepareExtraRelativeFiles(modules.map { |f| "../../" + f }, $collection)
1042
1043     harness = Dir[WASMTESTS_PATH + "spec-harness/" + "*.js"].map { |f| File.basename(f) }
1044     prepareExtraRelativeFiles(harness.map { |f| "../../spec-harness/" + f }, $collection)
1045
1046     runWithOutputHandler("default-wasm", noisyOutputHandler, "../spec-harness.js", *FTL_OPTIONS)
1047     if $mode != "quick"
1048       runWithOutputHandler("wasm-no-cjit-yes-tls-context", noisyOutputHandler, "../spec-harness.js",  "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1049       runWithOutputHandler("wasm-eager-jettison", noisyOutputHandler, "../spec-harness.js", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1050       runWithOutputHandler("wasm-no-call-ic", noisyOutputHandler, "../spec-harness.js", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1051       runWithOutputHandler("wasm-no-tls-context", noisyOutputHandler, "../spec-harness.js", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1052     end
1053 end
1054
1055 def runWebAssemblyLowExecutableMemory(*optionalTestSpecificOptions)
1056     return if !$jitTests
1057     return if !$isFTLPlatform
1058     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1059     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1060     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1061     # Only let WebAssembly get executable memory.
1062     run("default-wasm", "--useConcurrentGC=0" , "--useConcurrentJIT=0", "--jitMemoryReservationSize=15000", "--useBaselineJIT=0", "--useDFGJIT=0", "--useFTLJIT=0", "-m")
1063 end
1064
1065 def runChakra(mode, exception, baselineFile, extraFiles)
1066     raise unless $benchmark.to_s =~ /\.js$/
1067     failsWithException = exception != "NoException"
1068     testName = $~.pre_match
1069
1070     prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
1071     prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)
1072
1073     args = [pathToVM.to_s] + BASE_OPTIONS
1074     args += FTL_OPTIONS if $isFTLPlatform
1075     args += EAGER_OPTIONS
1076     args << "--exception=" + exception if failsWithException
1077     args << "--dumpException" if failsWithException
1078     args += ["jsc-lib.js"]
1079
1080     case mode
1081     when :baseline
1082         prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
1083         errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
1084         outputHandler = noisyOutputHandler
1085     when :pass
1086         errorHandler = chakraPassFailErrorHandler
1087         outputHandler = noisyOutputHandler
1088     when :skipDueToOutdatedOrBadTest
1089         return
1090     when :skip
1091         return
1092     else
1093         raise "Invalid mode: #{mode}"
1094     end
1095
1096     kind = "default"
1097     args << $benchmark.to_s
1098
1099     addRunCommand(kind, args, outputHandler, errorHandler)
1100 end
1101
1102 def runLayoutTest(kind, *options)
1103     raise unless $benchmark.to_s =~ /\.js$/
1104     testName = $~.pre_match
1105     if kind
1106         kind = "layout-" + kind
1107     else
1108         kind = "layout"
1109     end
1110
1111     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1112     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1113
1114     args = [pathToVM.to_s] + BASE_OPTIONS + options +
1115         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1116          $benchmark.to_s,
1117          (Pathname.new("resources") + "standalone-post.js").to_s]
1118     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1119 end
1120
1121 def runLayoutTestNoFTL
1122     runLayoutTest("no-ftl")
1123 end
1124
1125 def runLayoutTestNoLLInt
1126     runLayoutTest("no-llint", "--useLLInt=false")
1127 end
1128
1129 def runLayoutTestNoCJIT
1130     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1131 end
1132
1133 def runLayoutTestDFGEagerNoCJIT
1134     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1135 end
1136
1137 def runLayoutTestDefault
1138     runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
1139 end
1140
1141 def runLayoutTestFTLNoCJIT
1142     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1143 end
1144
1145 def runLayoutTestFTLEagerNoCJIT
1146     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1147 end
1148
1149 def runLayoutTestFTLEagerNoCJITB3O1
1150     runLayoutTest("ftl-eager-no-cjit-b3o1", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS))
1151 end
1152
1153 def noFTLRunLayoutTest
1154     if !$jitTests
1155         return
1156     end
1157
1158     runLayoutTestNoLLInt
1159     runLayoutTestNoCJIT
1160     runLayoutTestDFGEagerNoCJIT
1161 end
1162
1163 def defaultQuickRunLayoutTest
1164     runLayoutTestDefault
1165     if $jitTests
1166         if $isFTLPlatform
1167             runLayoutTestNoFTL
1168             runLayoutTestFTLNoCJIT
1169             runLayoutTestFTLEagerNoCJIT
1170         else
1171             noFTLRunLayoutTest
1172         end
1173     end
1174 end
1175
1176 def defaultRunLayoutTest
1177     if $mode == "quick"
1178         defaultQuickRunLayoutTest
1179     else
1180         runLayoutTestDefault
1181         if $jitTests
1182             noFTLRunLayoutTest
1183
1184             return if !$isFTLPlatform
1185
1186             runLayoutTestNoFTL
1187             runLayoutTestFTLNoCJIT
1188             runLayoutTestFTLEagerNoCJIT
1189         end
1190     end
1191 end
1192
1193 def noEagerNoNoLLIntTestsRunLayoutTest
1194     runLayoutTestDefault
1195     if $jitTests
1196         runLayoutTestNoCJIT
1197
1198         return if !$isFTLPlatform
1199
1200         runLayoutTestNoFTL
1201         runLayoutTestFTLNoCJIT
1202     end
1203 end
1204
1205 def noNoLLIntRunLayoutTest
1206     runLayoutTestDefault
1207     if $jitTests
1208         runLayoutTestNoCJIT
1209         runLayoutTestDFGEagerNoCJIT
1210
1211         return if !$isFTLPlatform
1212
1213         runLayoutTestNoFTL
1214         runLayoutTestFTLNoCJIT
1215         runLayoutTestFTLEagerNoCJIT
1216     end
1217 end
1218
1219 def prepareExtraRelativeFiles(extraFiles, destination)
1220     Dir.chdir($outputDir) {
1221         extraFiles.each {
1222             | file |
1223             dest = destination + file
1224             FileUtils.mkdir_p(dest.dirname)
1225             FileUtils.cp $extraFilesBaseDir + file, dest
1226         }
1227     }
1228 end
1229
1230 def baseDirForCollection(collectionName)
1231     Pathname(".tests") + collectionName
1232 end
1233
1234 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1235     raise unless absoluteBase.absolute?
1236     Dir.chdir($outputDir) {
1237         collectionBaseDir = baseDirForCollection($collectionName)
1238         extraFiles.each {
1239             | file |
1240             destination = collectionBaseDir + file
1241             FileUtils.mkdir_p destination.dirname unless destination.directory?
1242             FileUtils.cp absoluteBase + file, destination
1243         }
1244     }
1245 end
1246
1247 def runMozillaTest(kind, mode, extraFiles, *options)
1248     if kind
1249         kind = "mozilla-" + kind
1250     else
1251         kind = "mozilla"
1252     end
1253     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1254     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1255     case mode
1256     when :normal
1257         errorHandler = mozillaErrorHandler
1258     when :negative
1259         errorHandler = mozillaExit3ErrorHandler
1260     when :fail
1261         errorHandler = mozillaFailErrorHandler
1262     when :failDueToOutdatedOrBadTest
1263         errorHandler = mozillaFailErrorHandler
1264     when :skip
1265         return
1266     else
1267         raise "Invalid mode: #{mode}"
1268     end
1269     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1270 end
1271
1272 def runMozillaTestDefault(mode, *extraFiles)
1273     runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
1274 end
1275
1276 def runMozillaTestNoFTL(mode, *extraFiles)
1277     runMozillaTest("no-ftl", mode, extraFiles)
1278 end
1279
1280 def runMozillaTestLLInt(mode, *extraFiles)
1281     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1282 end
1283
1284 def runMozillaTestBaselineJIT(mode, *extraFiles)
1285     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1286 end
1287
1288 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1289     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1290 end
1291
1292 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1293     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1294 end
1295
1296 def defaultQuickRunMozillaTest(mode, *extraFiles)
1297     if $jitTests
1298         runMozillaTestDefault(mode, *extraFiles)
1299         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1300     else
1301         runMozillaTestNoFTL(mode, *extraFiles)
1302         if $jitTests
1303             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1304         end
1305     end
1306 end
1307
1308 def defaultRunMozillaTest(mode, *extraFiles)
1309     if $mode == "quick"
1310         defaultQuickRunMozillaTest(mode, *extraFiles)
1311     else
1312         runMozillaTestNoFTL(mode, *extraFiles)
1313         if $jitTests
1314             runMozillaTestLLInt(mode, *extraFiles)
1315             runMozillaTestBaselineJIT(mode, *extraFiles)
1316             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1317             runMozillaTestDefault(mode, *extraFiles)
1318             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
1319         end
1320     end
1321 end
1322
1323 def runNoisyTestImpl(kind, options, additionalEnv)
1324     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler, *additionalEnv)
1325 end
1326
1327 def runNoisyTest(kind, *options)
1328     runNoisyTestImpl(kind, options, [])
1329 end
1330
1331 def runNoisyTestWithEnv(kind, *additionalEnv)
1332     runNoisyTestImpl(kind, [], additionalEnv)
1333 end
1334
1335 def runNoisyTestDefault
1336     runNoisyTest("default", *FTL_OPTIONS)
1337 end
1338
1339 def runNoisyTestNoFTL
1340     runNoisyTest("no-ftl")
1341 end
1342
1343 def runNoisyTestNoCJIT
1344     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1345 end
1346
1347 def runNoisyTestNoCJITB3O1
1348     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS))
1349 end
1350
1351 def runNoisyTestEagerNoCJIT
1352     runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1353 end
1354
1355 def defaultRunNoisyTest
1356     runNoisyTestDefault
1357     if $jitTests and $isFTLPlatform
1358         runNoisyTestNoFTL
1359         runNoisyTestNoCJIT
1360         runNoisyTestNoCJITB3O1
1361         runNoisyTestEagerNoCJIT
1362     end
1363 end
1364
1365 def skip
1366     $didAddRunCommand = true
1367     $skipped = true
1368     puts "Skipping #{$collectionName}/#{$benchmark}"
1369 end
1370
1371 def allJSFiles(path)
1372     if path.file?
1373         [path]
1374     else
1375         result = []
1376         Dir.foreach(path) {
1377             | filename |
1378             next unless filename =~ /\.js$/
1379             next unless (path + filename).file?
1380             result << path + filename
1381         }
1382         result
1383     end
1384 end
1385
1386 def uniqueifyName(names, name)
1387     result = name.to_s
1388     toAdd = 1
1389     while names[result]
1390         result = "#{name}-#{toAdd}"
1391         toAdd += 1
1392     end
1393     names[result] = true
1394     result
1395 end
1396
1397 def simplifyCollectionName(collectionPath)
1398     outerDir = collectionPath.dirname
1399     name = collectionPath.basename
1400     lastName = name
1401     if collectionPath.directory?
1402         while lastName.to_s =~ /test/
1403             lastName = outerDir.basename
1404             name = lastName + name
1405             outerDir = outerDir.dirname
1406         end
1407     end
1408     uniqueifyName($collectionNames, name)
1409 end
1410
1411 def prepareCollection(name)
1412     FileUtils.mkdir_p $outputDir + name
1413
1414     absoluteCollection = $collection.realpath
1415
1416     Dir.chdir($outputDir) {
1417         bundleDir = baseDirForCollection(name)
1418
1419         # Create the proper directory structures.
1420         FileUtils.mkdir_p bundleDir
1421         if bundleDir.basename == $collection.basename
1422             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1423             $collection = bundleDir
1424         else
1425             FileUtils.cp_r absoluteCollection, bundleDir
1426             $collection = bundleDir + $collection.basename
1427         end
1428
1429         $extraFilesBaseDir = absoluteCollection
1430     }
1431 end
1432
1433 $collectionNames = {}
1434
1435 def handleCollectionFile(collection)
1436     collectionName = simplifyCollectionName(collection)
1437    
1438     paths = {}
1439     subCollections = []
1440     YAML::load(IO::read(collection)).each {
1441         | entry |
1442         if entry["collection"]
1443             subCollections << entry["collection"]
1444             next
1445         end
1446         
1447         if Pathname.new(entry["path"]).absolute?
1448             raise "Absolute path: " + entry["path"] + " in #{collection}"
1449         end
1450         
1451         if paths[entry["path"]]
1452             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1453         end
1454         
1455         subCollection = collection.dirname + entry["path"]
1456         
1457         if subCollection.file?
1458             subCollectionName = Pathname.new(entry["path"]).dirname
1459         else
1460             subCollectionName = entry["path"]
1461         end
1462         
1463         $collection = subCollection
1464         $collectionName = Pathname.new(collectionName)
1465         Pathname.new(subCollectionName).each_filename {
1466             | filename |
1467             next if filename =~ /^\./
1468             $collectionName += filename
1469         }
1470         $collectionName = $collectionName.to_s
1471         
1472         prepareCollection($collectionName)
1473       
1474         Dir.chdir($outputDir) {
1475             pathsToSearch = [$collection]
1476             if entry["tests"]
1477                 if entry["tests"].is_a? Array
1478                     pathsToSearch = entry["tests"].map {
1479                         | testName |
1480                         pathsToSearch[0] + testName
1481                     }
1482                 else
1483                     pathsToSearch[0] += entry["tests"]
1484                 end
1485             end
1486             pathsToSearch.each {
1487                 | pathToSearch |
1488                 allJSFiles(pathToSearch).each {
1489                     | path |
1490                     
1491                     $benchmark = path.basename
1492                     $benchmarkDirectory = path.dirname
1493                     
1494                     $runCommandOptions = {}
1495                     eval entry["cmd"]
1496                 }
1497             }
1498         }
1499     }
1500     
1501     subCollections.each {
1502         | subCollection |
1503         handleCollection(collection.dirname + subCollection)
1504     }
1505 end
1506
1507 def handleCollectionDirectory(collection)
1508     collectionName = simplifyCollectionName(collection)
1509     
1510     $collection = collection
1511     $collectionName = collectionName
1512     prepareCollection(collectionName)
1513    
1514     Dir.chdir($outputDir) {
1515         $benchmarkDirectory = $collection
1516         allJSFiles($collection).each {
1517             | path |
1518             
1519             $benchmark = path.basename
1520             
1521             $runCommandOptions = {}
1522             defaultRun unless parseRunCommands
1523         }
1524     }
1525 end
1526
1527 def handleCollection(collection)
1528     collection = Pathname.new(collection)
1529     
1530     if collection.file?
1531         handleCollectionFile(collection)
1532     else
1533         handleCollectionDirectory(collection)
1534     end
1535 end
1536
1537 def appendFailure(plan)
1538     File.open($outputDir + "failed", "a") {
1539         | outp |
1540         outp.puts plan.name
1541     }
1542     $numFailures += 1
1543 end
1544
1545 def appendPass(plan)
1546     File.open($outputDir + "passed", "a") {
1547         | outp |
1548         outp.puts plan.name
1549     }
1550     $numPasses += 1
1551 end
1552
1553 def appendResult(plan, didPass)
1554     File.open($outputDir + "results", "a") {
1555         | outp |
1556         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1557     }
1558 end
1559
1560 def prepareBundle
1561     raise if $bundle
1562
1563     if $doNotMessWithVMPath
1564         if !$remote and !$tarball
1565             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1566             $jscPath = Pathname.new($jscPath).realpath
1567         else
1568             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1569         end
1570     else
1571         originalJSCPath = $jscPath
1572         vmDir = $outputDir + ".vm"
1573         FileUtils.mkdir_p vmDir
1574         
1575         frameworkPath = frameworkFromJSCPath($jscPath)
1576         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1577         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1578         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1579
1580         if frameworkPath
1581             source = frameworkPath
1582             destination = Pathname.new(".vm")
1583         elsif $hostOS == "windows"
1584             # Make sure to copy dll along with jsc on Windows
1585             source = [originalJSCPath] + Dir.glob(File.dirname(originalJSCPath) + "/jscLib.dll")
1586             destination = $jscPath.dirname
1587
1588             Dir.chdir($outputDir) {
1589                 FileUtils.mkdir_p destination
1590             }
1591         else
1592             source = originalJSCPath
1593             destination = $jscPath
1594
1595             Dir.chdir($outputDir) {
1596                 FileUtils.mkdir_p $jscPath.dirname
1597             }
1598         end
1599
1600         Dir.chdir($outputDir) {
1601             if $copyVM
1602                 FileUtils.cp_r source, destination
1603             else
1604                 begin 
1605                     FileUtils.ln_s source, destination
1606                 rescue Exception
1607                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1608                     FileUtils.cp_r source, destination
1609                 end
1610             end
1611
1612             if $remote and $hostOS == "linux"
1613                 begin
1614                     dependencies = `ldd #{source}`
1615                     dependencies.split(/\n/).each {
1616                         | dependency |
1617                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1618                     }
1619                 rescue
1620                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1621                 end
1622             end
1623         }
1624     end
1625     
1626     Dir.chdir($outputDir) {
1627         FileUtils.cp_r HELPERS_PATH, ".helpers"
1628     }
1629
1630     ARGV.each {
1631         | collection |
1632         handleCollection(collection)
1633     }
1634
1635     puts
1636 end
1637
1638 def cleanOldResults
1639     raise unless $bundle
1640
1641     eachResultFile($outputDir) {
1642         | path |
1643         FileUtils.rm_f path
1644     }
1645 end
1646
1647 def cleanEmptyResultFiles
1648     eachResultFile($outputDir) {
1649         | path |
1650         next unless path.basename.to_s =~ /\.out$/
1651         next unless FileTest.size(path) == 0
1652         FileUtils.rm_f path
1653     }
1654 end
1655
1656 def eachResultFile(startingDir, &block)
1657     dirsToClean = [startingDir]
1658     until dirsToClean.empty?
1659         nextDir = dirsToClean.pop
1660         Dir.foreach(nextDir) {
1661             | entry |
1662             next if entry =~ /^\./
1663             path = nextDir + entry
1664             if path.directory?
1665                 dirsToClean.push(path)
1666             else
1667                 block.call(path)
1668             end
1669         }
1670     end
1671 end
1672
1673 def prepareTestRunner
1674     raise if $bundle
1675
1676     $runlist.each_with_index {
1677         | plan, index |
1678         plan.index = index
1679     }
1680
1681     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1682     toDelete = []
1683     Dir.foreach($runnerDir) {
1684         | filename |
1685         if filename =~ /^test_/
1686             toDelete << filename
1687         end
1688     }
1689     
1690     toDelete.each {
1691         | filename |
1692         File.unlink($runnerDir + filename)
1693     }
1694
1695     $runlist.each {
1696         | plan |
1697         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1698     }
1699
1700     case $testRunnerType
1701     when :make
1702         prepareMakeTestRunner
1703     when :shell
1704         prepareShellTestRunner
1705     when :ruby
1706         prepareRubyTestRunner
1707     else
1708         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1709     end
1710 end
1711
1712 def cleanRunnerDirectory
1713     raise unless $bundle
1714     Dir.foreach($runnerDir) {
1715         | filename |
1716         next unless filename =~ /^test_fail/
1717         FileUtils.rm_f $runnerDir + filename
1718     }
1719 end
1720
1721 def sshRead(cmd)
1722     raise unless $remote
1723
1724     result = ""
1725     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1726       | inp |
1727       inp.each_line {
1728         | line |
1729         result += line
1730       }
1731     }
1732     raise "#{$?}" unless $?.success?
1733     result
1734 end
1735
1736 def runCommandOnTester(cmd)
1737     if $remote
1738         result = sshRead(cmd)
1739     else
1740         result = `#{cmd}`
1741     end
1742 end
1743
1744 def numberOfProcessors
1745     if $hostOS == "windows"
1746         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1747     else
1748         begin
1749             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1750         rescue
1751             numProcessors = 0
1752         end
1753
1754         if numProcessors == 0
1755             begin
1756                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1757             rescue
1758                 numProcessors == 0
1759             end
1760         end
1761     end
1762
1763     if numProcessors == 0
1764         numProcessors = 1
1765     end
1766     return numProcessors
1767 end
1768
1769 def runAndMonitorTestRunnerCommand(*cmd)
1770     numberOfTests = 0
1771     Dir.chdir($runnerDir) {
1772         # -1 for the runscript, and -2 for '..' and '.'
1773         numberOfTests = Dir.entries(".").count - 3
1774     }
1775     unless $progressMeter
1776         mysys(cmd.join(' '))
1777     else
1778        running = {}
1779        didRun = {}
1780        didFail = {}
1781        blankLine = true
1782        prevStringLength = 0
1783        IO.popen(cmd.join(' '), mode="r") {
1784            | inp |
1785            inp.each_line {
1786                | line |
1787                line = line.scrub.chomp
1788                if line =~ /^Running /
1789                    running[$~.post_match] = true
1790                elsif line =~ /^PASS: /
1791                    didRun[$~.post_match] = true
1792                elsif line =~ /^FAIL: /
1793                    didRun[$~.post_match] = true
1794                    didFail[$~.post_match] = true
1795                else
1796                    unless blankLine
1797                        print("\r" + " " * prevStringLength + "\r")
1798                    end
1799                    puts line
1800                    blankLine = true
1801                end
1802
1803                def lpad(str, chars)
1804                    str = str.to_s
1805                    if str.length > chars
1806                        str
1807                    else
1808                       "%#{chars}s"%(str)
1809                    end
1810                end
1811
1812                string  = ""
1813                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1814                unless didFail.empty?
1815                    string += " (failed #{didFail.size})"
1816                end
1817                string += " "
1818                (running.size - didRun.size).times {
1819                    string += "."
1820                }
1821                if string.length < prevStringLength
1822                    print string
1823                    print(" " * (prevStringLength - string.length))
1824                end
1825                print string
1826                prevStringLength = string.length
1827                blankLine = false
1828                $stdout.flush
1829            }
1830        }
1831        puts
1832        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1833     end
1834 end
1835
1836 def runTestRunner
1837     if $remote
1838         if !$remoteDirectory
1839             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1840         end
1841         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1842         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1843         remoteScript = "\""
1844         remoteScript += "cd #{$remoteDirectory} && "
1845         remoteScript += "rm -rf #{$outputDir.basename} && "
1846         remoteScript += "tar xzf #{$tarFileName} && "
1847         remoteScript += "cd #{$outputDir.basename}/.runner && "
1848         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1849         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1850         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
1851         $envVars.each { |var| remoteScript += "export " << var << "\n" }
1852         remoteScript += "#{testRunnerCommand}\""
1853         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1854     else
1855         Dir.chdir($runnerDir) {
1856             runAndMonitorTestRunnerCommand(testRunnerCommand)
1857         }
1858     end
1859 end
1860
1861 def detectFailures
1862     raise if $bundle
1863
1864     failures = []
1865     
1866     if $remote
1867         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1868         output.split(/\n/).each {
1869             | line |
1870             next unless line =~ /test_fail_/
1871             failures << $~.post_match.to_i
1872         }
1873     else
1874         Dir.foreach($runnerDir) {
1875             | filename |
1876             next unless filename =~ /test_fail_/
1877             failures << $~.post_match.to_i
1878         }
1879     end
1880
1881     failureSet = {}
1882
1883     failures.each {
1884         | failure | 
1885         appendFailure($runlist[failure])
1886         failureSet[failure] = true
1887     }
1888
1889     familyMap = {}
1890     $runlist.each_with_index {
1891         | plan, index |
1892         unless familyMap[plan.family]
1893             familyMap[plan.family] = []
1894         end
1895         if failureSet[index]
1896             appendResult(plan, false)
1897             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
1898             next
1899         else
1900             appendResult(plan, true)
1901             familyMap[plan.family] << {:result => "PASS", :plan => plan};
1902         end
1903         appendPass(plan)
1904     }
1905
1906     File.open($outputDir + "resultsByFamily", "w") {
1907         | outp |
1908         first = true
1909         familyMap.keys.sort.each {
1910             | familyName |
1911             if first
1912                 first = false
1913             else
1914                 outp.puts
1915             end
1916             
1917             outp.print "#{familyName}:"
1918
1919             numPassed = 0
1920             familyMap[familyName].each {
1921                 | entry |
1922                 if entry[:result] == "PASS"
1923                     numPassed += 1
1924                 end
1925             }
1926
1927             if numPassed == familyMap[familyName].size
1928                 outp.puts " PASSED"
1929             elsif numPassed == 0
1930                 outp.puts " FAILED"
1931             else
1932                 outp.puts
1933                 familyMap[familyName].each {
1934                     | entry |
1935                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
1936                 }
1937             end
1938         }
1939     }
1940 end
1941
1942 def compressBundle
1943     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
1944     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1945     raise unless system(cmd)
1946 end
1947
1948 def clean(file)
1949     FileUtils.rm_rf file unless $bundle
1950 end
1951
1952 clean($outputDir + "failed")
1953 clean($outputDir + "passed")
1954 clean($outputDir + "results")
1955 clean($outputDir + "resultsByFamily")
1956 clean($outputDir + ".vm")
1957 clean($outputDir + ".helpers")
1958 clean($outputDir + ".runner")
1959 clean($outputDir + ".tests")
1960 clean($outputDir + "_payload")
1961
1962 Dir.mkdir($outputDir) unless $outputDir.directory?
1963
1964 $outputDir = $outputDir.realpath
1965 $runnerDir = $outputDir + ".runner"
1966
1967 if !$numChildProcesses
1968     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
1969         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
1970     else
1971         $numChildProcesses = numberOfProcessors
1972     end
1973 end
1974
1975 if ENV["JSCTEST_timeout"]
1976     # In the worst case, the processors just interfere with each other.
1977     # Increase the timeout proportionally to the number of processors.
1978     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
1979 end
1980
1981 def runBundle
1982     raise unless $bundle
1983
1984     cleanRunnerDirectory
1985     cleanOldResults
1986     runTestRunner
1987     cleanEmptyResultFiles
1988 end
1989
1990 def runNormal
1991     raise if $bundle or $tarball
1992
1993     prepareBundle
1994     prepareTestRunner
1995     runTestRunner
1996     cleanEmptyResultFiles
1997     detectFailures
1998 end
1999
2000 def runTarball
2001     raise unless $tarball
2002
2003     prepareBundle 
2004     prepareTestRunner
2005     compressBundle
2006 end
2007
2008 def runRemote
2009     raise unless $remote
2010
2011     prepareBundle
2012     prepareTestRunner
2013     compressBundle
2014     runTestRunner
2015     detectFailures
2016 end
2017
2018 puts
2019 if $bundle
2020     runBundle
2021 elsif $remote
2022     runRemote
2023 elsif $tarball
2024     runTarball
2025 else
2026     runNormal
2027 end