[ARM] Disable tests that run out of memory
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2016 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
47 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
48 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
49 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
50 WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
51 CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
52 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
53 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
54
55 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
56
57 begin
58     require 'shellwords'
59 rescue Exception => e
60     $stderr.puts "Warning: did not find shellwords, not running any tests."
61     exit 0
62 end
63
64 $canRunDisplayProfilerOutput = false
65
66 begin
67     require 'rubygems'
68     require 'json'
69     require 'highline'
70     $canRunDisplayProfilerOutput = true
71 rescue Exception => e
72     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
73     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
74     $stderr.puts "Error: #{e.inspect}"
75 end
76
77 def printCommandArray(*cmd)
78     begin
79         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
80     rescue
81         commandArray = cmd.join(' ')
82     end
83     $stderr.puts ">> #{commandArray}"
84 end
85
86 def mysys(*cmd)
87     printCommandArray(*cmd) if $verbosity >= 1
88     raise "Command failed: #{$?.inspect}" unless system(*cmd)
89 end
90
91 def escapeAll(array)
92     array.map {
93         | v |
94         raise "Detected a non-string in #{inspect}" unless v.is_a? String
95         Shellwords.shellescape(v)
96     }.join(' ')
97 end
98
99
100 $jscPath = nil
101 $doNotMessWithVMPath = false
102 $jitTests = true
103 $memoryLimited = false
104 $outputDir = Pathname.new("results")
105 $verbosity = 0
106 $bundle = nil
107 $tarball = false
108 $tarFileName = "payload.tar.gz"
109 $copyVM = false
110 $testRunnerType = nil
111 $testWriter = "default"
112 $remoteUser = nil
113 $remoteHost = nil
114 $remotePort = nil
115 $remoteDirectory = nil
116 $architecture = nil
117 $hostOS = nil
118 $filter = nil
119 $envVars = []
120 $mode = "full"
121 $buildType = "release"
122 $forceCollectContinuously = false
123
124 def usage
125     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
126     puts
127     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
128     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
129     puts "                            --jsc specifies an already present JavaScriptCore to test."
130     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
131     puts "                            Skip tests tagged with //@skip if $memoryLimited"
132     puts "--no-jit                    Do not run JIT specific tests."
133     puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
134     puts "                            platform."
135     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
136     puts "--verbose            (-v)   Print more things while running."
137     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
138     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
139     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
140     puts "                            e.g. x86, x86_64, arm."
141     puts "--os                        Specify os instead of determining from JavaScriptCore build."
142     puts "                            e.g. darwin, linux & windows."
143     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
144     puts "                            In general the shell runner is slower than the make runner."
145     puts "--make-runner               Uses the faster make-based runner."
146     puts "--ruby-runner               Uses the ruby runner for machines without unix shell or make."
147     puts "--test-writer [writer]      Specifies the test script format."
148     puts "                            default is to use shell scripts to run the tests"
149     puts "                            \"ruby\" to use ruby scripts for systems without a unix shell."
150     puts "--remote                    Specify a remote host on which to run tests from command line argument."
151     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
152     puts "--child-processes    (-c)   Specify the number of child processes."
153     puts "--filter                    Only run tests whose name matches the given regular expression."
154     puts "--help               (-h)   Print this message."
155     puts "--env-vars                  Add a list of environment variables to set before running jsc."
156     puts "                            Each environment variable should be separated by a space."
157     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
158     puts "                            it will override the default value."
159     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
160     puts "--basic                     Run with default and these additional modes: no-llint,"
161     puts "                            no-cjit-validate-phases, no-cjit-collect-continuously, dfg-eager"
162     puts "                            and for FTL platforms: no-ftl, ftl-eager-no-cjit and"
163     puts "                            ftl-no-cjit-small-pool."
164     exit 1
165 end
166
167 jscArg = nil
168
169 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
170                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
171                ['--no-copy', GetoptLong::NO_ARGUMENT],
172                ['--memory-limited', GetoptLong::NO_ARGUMENT],
173                ['--no-jit', GetoptLong::NO_ARGUMENT],
174                ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
175                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
176                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
177                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
178                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
179                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
180                ['--os', GetoptLong::REQUIRED_ARGUMENT],
181                ['--shell-runner', GetoptLong::NO_ARGUMENT],
182                ['--make-runner', GetoptLong::NO_ARGUMENT],
183                ['--ruby-runner', GetoptLong::NO_ARGUMENT],
184                ['--test-writer', GetoptLong::REQUIRED_ARGUMENT],
185                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
186                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
187                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
188                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
189                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
190                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
191                ['--debug', GetoptLong::NO_ARGUMENT],
192                ['--release', GetoptLong::NO_ARGUMENT],
193                ['--quick', '-q', GetoptLong::NO_ARGUMENT],
194                ['--basic', GetoptLong::NO_ARGUMENT]).each {
195     | opt, arg |
196     case opt
197     when '--help'
198         usage
199     when '--jsc'
200         jscArg = arg
201     when '--no-copy'
202         $doNotMessWithVMPath = true
203     when '--output-dir'
204         $outputDir = Pathname.new(arg)
205     when '--memory-limited'
206         $memoryLimited = true
207     when '--no-jit'
208         $jitTests = false
209     when '--force-collectContinuously'
210         $forceCollectContinuously = true;
211     when '--verbose'
212         $verbosity += 1
213     when '--run-bundle'
214         $bundle = Pathname.new(arg)
215     when '--tarball'
216         $tarball = true
217         $copyVM = true
218         $tarFileName = arg unless arg == ''
219     when '--force-vm-copy'
220         $copyVM = true
221     when '--shell-runner'
222         $testRunnerType = :shell
223     when '--make-runner'
224         $testRunnerType = :make
225     when '--ruby-runner'
226         $testRunnerType = :ruby
227     when '--test-writer'
228         $testWriter = arg
229     when '--remote'
230         $copyVM = true
231         $tarball = true
232         $remote = true
233         uri = URI("ssh://" + arg)
234         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
235     when '--remote-config-file'
236         $remoteConfigFile = arg
237     when '--child-processes'
238         $numChildProcesses = arg.to_i
239     when '--filter'
240         $filter = Regexp.new(arg)
241     when '--arch'
242         $architecture = arg
243     when '--os'
244         $hostOS = arg
245     when '--env-vars'
246         $envVars = arg.gsub(/\s+/, ' ').split(' ')
247     when '--quick'
248         $mode = "quick"
249     when '--basic'
250         $mode = "basic"
251     when '--debug'
252         $buildType = "debug"
253     when '--release'
254         $buildType = "release"
255     end
256 }
257
258 if $remoteConfigFile
259     file = File.read($remoteConfigFile)
260     config = JSON.parse(file)
261
262     if !$remote and config['remote']
263         $copyVM = true
264         $tarball = true
265         $remote = true
266         uri = URI("ssh://" + config['remote'])
267         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
268     end
269
270     if config['remoteDirectory']
271         $remoteDirectory = config['remoteDirectory']
272     end
273 end
274
275 unless jscArg
276     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
277     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
278     command += ($buildType == "release") ? " --release" : " --debug"
279     command += " --executablePath"
280
281     output = `#{command}`.split("\n")
282     if !output.length
283         $stderr.puts "Error: must specify --jsc <path>"
284         exit 1
285     end
286
287     output = output[0]
288     jscArg = Pathname.new(output).join("jsc")
289     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
290     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
291     if !File.file?(jscArg)
292         $stderr.puts "Error: must specify --jsc <path>"
293         exit 1
294     end
295
296     puts "Using the following jsc path: #{jscArg}"
297 end
298
299 if $doNotMessWithVMPath
300     $jscPath = Pathname.new(jscArg)
301 else
302     $jscPath = Pathname.new(jscArg).realpath
303 end
304
305 $progressMeter = ($verbosity == 0 and $stdout.tty?)
306
307 if $bundle
308     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
309     $outputDir = $bundle
310 end
311
312 # Try to determine architecture. Return nil on failure.
313 def machOArchitectureCode
314     begin 
315         otoolLines = `otool -afh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
316         otoolLines.each_with_index {
317             | value, index |
318             if value =~ /magic/ and value =~ /cputype/
319                 return otoolLines[index + 1].split[1].to_i
320             end
321         }
322     rescue
323         $stderr.puts "Warning: unable to execute otool."
324     end
325     $stderr.puts "Warning: unable to determine architecture."
326     nil
327 end
328
329 def determineArchitectureFromMachOBinary
330     code = machOArchitectureCode
331     return nil unless code
332     is64BitFlag = 0x01000000
333     case code
334     when 7
335         "x86"
336     when 7 | is64BitFlag
337         "x86-64"
338     when 12
339         "arm"
340     when 12 | is64BitFlag
341         "arm64"
342     else
343         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
344         nil
345     end
346 end
347
348 def determineArchitectureFromELFBinary
349     f = File.open($jscPath.to_s)
350     data = f.read(19)
351
352     if !(data[0,4] == "\x7F\x45\x4C\x46")
353         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
354         return nil
355     end
356
357     code = data[18].ord
358     case code
359     when 3
360         "x86"
361     when 8
362         "mips"
363     when 62
364         "x86-64"
365     when 40
366         "arm"
367     when 183
368         "arm64"
369     else
370         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
371         nil
372     end
373 end
374
375 def determineArchitectureFromPEBinary
376     f = File.open($jscPath.to_s)
377     data = f.read(1024)
378
379     if !(data[0, 2] == "MZ")
380         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
381         return nil
382     end
383
384     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
385
386     if !(data[peHeaderAddr, 4] == "PE\0\0")
387         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
388         return nil
389     end
390
391     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
392
393     case machine
394     when 0x014c
395         "x86"
396     when 0x8664
397         "x86-64"
398     else
399         $stderr.puts "Warning: unsupported machine type: #{machine}"
400         nil
401     end
402 end
403
404 def determineArchitecture
405     case $hostOS
406     when "darwin"
407         determineArchitectureFromMachOBinary
408     when "linux"
409         determineArchitectureFromELFBinary
410     when "windows"
411         determineArchitectureFromPEBinary
412     else
413         $stderr.puts "Warning: unable to determine architecture on this platform."
414         nil
415     end
416 end
417
418 def determineOS
419     case RbConfig::CONFIG["host_os"]
420     when /darwin/i
421         "darwin"
422     when /linux/i
423         "linux"
424     when /mswin|mingw|cygwin/
425         "windows"
426     else
427         $stderr.puts "Warning: unable to determine host operating system"
428         nil
429     end
430 end
431
432 $hostOS = determineOS unless $hostOS
433 $architecture = determineArchitecture unless $architecture
434 $isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $architecture == "mips" || $hostOS == "windows")
435
436 def isFTLEnabled
437     $jitTests && $isFTLPlatform
438 end
439
440 if !$testRunnerType
441     if $remote and $hostOS == "darwin"
442         $testRunnerType = :shell
443     else
444         $testRunnerType = :make
445     end
446 end
447
448 if $testWriter
449     if /[^-a-zA-Z0-9_]/.match($testWriter)
450         raise "Invalid test writer #{$testWriter} given"
451     end
452 end
453
454 $numFailures = 0
455 $numPasses = 0
456
457 # We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
458 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--validateExceptionChecks=true", "--useDollarVM=true", "--maxPerThreadStackUsage=1572864"]
459 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
460 # NOTE: Tests rely on this using scribbleFreeCells.
461 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
462 B3O1_OPTIONS = ["--defaultB3OptLevel=1"]
463 FTL_OPTIONS = ["--useFTLJIT=true"]
464 PROBE_OSR_EXIT_OPTION = ["--useProbeOSRExit=true"]
465
466 require_relative "webkitruby/jsc-stress-test-writer-#{$testWriter}"
467
468 def shouldCollectContinuously?
469     $buildType == "release" or $forceCollectContinuously
470 end
471
472 COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false"] : []
473
474 $runlist = []
475
476 def frameworkFromJSCPath(jscPath)
477     parentDirectory = jscPath.dirname
478     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
479         parentDirectory.dirname
480     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
481         jscPath.dirname + "JavaScriptCore.framework"
482     else
483         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
484         nil
485     end
486 end
487
488 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
489     dir = Pathname.new(".")
490     $benchmarkDirectory.each_filename {
491         | pathComponent |
492         dir += ".."
493     }
494     dir + resourcePath
495 end
496
497 def pathToVM
498     pathToBundleResourceFromBenchmarkDirectory($jscPath)
499 end
500
501 def pathToHelpers
502     pathToBundleResourceFromBenchmarkDirectory(".helpers")
503 end
504
505 $runCommandOptions = {}
506
507 $uniqueFilenameCounter = 0
508 def uniqueFilename(extension)
509     payloadDir = $outputDir + "_payload"
510     Dir.mkdir payloadDir unless payloadDir.directory?
511     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
512     $uniqueFilenameCounter += 1
513     result
514 end
515
516 def baseOutputName(kind)
517     "#{$collectionName}/#{$benchmark}.#{kind}"
518 end
519
520 def addRunCommand(kind, command, outputHandler, errorHandler, *additionalEnv)
521     $didAddRunCommand = true
522     name = baseOutputName(kind)
523     if $filter and name !~ $filter
524         return
525     end
526     plan = Plan.new(
527         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
528         errorHandler)
529     plan.additionalEnv.push(*additionalEnv)
530     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
531         $runlist.unshift plan
532     else
533         $runlist << plan
534     end
535 end
536
537 # Returns true if there were run commands found in the file ($benchmarkDirectory +
538 # $benchmark), in which case those run commands have already been executed. Otherwise
539 # returns false, in which case you're supposed to add your own run commands.
540 def parseRunCommands
541     oldDidAddRunCommand = $didAddRunCommand
542     $didAddRunCommand = false
543     $skipped = false
544
545     Dir.chdir($outputDir) {
546         File.open($benchmarkDirectory + $benchmark) {
547             | inp |
548             inp.each_line {
549                 | line |
550                 begin
551                     doesMatch = line =~ /^\/\/@/
552                 rescue Exception => e
553                     # Apparently this happens in the case of some UTF8 stuff in some files, where
554                     # Ruby tries to be strict and throw exceptions.
555                     next
556                 end
557                 next unless doesMatch
558                 eval $~.post_match
559                 if $skipped
560                     break
561                 end
562             }
563         }
564     }
565
566     result = $didAddRunCommand
567     $didAddRunCommand = result or oldDidAddRunCommand
568     result
569 end
570
571 def slow!
572     $runCommandOptions[:isSlow] = true
573 end
574
575 def runWithOutputHandler(kind, outputHandler, *options)
576     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
577 end
578
579 def run(kind, *options)
580     runWithOutputHandler(kind, silentOutputHandler, *options)
581 end
582
583 def runNoFTL(*optionalTestSpecificOptions)
584     run("no-ftl", *optionalTestSpecificOptions)
585 end
586
587 def runWithRAMSize(size, *optionalTestSpecificOptions)
588     run("ram-size-#{size}", "--forceRAMSize=#{size}", *optionalTestSpecificOptions)
589 end
590
591 def runOneLargeHeap(*optionalTestSpecificOptions)
592     if $memoryLimited
593         $didAddRunCommand = true
594         puts "Skipping #{$collectionName}/#{$benchmark}"
595     else
596         run("default", *optionalTestSpecificOptions)
597     end
598 end
599
600 def runNoJIT(*optionalTestSpecificOptions)
601     run("no-jit", "--useJIT=false", *optionalTestSpecificOptions)
602 end
603
604 def runNoLLInt(*optionalTestSpecificOptions)
605     if $jitTests
606         run("no-llint", "--useLLInt=false", *optionalTestSpecificOptions)
607     end
608 end
609
610 # NOTE: Tests rely on this using scribbleFreeCells.
611 def runNoCJITValidate(*optionalTestSpecificOptions)
612     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
613 end
614
615 def runNoCJITValidatePhases(*optionalTestSpecificOptions)
616     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
617 end
618
619 def runNoCJITCollectContinuously(*optionalTestSpecificOptions)
620     run("no-cjit-collect-continuously", *(NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
621 end
622
623 def runDefault(*optionalTestSpecificOptions)
624     run("default", *(FTL_OPTIONS + optionalTestSpecificOptions))
625 end
626
627 def runBigIntEnabled(*optionalTestSpecificOptions)
628     run("big-int-enabled", "--useBigInt=true" , *(FTL_OPTIONS + optionalTestSpecificOptions))
629 end
630
631 def runFTLNoCJIT(*optionalTestSpecificOptions)
632     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
633 end
634
635 def runFTLNoCJITB3O1(*optionalTestSpecificOptions)
636     run("ftl-no-cjit-b3o1", "--useArrayAllocationProfiling=false", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
637 end
638
639 def runFTLNoCJITValidate(*optionalTestSpecificOptions)
640     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + PROBE_OSR_EXIT_OPTION + optionalTestSpecificOptions))
641 end
642
643 def runFTLNoCJITNoPutStackValidate(*optionalTestSpecificOptions)
644     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
645 end
646
647 def runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
648     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
649 end
650
651 def runFTLNoCJITOSRValidation(*optionalTestSpecificOptions)
652     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
653 end
654
655 def runDFGEager(*optionalTestSpecificOptions)
656     run("dfg-eager", *(EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + PROBE_OSR_EXIT_OPTION + optionalTestSpecificOptions))
657 end
658
659 def runDFGEagerNoCJITValidate(*optionalTestSpecificOptions)
660     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
661 end
662
663 def runFTLEager(*optionalTestSpecificOptions)
664     run("ftl-eager", "--airForceBriggsAllocator=true", "--forcePolyProto=true", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
665 end
666
667 def runFTLEagerWatchdog(*optionalTestSpecificOptions)
668     timeout = rand(100)
669     run("ftl-eager-watchdog-#{timeout}", "--watchdog=#{timeout}", "--watchdog-exception-ok", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
670 end
671
672 def runFTLEagerNoCJITValidate(*optionalTestSpecificOptions)
673     run("ftl-eager-no-cjit", "--validateGraph=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
674 end
675
676 def runFTLEagerNoCJITB3O1(*optionalTestSpecificOptions)
677     run("ftl-eager-no-cjit-b3o1", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
678 end
679
680 def runFTLEagerNoCJITOSRValidation(*optionalTestSpecificOptions)
681     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
682 end
683
684 def runNoCJITNoASO(*optionalTestSpecificOptions)
685     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
686 end
687
688 def runNoCJITNoAccessInlining(*optionalTestSpecificOptions)
689     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
690 end
691
692 def runFTLNoCJITNoAccessInlining(*optionalTestSpecificOptions)
693     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
694 end
695
696 def runFTLNoCJITSmallPool(*optionalTestSpecificOptions)
697     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
698 end
699
700 def runNoCJIT(*optionalTestSpecificOptions)
701     run("no-cjit", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
702 end
703
704 def runDFGMaximalFlushPhase(*optionalTestSpecificOptions)
705     run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
706 end
707
708 def runShadowChicken(*optionalTestSpecificOptions)
709     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true", *optionalTestSpecificOptions)
710 end
711
712 def defaultRun
713     if $mode == "quick"
714         defaultQuickRun
715     else
716         runDefault
717         if $jitTests
718             runNoLLInt
719             runNoCJITValidatePhases
720             runNoCJITCollectContinuously if shouldCollectContinuously?
721             runDFGEager
722             if $mode != "basic"
723                 runDFGEagerNoCJITValidate
724                 runDFGMaximalFlushPhase
725             end
726
727             return if !$isFTLPlatform
728
729             runNoFTL
730             runFTLEager
731             runFTLEagerNoCJITValidate
732             runFTLNoCJITSmallPool
733
734             return if $mode == "basic"
735
736             runFTLNoCJITValidate
737             runFTLNoCJITB3O1
738             runFTLNoCJITNoPutStackValidate
739             runFTLNoCJITNoInlineValidate
740             runFTLEagerNoCJITB3O1
741         end
742     end
743 end
744
745 def defaultNoNoLLIntRun
746     if $mode == "quick"
747         defaultQuickRun
748     else
749         runDefault
750         if $jitTests
751             runNoCJITValidatePhases
752             runNoCJITCollectContinuously if shouldCollectContinuously?
753             runDFGEager
754             if $mode != "basic"
755                 runDFGEagerNoCJITValidate
756                 runDFGMaximalFlushPhase
757             end
758
759             return if !$isFTLPlatform
760
761             runNoFTL
762             runFTLNoCJITValidate
763             runFTLNoCJITSmallPool
764
765             return if $mode == "basic"
766
767             runFTLNoCJITB3O1
768             runFTLNoCJITNoPutStackValidate
769             runFTLNoCJITNoInlineValidate
770             runFTLEager
771             runFTLEagerNoCJITValidate
772         end
773     end
774 end
775
776 def defaultQuickRun
777     runDefault
778     if $jitTests
779         runNoCJITValidate
780
781         return if !$isFTLPlatform
782
783         runNoFTL
784         runFTLNoCJITValidate
785     end
786 end
787
788 def defaultSpotCheckNoMaximalFlush
789     defaultQuickRun
790     runNoCJITNoAccessInlining
791
792     return if !$isFTLPlatform
793
794     runFTLNoCJITOSRValidation
795     runFTLNoCJITNoAccessInlining
796     runFTLNoCJITB3O1
797 end
798
799 def defaultSpotCheck
800     defaultSpotCheckNoMaximalFlush
801     runDFGMaximalFlushPhase
802 end
803
804 # This is expected to not do eager runs because eager runs can have a lot of recompilations
805 # for reasons that don't arise in the real world. It's used for tests that assert convergence
806 # by counting recompilations.
807 def defaultNoEagerRun
808     runDefault
809     if $jitTests
810         runNoLLInt
811         runNoCJITValidatePhases
812         runNoCJITCollectContinuously if shouldCollectContinuously?
813
814         return if !$isFTLPlatform
815
816         runNoFTL
817         runFTLNoCJITValidate
818
819         return if $mode == "basic"
820
821         runFTLNoCJITNoInlineValidate
822         runFTLNoCJITB3O1
823     end
824 end
825
826 def defaultNoSamplingProfilerRun
827     runDefault
828     if $jitTests
829         runNoLLInt
830         runNoCJITValidatePhases
831         runNoCJITCollectContinuously if shouldCollectContinuously?
832         runDFGEager
833         runDFGEagerNoCJITValidate
834         runDFGMaximalFlushPhase
835
836         return if !$isFTLPlatform
837
838         runNoFTL
839         runFTLNoCJITNoPutStackValidate
840         runFTLNoCJITNoInlineValidate
841         runFTLEager
842         runFTLEagerNoCJITValidate
843         runFTLNoCJITSmallPool
844     end
845 end
846
847 def runProfiler
848     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
849         skip
850         return
851     end
852
853     profilerOutput = uniqueFilename(".json")
854     if $canRunDisplayProfilerOutput
855         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
856     else
857         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
858         run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
859     end
860 end
861
862 def runExceptionFuzz
863     subCommand = escapeAll([pathToVM.to_s, "--useDollarVM=true", $benchmark.to_s])
864     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
865 end
866
867 def runExecutableAllocationFuzz(name, *options)
868     subCommand = escapeAll([pathToVM.to_s, "--useDollarVM=true", $benchmark.to_s] + options)
869     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
870 end
871
872 def runTypeProfiler
873     if !$jitTests
874         return
875     end
876
877     run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
878     run("ftl-no-cjit-type-profiler-force-poly-proto", "--useTypeProfiler=true", "--forcePolyProto=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
879
880     return if !$isFTLPlatform
881
882     run("ftl-type-profiler-ftl-eager", "--useTypeProfiler=true", *(FTL_OPTIONS + EAGER_OPTIONS))
883 end
884
885 def runControlFlowProfiler
886     if !$jitTests
887         return
888     end
889
890     return if !$isFTLPlatform
891
892     run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
893 end
894
895 def runTest262(mode, exception, includeFiles, flags)
896     failsWithException = exception != "NoException"
897     isStrict = false
898     isModule = false
899     isAsync = false
900
901     flags.each {
902         | flag |
903         case flag
904         when :strict
905             isStrict = true
906         when :module
907             isModule = true
908         when :async
909             isAsync = true
910         else
911             raise "Invalid flag for runTest262, #{flag}"
912         end
913     }
914
915     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
916
917     args = [pathToVM.to_s] + BASE_OPTIONS
918     args << "--exception=" + exception if failsWithException
919     args << "--test262-async" if isAsync
920     args += includeFiles
921
922     case mode
923     when :normal
924         errorHandler = simpleErrorHandler
925         outputHandler = silentOutputHandler
926     when :fail
927         errorHandler = expectedFailErrorHandler
928         outputHandler = noisyOutputHandler
929     when :failDueToOutdatedOrBadTest
930         errorHandler = expectedFailErrorHandler
931         outputHandler = noisyOutputHandler
932     when :skip
933         return
934     else
935         raise "Invalid mode: #{mode}"
936     end
937
938     if isStrict
939         kind = "default-strict"
940         args << "--strict-file=#{$benchmark}"
941     else
942         kind = "default"
943         if isModule
944             args << "--module-file=#{$benchmark}"
945         else
946             args << $benchmark.to_s
947         end
948     end
949
950     addRunCommand(kind, args, outputHandler, errorHandler)
951 end
952
953 def prepareTest262Fixture
954     # This function is used to add the files used by Test262 modules tests.
955     prepareExtraRelativeFiles([""], $collection)
956 end
957
958 def runES6(mode)
959     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
960     case mode
961     when :normal
962         errorHandler = simpleErrorHandler
963     when :fail
964         errorHandler = expectedFailErrorHandler
965     when :failDueToOutdatedOrBadTest
966         errorHandler = expectedFailErrorHandler
967     when :skip
968         return
969     else
970         raise "Invalid mode: #{mode}"
971     end
972     addRunCommand("default", args, noisyOutputHandler, errorHandler)
973 end
974
975 def defaultRunModules(noLLInt: true)
976     run("default-modules", "-m")
977
978     if !$jitTests
979         return
980     end
981
982     run("no-llint-modules", "-m", "--useLLInt=false") if noLLInt
983     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
984     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
985     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
986
987     return if !$isFTLPlatform
988
989     run("default-ftl-modules", "-m", *FTL_OPTIONS)
990     run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
991     run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
992     run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
993     run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
994     run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
995 end
996
997 def noNoLLIntRunModules
998     defaultRunModules(noLLInt: false)
999 end
1000
1001 def runWebAssembly
1002     return if !$jitTests
1003     return if !$isFTLPlatform
1004     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1005     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1006     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1007     run("default-wasm", "-m", *FTL_OPTIONS)
1008     if $mode != "quick"
1009         run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1010         run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1011         run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1012         run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1013         run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
1014     end
1015 end
1016
1017 def runWebAssemblyEmscripten(mode)
1018     case mode
1019     when :skip
1020         return
1021     end
1022     return if !$jitTests
1023     return if !$isFTLPlatform
1024     wasm = $benchmark.to_s.sub! '.js', '.wasm'
1025     prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
1026     run("default-wasm", *FTL_OPTIONS)
1027     if $mode != "quick"
1028         run("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1029         run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1030         run("wasm-no-call-ic", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1031         run("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1032     end
1033 end
1034
1035 def runWebAssemblySpecTest(mode)
1036     case mode
1037     when :skip
1038         return
1039     end
1040     return if !$jitTests
1041     return if !$isFTLPlatform
1042     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1043
1044     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1045     prepareExtraRelativeFiles(modules.map { |f| "../../" + f }, $collection)
1046
1047     harness = Dir[WASMTESTS_PATH + "spec-harness/" + "*.js"].map { |f| File.basename(f) }
1048     prepareExtraRelativeFiles(harness.map { |f| "../../spec-harness/" + f }, $collection)
1049
1050     runWithOutputHandler("default-wasm", noisyOutputHandler, "../spec-harness.js", *FTL_OPTIONS)
1051     if $mode != "quick"
1052       runWithOutputHandler("wasm-no-cjit-yes-tls-context", noisyOutputHandler, "../spec-harness.js",  "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1053       runWithOutputHandler("wasm-eager-jettison", noisyOutputHandler, "../spec-harness.js", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1054       runWithOutputHandler("wasm-no-call-ic", noisyOutputHandler, "../spec-harness.js", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1055       runWithOutputHandler("wasm-no-tls-context", noisyOutputHandler, "../spec-harness.js", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1056     end
1057 end
1058
1059 def runWebAssemblyLowExecutableMemory(*optionalTestSpecificOptions)
1060     return if !$jitTests
1061     return if !$isFTLPlatform
1062     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1063     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1064     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1065     # Only let WebAssembly get executable memory.
1066     run("default-wasm", "--useConcurrentGC=0" , "--useConcurrentJIT=0", "--jitMemoryReservationSize=15000", "--useBaselineJIT=0", "--useDFGJIT=0", "--useFTLJIT=0", "-m")
1067 end
1068
1069 def runChakra(mode, exception, baselineFile, extraFiles)
1070     raise unless $benchmark.to_s =~ /\.js$/
1071     failsWithException = exception != "NoException"
1072     testName = $~.pre_match
1073
1074     prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
1075     prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)
1076
1077     args = [pathToVM.to_s] + BASE_OPTIONS
1078     args += FTL_OPTIONS if $isFTLPlatform
1079     args += EAGER_OPTIONS
1080     args << "--exception=" + exception if failsWithException
1081     args << "--dumpException" if failsWithException
1082     args += ["jsc-lib.js"]
1083
1084     case mode
1085     when :baseline
1086         prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
1087         errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
1088         outputHandler = noisyOutputHandler
1089     when :pass
1090         errorHandler = chakraPassFailErrorHandler
1091         outputHandler = noisyOutputHandler
1092     when :skipDueToOutdatedOrBadTest
1093         return
1094     when :skip
1095         return
1096     else
1097         raise "Invalid mode: #{mode}"
1098     end
1099
1100     kind = "default"
1101     args << $benchmark.to_s
1102
1103     addRunCommand(kind, args, outputHandler, errorHandler)
1104 end
1105
1106 def runLayoutTest(kind, *options)
1107     raise unless $benchmark.to_s =~ /\.js$/
1108     testName = $~.pre_match
1109     if kind
1110         kind = "layout-" + kind
1111     else
1112         kind = "layout"
1113     end
1114
1115     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1116     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1117
1118     args = [pathToVM.to_s] + BASE_OPTIONS + options +
1119         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1120          $benchmark.to_s,
1121          (Pathname.new("resources") + "standalone-post.js").to_s]
1122     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1123 end
1124
1125 def runLayoutTestNoFTL
1126     runLayoutTest("no-ftl")
1127 end
1128
1129 def runLayoutTestNoLLInt
1130     runLayoutTest("no-llint", "--useLLInt=false")
1131 end
1132
1133 def runLayoutTestNoCJIT
1134     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1135 end
1136
1137 def runLayoutTestDFGEagerNoCJIT
1138     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1139 end
1140
1141 def runLayoutTestDefault
1142     runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
1143 end
1144
1145 def runLayoutTestFTLNoCJIT
1146     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1147 end
1148
1149 def runLayoutTestFTLEagerNoCJIT
1150     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1151 end
1152
1153 def runLayoutTestFTLEagerNoCJITB3O1
1154     runLayoutTest("ftl-eager-no-cjit-b3o1", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS))
1155 end
1156
1157 def noFTLRunLayoutTest
1158     if !$jitTests
1159         return
1160     end
1161
1162     runLayoutTestNoLLInt
1163     runLayoutTestNoCJIT
1164     runLayoutTestDFGEagerNoCJIT
1165 end
1166
1167 def defaultQuickRunLayoutTest
1168     runLayoutTestDefault
1169     if $jitTests
1170         if $isFTLPlatform
1171             runLayoutTestNoFTL
1172             runLayoutTestFTLNoCJIT
1173             runLayoutTestFTLEagerNoCJIT
1174         else
1175             noFTLRunLayoutTest
1176         end
1177     end
1178 end
1179
1180 def defaultRunLayoutTest
1181     if $mode == "quick"
1182         defaultQuickRunLayoutTest
1183     else
1184         runLayoutTestDefault
1185         if $jitTests
1186             noFTLRunLayoutTest
1187
1188             return if !$isFTLPlatform
1189
1190             runLayoutTestNoFTL
1191             runLayoutTestFTLNoCJIT
1192             runLayoutTestFTLEagerNoCJIT
1193         end
1194     end
1195 end
1196
1197 def noEagerNoNoLLIntTestsRunLayoutTest
1198     runLayoutTestDefault
1199     if $jitTests
1200         runLayoutTestNoCJIT
1201
1202         return if !$isFTLPlatform
1203
1204         runLayoutTestNoFTL
1205         runLayoutTestFTLNoCJIT
1206     end
1207 end
1208
1209 def noNoLLIntRunLayoutTest
1210     runLayoutTestDefault
1211     if $jitTests
1212         runLayoutTestNoCJIT
1213         runLayoutTestDFGEagerNoCJIT
1214
1215         return if !$isFTLPlatform
1216
1217         runLayoutTestNoFTL
1218         runLayoutTestFTLNoCJIT
1219         runLayoutTestFTLEagerNoCJIT
1220     end
1221 end
1222
1223 def prepareExtraRelativeFiles(extraFiles, destination)
1224     Dir.chdir($outputDir) {
1225         extraFiles.each {
1226             | file |
1227             dest = destination + file
1228             FileUtils.mkdir_p(dest.dirname)
1229             FileUtils.cp $extraFilesBaseDir + file, dest
1230         }
1231     }
1232 end
1233
1234 def baseDirForCollection(collectionName)
1235     Pathname(".tests") + collectionName
1236 end
1237
1238 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1239     raise unless absoluteBase.absolute?
1240     Dir.chdir($outputDir) {
1241         collectionBaseDir = baseDirForCollection($collectionName)
1242         extraFiles.each {
1243             | file |
1244             destination = collectionBaseDir + file
1245             FileUtils.mkdir_p destination.dirname unless destination.directory?
1246             FileUtils.cp absoluteBase + file, destination
1247         }
1248     }
1249 end
1250
1251 def runMozillaTest(kind, mode, extraFiles, *options)
1252     if kind
1253         kind = "mozilla-" + kind
1254     else
1255         kind = "mozilla"
1256     end
1257     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1258     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1259     case mode
1260     when :normal
1261         errorHandler = mozillaErrorHandler
1262     when :negative
1263         errorHandler = mozillaExit3ErrorHandler
1264     when :fail
1265         errorHandler = mozillaFailErrorHandler
1266     when :failDueToOutdatedOrBadTest
1267         errorHandler = mozillaFailErrorHandler
1268     when :skip
1269         return
1270     else
1271         raise "Invalid mode: #{mode}"
1272     end
1273     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1274 end
1275
1276 def runMozillaTestDefault(mode, *extraFiles)
1277     runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
1278 end
1279
1280 def runMozillaTestNoFTL(mode, *extraFiles)
1281     runMozillaTest("no-ftl", mode, extraFiles)
1282 end
1283
1284 def runMozillaTestLLInt(mode, *extraFiles)
1285     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1286 end
1287
1288 def runMozillaTestBaselineJIT(mode, *extraFiles)
1289     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1290 end
1291
1292 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1293     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1294 end
1295
1296 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1297     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1298 end
1299
1300 def defaultQuickRunMozillaTest(mode, *extraFiles)
1301     if $jitTests
1302         runMozillaTestDefault(mode, *extraFiles)
1303         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1304     else
1305         runMozillaTestNoFTL(mode, *extraFiles)
1306         if $jitTests
1307             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1308         end
1309     end
1310 end
1311
1312 def defaultRunMozillaTest(mode, *extraFiles)
1313     if $mode == "quick"
1314         defaultQuickRunMozillaTest(mode, *extraFiles)
1315     else
1316         runMozillaTestNoFTL(mode, *extraFiles)
1317         if $jitTests
1318             runMozillaTestLLInt(mode, *extraFiles)
1319             runMozillaTestBaselineJIT(mode, *extraFiles)
1320             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1321             runMozillaTestDefault(mode, *extraFiles)
1322             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
1323         end
1324     end
1325 end
1326
1327 def runNoisyTestImpl(kind, options, additionalEnv)
1328     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler, *additionalEnv)
1329 end
1330
1331 def runNoisyTest(kind, *options)
1332     runNoisyTestImpl(kind, options, [])
1333 end
1334
1335 def runNoisyTestWithEnv(kind, *additionalEnv)
1336     runNoisyTestImpl(kind, [], additionalEnv)
1337 end
1338
1339 def runNoisyTestDefault
1340     runNoisyTest("default", *FTL_OPTIONS)
1341 end
1342
1343 def runNoisyTestNoFTL
1344     runNoisyTest("no-ftl")
1345 end
1346
1347 def runNoisyTestNoCJIT
1348     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1349 end
1350
1351 def runNoisyTestNoCJITB3O1
1352     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS))
1353 end
1354
1355 def runNoisyTestEagerNoCJIT
1356     runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1357 end
1358
1359 def defaultRunNoisyTest
1360     runNoisyTestDefault
1361     if $jitTests and $isFTLPlatform
1362         runNoisyTestNoFTL
1363         runNoisyTestNoCJIT
1364         runNoisyTestNoCJITB3O1
1365         runNoisyTestEagerNoCJIT
1366     end
1367 end
1368
1369 def skip
1370     $didAddRunCommand = true
1371     $skipped = true
1372     puts "Skipping #{$collectionName}/#{$benchmark}"
1373 end
1374
1375 def allJSFiles(path)
1376     if path.file?
1377         [path]
1378     else
1379         result = []
1380         Dir.foreach(path) {
1381             | filename |
1382             next unless filename =~ /\.js$/
1383             next unless (path + filename).file?
1384             result << path + filename
1385         }
1386         result
1387     end
1388 end
1389
1390 def uniqueifyName(names, name)
1391     result = name.to_s
1392     toAdd = 1
1393     while names[result]
1394         result = "#{name}-#{toAdd}"
1395         toAdd += 1
1396     end
1397     names[result] = true
1398     result
1399 end
1400
1401 def simplifyCollectionName(collectionPath)
1402     outerDir = collectionPath.dirname
1403     name = collectionPath.basename
1404     lastName = name
1405     if collectionPath.directory?
1406         while lastName.to_s =~ /test/
1407             lastName = outerDir.basename
1408             name = lastName + name
1409             outerDir = outerDir.dirname
1410         end
1411     end
1412     uniqueifyName($collectionNames, name)
1413 end
1414
1415 def prepareCollection(name)
1416     FileUtils.mkdir_p $outputDir + name
1417
1418     absoluteCollection = $collection.realpath
1419
1420     Dir.chdir($outputDir) {
1421         bundleDir = baseDirForCollection(name)
1422
1423         # Create the proper directory structures.
1424         FileUtils.mkdir_p bundleDir
1425         if bundleDir.basename == $collection.basename
1426             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1427             $collection = bundleDir
1428         else
1429             FileUtils.cp_r absoluteCollection, bundleDir
1430             $collection = bundleDir + $collection.basename
1431         end
1432
1433         $extraFilesBaseDir = absoluteCollection
1434     }
1435 end
1436
1437 $collectionNames = {}
1438
1439 def handleCollectionFile(collection)
1440     collectionName = simplifyCollectionName(collection)
1441    
1442     paths = {}
1443     subCollections = []
1444     YAML::load(IO::read(collection)).each {
1445         | entry |
1446         if entry["collection"]
1447             subCollections << entry["collection"]
1448             next
1449         end
1450         
1451         if Pathname.new(entry["path"]).absolute?
1452             raise "Absolute path: " + entry["path"] + " in #{collection}"
1453         end
1454         
1455         if paths[entry["path"]]
1456             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1457         end
1458         
1459         subCollection = collection.dirname + entry["path"]
1460         
1461         if subCollection.file?
1462             subCollectionName = Pathname.new(entry["path"]).dirname
1463         else
1464             subCollectionName = entry["path"]
1465         end
1466         
1467         $collection = subCollection
1468         $collectionName = Pathname.new(collectionName)
1469         Pathname.new(subCollectionName).each_filename {
1470             | filename |
1471             next if filename =~ /^\./
1472             $collectionName += filename
1473         }
1474         $collectionName = $collectionName.to_s
1475         
1476         prepareCollection($collectionName)
1477       
1478         Dir.chdir($outputDir) {
1479             pathsToSearch = [$collection]
1480             if entry["tests"]
1481                 if entry["tests"].is_a? Array
1482                     pathsToSearch = entry["tests"].map {
1483                         | testName |
1484                         pathsToSearch[0] + testName
1485                     }
1486                 else
1487                     pathsToSearch[0] += entry["tests"]
1488                 end
1489             end
1490             pathsToSearch.each {
1491                 | pathToSearch |
1492                 allJSFiles(pathToSearch).each {
1493                     | path |
1494                     
1495                     $benchmark = path.basename
1496                     $benchmarkDirectory = path.dirname
1497                     
1498                     $runCommandOptions = {}
1499                     eval entry["cmd"]
1500                 }
1501             }
1502         }
1503     }
1504     
1505     subCollections.each {
1506         | subCollection |
1507         handleCollection(collection.dirname + subCollection)
1508     }
1509 end
1510
1511 def handleCollectionDirectory(collection)
1512     collectionName = simplifyCollectionName(collection)
1513     
1514     $collection = collection
1515     $collectionName = collectionName
1516     prepareCollection(collectionName)
1517    
1518     Dir.chdir($outputDir) {
1519         $benchmarkDirectory = $collection
1520         allJSFiles($collection).each {
1521             | path |
1522             
1523             $benchmark = path.basename
1524             
1525             $runCommandOptions = {}
1526             defaultRun unless parseRunCommands
1527         }
1528     }
1529 end
1530
1531 def handleCollection(collection)
1532     collection = Pathname.new(collection)
1533     
1534     if collection.file?
1535         handleCollectionFile(collection)
1536     else
1537         handleCollectionDirectory(collection)
1538     end
1539 end
1540
1541 def appendFailure(plan)
1542     File.open($outputDir + "failed", "a") {
1543         | outp |
1544         outp.puts plan.name
1545     }
1546     $numFailures += 1
1547 end
1548
1549 def appendPass(plan)
1550     File.open($outputDir + "passed", "a") {
1551         | outp |
1552         outp.puts plan.name
1553     }
1554     $numPasses += 1
1555 end
1556
1557 def appendResult(plan, didPass)
1558     File.open($outputDir + "results", "a") {
1559         | outp |
1560         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1561     }
1562 end
1563
1564 def prepareBundle
1565     raise if $bundle
1566
1567     if $doNotMessWithVMPath
1568         if !$remote and !$tarball
1569             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1570             $jscPath = Pathname.new($jscPath).realpath
1571         else
1572             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1573         end
1574     else
1575         originalJSCPath = $jscPath
1576         vmDir = $outputDir + ".vm"
1577         FileUtils.mkdir_p vmDir
1578         
1579         frameworkPath = frameworkFromJSCPath($jscPath)
1580         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1581         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1582         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1583
1584         if frameworkPath
1585             source = frameworkPath
1586             destination = Pathname.new(".vm")
1587         elsif $hostOS == "windows"
1588             # Make sure to copy dll along with jsc on Windows
1589             source = [originalJSCPath] + Dir.glob(File.dirname(originalJSCPath) + "/jscLib.dll")
1590             destination = $jscPath.dirname
1591
1592             Dir.chdir($outputDir) {
1593                 FileUtils.mkdir_p destination
1594             }
1595         else
1596             source = originalJSCPath
1597             destination = $jscPath
1598
1599             Dir.chdir($outputDir) {
1600                 FileUtils.mkdir_p $jscPath.dirname
1601             }
1602         end
1603
1604         Dir.chdir($outputDir) {
1605             if $copyVM
1606                 FileUtils.cp_r source, destination
1607             else
1608                 begin 
1609                     FileUtils.ln_s source, destination
1610                 rescue Exception
1611                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1612                     FileUtils.cp_r source, destination
1613                 end
1614             end
1615
1616             if $remote and $hostOS == "linux"
1617                 begin
1618                     dependencies = `ldd #{source}`
1619                     dependencies.split(/\n/).each {
1620                         | dependency |
1621                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1622                     }
1623                 rescue
1624                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1625                 end
1626             end
1627         }
1628     end
1629     
1630     Dir.chdir($outputDir) {
1631         FileUtils.cp_r HELPERS_PATH, ".helpers"
1632     }
1633
1634     ARGV.each {
1635         | collection |
1636         handleCollection(collection)
1637     }
1638
1639     puts
1640 end
1641
1642 def cleanOldResults
1643     raise unless $bundle
1644
1645     eachResultFile($outputDir) {
1646         | path |
1647         FileUtils.rm_f path
1648     }
1649 end
1650
1651 def cleanEmptyResultFiles
1652     eachResultFile($outputDir) {
1653         | path |
1654         next unless path.basename.to_s =~ /\.out$/
1655         next unless FileTest.size(path) == 0
1656         FileUtils.rm_f path
1657     }
1658 end
1659
1660 def eachResultFile(startingDir, &block)
1661     dirsToClean = [startingDir]
1662     until dirsToClean.empty?
1663         nextDir = dirsToClean.pop
1664         Dir.foreach(nextDir) {
1665             | entry |
1666             next if entry =~ /^\./
1667             path = nextDir + entry
1668             if path.directory?
1669                 dirsToClean.push(path)
1670             else
1671                 block.call(path)
1672             end
1673         }
1674     end
1675 end
1676
1677 def prepareTestRunner
1678     raise if $bundle
1679
1680     $runlist.each_with_index {
1681         | plan, index |
1682         plan.index = index
1683     }
1684
1685     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1686     toDelete = []
1687     Dir.foreach($runnerDir) {
1688         | filename |
1689         if filename =~ /^test_/
1690             toDelete << filename
1691         end
1692     }
1693     
1694     toDelete.each {
1695         | filename |
1696         File.unlink($runnerDir + filename)
1697     }
1698
1699     $runlist.each {
1700         | plan |
1701         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1702     }
1703
1704     case $testRunnerType
1705     when :make
1706         prepareMakeTestRunner
1707     when :shell
1708         prepareShellTestRunner
1709     when :ruby
1710         prepareRubyTestRunner
1711     else
1712         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1713     end
1714 end
1715
1716 def cleanRunnerDirectory
1717     raise unless $bundle
1718     Dir.foreach($runnerDir) {
1719         | filename |
1720         next unless filename =~ /^test_fail/
1721         FileUtils.rm_f $runnerDir + filename
1722     }
1723 end
1724
1725 def sshRead(cmd)
1726     raise unless $remote
1727
1728     result = ""
1729     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1730       | inp |
1731       inp.each_line {
1732         | line |
1733         result += line
1734       }
1735     }
1736     raise "#{$?}" unless $?.success?
1737     result
1738 end
1739
1740 def runCommandOnTester(cmd)
1741     if $remote
1742         result = sshRead(cmd)
1743     else
1744         result = `#{cmd}`
1745     end
1746 end
1747
1748 def numberOfProcessors
1749     if $hostOS == "windows"
1750         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1751     else
1752         begin
1753             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1754         rescue
1755             numProcessors = 0
1756         end
1757
1758         if numProcessors == 0
1759             begin
1760                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1761             rescue
1762                 numProcessors == 0
1763             end
1764         end
1765     end
1766
1767     if numProcessors == 0
1768         numProcessors = 1
1769     end
1770     return numProcessors
1771 end
1772
1773 def runAndMonitorTestRunnerCommand(*cmd)
1774     numberOfTests = 0
1775     Dir.chdir($runnerDir) {
1776         # -1 for the runscript, and -2 for '..' and '.'
1777         numberOfTests = Dir.entries(".").count - 3
1778     }
1779     unless $progressMeter
1780         mysys(cmd.join(' '))
1781     else
1782        running = {}
1783        didRun = {}
1784        didFail = {}
1785        blankLine = true
1786        prevStringLength = 0
1787        IO.popen(cmd.join(' '), mode="r") {
1788            | inp |
1789            inp.each_line {
1790                | line |
1791                line = line.scrub.chomp
1792                if line =~ /^Running /
1793                    running[$~.post_match] = true
1794                elsif line =~ /^PASS: /
1795                    didRun[$~.post_match] = true
1796                elsif line =~ /^FAIL: /
1797                    didRun[$~.post_match] = true
1798                    didFail[$~.post_match] = true
1799                else
1800                    unless blankLine
1801                        print("\r" + " " * prevStringLength + "\r")
1802                    end
1803                    puts line
1804                    blankLine = true
1805                end
1806
1807                def lpad(str, chars)
1808                    str = str.to_s
1809                    if str.length > chars
1810                        str
1811                    else
1812                       "%#{chars}s"%(str)
1813                    end
1814                end
1815
1816                string  = ""
1817                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1818                unless didFail.empty?
1819                    string += " (failed #{didFail.size})"
1820                end
1821                string += " "
1822                (running.size - didRun.size).times {
1823                    string += "."
1824                }
1825                if string.length < prevStringLength
1826                    print string
1827                    print(" " * (prevStringLength - string.length))
1828                end
1829                print string
1830                prevStringLength = string.length
1831                blankLine = false
1832                $stdout.flush
1833            }
1834        }
1835        puts
1836        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1837     end
1838 end
1839
1840 def runTestRunner
1841     if $remote
1842         if !$remoteDirectory
1843             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1844         end
1845         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1846         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1847         remoteScript = "\""
1848         remoteScript += "cd #{$remoteDirectory} && "
1849         remoteScript += "rm -rf #{$outputDir.basename} && "
1850         remoteScript += "tar xzf #{$tarFileName} && "
1851         remoteScript += "cd #{$outputDir.basename}/.runner && "
1852         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1853         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1854         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
1855         $envVars.each { |var| remoteScript += "export " << var << "\n" }
1856         remoteScript += "#{testRunnerCommand}\""
1857         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1858     else
1859         Dir.chdir($runnerDir) {
1860             runAndMonitorTestRunnerCommand(testRunnerCommand)
1861         }
1862     end
1863 end
1864
1865 def detectFailures
1866     raise if $bundle
1867
1868     failures = []
1869     
1870     if $remote
1871         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1872         output.split(/\n/).each {
1873             | line |
1874             next unless line =~ /test_fail_/
1875             failures << $~.post_match.to_i
1876         }
1877     else
1878         Dir.foreach($runnerDir) {
1879             | filename |
1880             next unless filename =~ /test_fail_/
1881             failures << $~.post_match.to_i
1882         }
1883     end
1884
1885     failureSet = {}
1886
1887     failures.each {
1888         | failure | 
1889         appendFailure($runlist[failure])
1890         failureSet[failure] = true
1891     }
1892
1893     familyMap = {}
1894     $runlist.each_with_index {
1895         | plan, index |
1896         unless familyMap[plan.family]
1897             familyMap[plan.family] = []
1898         end
1899         if failureSet[index]
1900             appendResult(plan, false)
1901             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
1902             next
1903         else
1904             appendResult(plan, true)
1905             familyMap[plan.family] << {:result => "PASS", :plan => plan};
1906         end
1907         appendPass(plan)
1908     }
1909
1910     File.open($outputDir + "resultsByFamily", "w") {
1911         | outp |
1912         first = true
1913         familyMap.keys.sort.each {
1914             | familyName |
1915             if first
1916                 first = false
1917             else
1918                 outp.puts
1919             end
1920             
1921             outp.print "#{familyName}:"
1922
1923             numPassed = 0
1924             familyMap[familyName].each {
1925                 | entry |
1926                 if entry[:result] == "PASS"
1927                     numPassed += 1
1928                 end
1929             }
1930
1931             if numPassed == familyMap[familyName].size
1932                 outp.puts " PASSED"
1933             elsif numPassed == 0
1934                 outp.puts " FAILED"
1935             else
1936                 outp.puts
1937                 familyMap[familyName].each {
1938                     | entry |
1939                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
1940                 }
1941             end
1942         }
1943     }
1944 end
1945
1946 def compressBundle
1947     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
1948     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1949     raise unless system(cmd)
1950 end
1951
1952 def clean(file)
1953     FileUtils.rm_rf file unless $bundle
1954 end
1955
1956 clean($outputDir + "failed")
1957 clean($outputDir + "passed")
1958 clean($outputDir + "results")
1959 clean($outputDir + "resultsByFamily")
1960 clean($outputDir + ".vm")
1961 clean($outputDir + ".helpers")
1962 clean($outputDir + ".runner")
1963 clean($outputDir + ".tests")
1964 clean($outputDir + "_payload")
1965
1966 Dir.mkdir($outputDir) unless $outputDir.directory?
1967
1968 $outputDir = $outputDir.realpath
1969 $runnerDir = $outputDir + ".runner"
1970
1971 if !$numChildProcesses
1972     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
1973         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
1974     else
1975         $numChildProcesses = numberOfProcessors
1976     end
1977 end
1978
1979 if ENV["JSCTEST_timeout"]
1980     # In the worst case, the processors just interfere with each other.
1981     # Increase the timeout proportionally to the number of processors.
1982     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
1983 end
1984
1985 def runBundle
1986     raise unless $bundle
1987
1988     cleanRunnerDirectory
1989     cleanOldResults
1990     runTestRunner
1991     cleanEmptyResultFiles
1992 end
1993
1994 def runNormal
1995     raise if $bundle or $tarball
1996
1997     prepareBundle
1998     prepareTestRunner
1999     runTestRunner
2000     cleanEmptyResultFiles
2001     detectFailures
2002 end
2003
2004 def runTarball
2005     raise unless $tarball
2006
2007     prepareBundle 
2008     prepareTestRunner
2009     compressBundle
2010 end
2011
2012 def runRemote
2013     raise unless $remote
2014
2015     prepareBundle
2016     prepareTestRunner
2017     compressBundle
2018     runTestRunner
2019     detectFailures
2020 end
2021
2022 puts
2023 if $bundle
2024     runBundle
2025 elsif $remote
2026     runRemote
2027 elsif $tarball
2028     runTarball
2029 else
2030     runNormal
2031 end