WebAssembly: We should only create wrappers for functions that can be exported
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2016 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
47 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
48 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
49 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
50 WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
51 CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
52 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
53 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
54
55 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
56
57 begin
58     require 'shellwords'
59 rescue Exception => e
60     $stderr.puts "Warning: did not find shellwords, not running any tests."
61     exit 0
62 end
63
64 $canRunDisplayProfilerOutput = false
65
66 begin
67     require 'rubygems'
68     require 'json'
69     require 'highline'
70     $canRunDisplayProfilerOutput = true
71 rescue Exception => e
72     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
73     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
74     $stderr.puts "Error: #{e.inspect}"
75 end
76
77 def printCommandArray(*cmd)
78     begin
79         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
80     rescue
81         commandArray = cmd.join(' ')
82     end
83     $stderr.puts ">> #{commandArray}"
84 end
85
86 def mysys(*cmd)
87     printCommandArray(*cmd) if $verbosity >= 1
88     raise "Command failed: #{$?.inspect}" unless system(*cmd)
89 end
90
91 def escapeAll(array)
92     array.map {
93         | v |
94         raise "Detected a non-string in #{inspect}" unless v.is_a? String
95         Shellwords.shellescape(v)
96     }.join(' ')
97 end
98
99
100 $jscPath = nil
101 $doNotMessWithVMPath = false
102 $jitTests = true
103 $memoryLimited = false
104 $outputDir = Pathname.new("results")
105 $verbosity = 0
106 $bundle = nil
107 $tarball = false
108 $tarFileName = "payload.tar.gz"
109 $copyVM = false
110 $testRunnerType = nil
111 $remoteUser = nil
112 $remoteHost = nil
113 $remotePort = nil
114 $remoteDirectory = nil
115 $architecture = nil
116 $hostOS = nil
117 $filter = nil
118 $envVars = []
119 $quickMode = false
120 $buildType = "release"
121 $forceCollectContinuously = false
122
123 def usage
124     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
125     puts
126     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
127     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
128     puts "                            --jsc specifies an already present JavaScriptCore to test."
129     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
130     puts "                            Skip tests tagged with //@skip if $memoryLimited"
131     puts "--no-jit                    Do not run JIT specific tests."
132     puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
133     puts "                            platform."
134     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
135     puts "--verbose            (-v)   Print more things while running."
136     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
137     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
138     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
139     puts "                            e.g. x86, x86_64, arm."
140     puts "--os                        Specify os instead of determining from JavaScriptCore build."
141     puts "                            e.g. darwin, linux & windows."
142     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
143     puts "                            In general the shell runner is slower than the make runner."
144     puts "--make-runner               Uses the faster make-based runner."
145     puts "--remote                    Specify a remote host on which to run tests from command line argument."
146     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
147     puts "--child-processes    (-c)   Specify the number of child processes."
148     puts "--filter                    Only run tests whose name matches the given regular expression."
149     puts "--help               (-h)   Print this message."
150     puts "--env-vars                  Add a list of environment variables to set before running jsc."
151     puts "                            Each environment variable should be separated by a space."
152     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
153     puts "                            it will override the default value."
154     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
155     exit 1
156 end
157
158 jscArg = nil
159
160 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
161                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
162                ['--no-copy', GetoptLong::NO_ARGUMENT],
163                ['--memory-limited', GetoptLong::NO_ARGUMENT],
164                ['--no-jit', GetoptLong::NO_ARGUMENT],
165                ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
166                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
167                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
168                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
169                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
170                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
171                ['--os', GetoptLong::REQUIRED_ARGUMENT],
172                ['--shell-runner', GetoptLong::NO_ARGUMENT],
173                ['--make-runner', GetoptLong::NO_ARGUMENT],
174                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
175                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
176                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
177                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
178                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
179                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
180                ['--debug', GetoptLong::NO_ARGUMENT],
181                ['--release', GetoptLong::NO_ARGUMENT],
182                ['--quick', '-q', GetoptLong::NO_ARGUMENT]).each {
183     | opt, arg |
184     case opt
185     when '--help'
186         usage
187     when '--jsc'
188         jscArg = arg
189     when '--no-copy'
190         $doNotMessWithVMPath = true
191     when '--output-dir'
192         $outputDir = Pathname.new(arg)
193     when '--memory-limited'
194         $memoryLimited = true
195     when '--no-jit'
196         $jitTests = false
197     when '--force-collectContinuously'
198         $forceCollectContinuously = true;
199     when '--verbose'
200         $verbosity += 1
201     when '--run-bundle'
202         $bundle = Pathname.new(arg)
203     when '--tarball'
204         $tarball = true
205         $copyVM = true
206         $tarFileName = arg unless arg == ''
207     when '--force-vm-copy'
208         $copyVM = true
209     when '--shell-runner'
210         $testRunnerType = :shell
211     when '--make-runner'
212         $testRunnerType = :make
213     when '--remote'
214         $copyVM = true
215         $tarball = true
216         $remote = true
217         uri = URI("ssh://" + arg)
218         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
219     when '--remote-config-file'
220         $remoteConfigFile = arg
221     when '--child-processes'
222         $numChildProcesses = arg.to_i
223     when '--filter'
224         $filter = Regexp.new(arg)
225     when '--arch'
226         $architecture = arg
227     when '--os'
228         $hostOS = arg
229     when '--env-vars'
230         $envVars = arg.gsub(/\s+/, ' ').split(' ')
231     when '--quick'
232         $quickMode = true
233     when '--debug'
234         $buildType = "debug"
235     when '--release'
236         $buildType = "release"
237     end
238 }
239
240 if $remoteConfigFile
241     file = File.read($remoteConfigFile)
242     config = JSON.parse(file)
243
244     if !$remote and config['remote']
245         $copyVM = true
246         $tarball = true
247         $remote = true
248         uri = URI("ssh://" + config['remote'])
249         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
250     end
251
252     if config['remoteDirectory']
253         $remoteDirectory = config['remoteDirectory']
254     end
255 end
256
257 unless jscArg
258     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
259     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
260     command += ($buildType == "release") ? " --release" : " --debug"
261     command += " --executablePath"
262
263     output = `#{command}`.split("\n")
264     if !output.length
265         $stderr.puts "Error: must specify --jsc <path>"
266         exit 1
267     end
268
269     output = output[0]
270     jscArg = Pathname.new(output).join("jsc")
271     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
272     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
273     if !File.file?(jscArg)
274         $stderr.puts "Error: must specify --jsc <path>"
275         exit 1
276     end
277
278     puts "Using the following jsc path: #{jscArg}"
279 end
280
281 if $doNotMessWithVMPath
282     $jscPath = Pathname.new(jscArg)
283 else
284     $jscPath = Pathname.new(jscArg).realpath
285 end
286
287 $progressMeter = ($verbosity == 0 and $stdout.tty?)
288
289 if $bundle
290     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
291     $outputDir = $bundle
292 end
293
294 # Try to determine architecture. Return nil on failure.
295 def machOArchitectureCode
296     begin 
297         otoolLines = `otool -afh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
298         otoolLines.each_with_index {
299             | value, index |
300             if value =~ /magic/ and value =~ /cputype/
301                 return otoolLines[index + 1].split[1].to_i
302             end
303         }
304     rescue
305         $stderr.puts "Warning: unable to execute otool."
306     end
307     $stderr.puts "Warning: unable to determine architecture."
308     nil
309 end
310
311 def determineArchitectureFromMachOBinary
312     code = machOArchitectureCode
313     return nil unless code
314     is64BitFlag = 0x01000000
315     case code
316     when 7
317         "x86"
318     when 7 | is64BitFlag
319         "x86-64"
320     when 12
321         "arm"
322     when 12 | is64BitFlag
323         "arm64"
324     else
325         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
326         nil
327     end
328 end
329
330 def determineArchitectureFromELFBinary
331     f = File.open($jscPath.to_s)
332     data = f.read(19)
333
334     if !(data[0,4] == "\x7F\x45\x4C\x46")
335         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
336         return nil
337     end
338
339     code = data[18].ord
340     case code
341     when 3
342         "x86"
343     when 8
344         "mips"
345     when 62
346         "x86-64"
347     when 40
348         "arm"
349     when 183
350         "arm64"
351     else
352         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
353         nil
354     end
355 end
356
357 def determineArchitectureFromPEBinary
358     f = File.open($jscPath.to_s)
359     data = f.read(1024)
360
361     if !(data[0, 2] == "MZ")
362         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
363         return nil
364     end
365
366     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
367
368     if !(data[peHeaderAddr, 4] == "PE\0\0")
369         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
370         return nil
371     end
372
373     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
374
375     case machine
376     when 0x014c
377         "x86"
378     when 0x8664
379         "x86-64"
380     else
381         $stderr.puts "Warning: unsupported machine type: #{machine}"
382         nil
383     end
384 end
385
386 def determineArchitecture
387     case $hostOS
388     when "darwin"
389         determineArchitectureFromMachOBinary
390     when "linux"
391         determineArchitectureFromELFBinary
392     when "windows"
393         determineArchitectureFromPEBinary
394     else
395         $stderr.puts "Warning: unable to determine architecture on this platform."
396         nil
397     end
398 end
399
400 def determineOS
401     case RbConfig::CONFIG["host_os"]
402     when /darwin/i
403         "darwin"
404     when /linux/i
405         "linux"
406     when /mswin|mingw|cygwin/
407         "windows"
408     else
409         $stderr.puts "Warning: unable to determine host operating system"
410         nil
411     end
412 end
413
414 $hostOS = determineOS unless $hostOS
415 $architecture = determineArchitecture unless $architecture
416 $isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $architecture == "mips" || $hostOS == "windows")
417
418 if !$testRunnerType
419     if $remote and $hostOS == "darwin"
420         $testRunnerType = :shell
421     else
422         $testRunnerType = :make
423     end
424 end
425
426 $numFailures = 0
427 $numPasses = 0
428
429 # We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
430 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--maxPerThreadStackUsage=1572864"]
431 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
432 # NOTE: Tests rely on this using scribbleFreeCells.
433 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
434 B3O1_OPTIONS = ["--defaultB3OptLevel=1"]
435 FTL_OPTIONS = ["--useFTLJIT=true"]
436
437 def shouldCollectContinuously?
438     $buildType == "release" or $forceCollectContinuously
439 end
440
441 COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false"] : []
442
443 $runlist = []
444
445 def frameworkFromJSCPath(jscPath)
446     parentDirectory = jscPath.dirname
447     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
448         parentDirectory.dirname
449     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
450         jscPath.dirname + "JavaScriptCore.framework"
451     else
452         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
453         nil
454     end
455 end
456
457 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
458     dir = Pathname.new(".")
459     $benchmarkDirectory.each_filename {
460         | pathComponent |
461         dir += ".."
462     }
463     dir + resourcePath
464 end
465
466 def pathToVM
467     pathToBundleResourceFromBenchmarkDirectory($jscPath)
468 end
469
470 def pathToHelpers
471     pathToBundleResourceFromBenchmarkDirectory(".helpers")
472 end
473
474 def prefixCommand(prefix)
475     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
476 end
477
478 def redirectAndPrefixCommand(prefix)
479     prefixCommand(prefix) + " 2>&1"
480 end
481
482 def pipeAndPrefixCommand(outputFilename, prefix)
483     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
484 end
485
486 # Output handler for tests that are expected to be silent.
487 def silentOutputHandler
488     Proc.new {
489         | name |
490         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
491     }
492 end
493
494 # Output handler for tests that are expected to produce meaningful output.
495 def noisyOutputHandler
496     Proc.new {
497         | name |
498         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
499     }
500 end
501
502 # Error handler for tests that fail exactly when they return non-zero exit status.
503 # This is useful when a test is expected to fail.
504 def simpleErrorHandler
505     Proc.new {
506         | outp, plan |
507         outp.puts "if test -e #{plan.failFile}"
508         outp.puts "then"
509         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
510         outp.puts "    " + plan.failCommand
511         outp.puts "else"
512         outp.puts "    " + plan.successCommand
513         outp.puts "fi"
514     }
515 end
516
517 # Error handler for tests that fail exactly when they return zero exit status.
518 def expectedFailErrorHandler
519     Proc.new {
520         | outp, plan |
521         outp.puts "if test -e #{plan.failFile}"
522         outp.puts "then"
523         outp.puts "    " + plan.successCommand
524         outp.puts "else"
525         outp.puts "    (echo ERROR: Unexpected exit code: 0) | " + redirectAndPrefixCommand(plan.name)
526         outp.puts "    " + plan.failCommand
527         outp.puts "fi"
528     }
529 end
530
531 # Error handler for tests that fail exactly when they return non-zero exit status and produce
532 # lots of spew. This will echo that spew when the test fails.
533 def noisyErrorHandler
534     Proc.new {
535         | outp, plan |
536         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
537     
538         outp.puts "if test -e #{plan.failFile}"
539         outp.puts "then"
540         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
541         outp.puts "    " + plan.failCommand
542         outp.puts "else"
543         outp.puts "    " + plan.successCommand
544         outp.puts "fi"
545     }
546 end
547
548 # Error handler for tests that diff their output with some expectation.
549 def diffErrorHandler(expectedFilename)
550     Proc.new {
551         | outp, plan |
552         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
553         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
554         
555         outp.puts "if test -e #{plan.failFile}"
556         outp.puts "then"
557         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
558         outp.puts "    " + plan.failCommand
559         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
560         outp.puts "then"
561         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
562         outp.puts "    if [ $? -eq 0 ]"
563         outp.puts "    then"
564         outp.puts "    " + plan.successCommand
565         outp.puts "    else"
566         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
567         outp.puts "        " + plan.failCommand
568         outp.puts "    fi"
569         outp.puts "else"
570         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
571         outp.puts "    " + plan.failCommand
572         outp.puts "fi"
573     }
574 end
575
576 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
577 # tests.
578 def mozillaErrorHandler
579     Proc.new {
580         | outp, plan |
581         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
582
583         outp.puts "if test -e #{plan.failFile}"
584         outp.puts "then"
585         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
586         outp.puts "    " + plan.failCommand
587         outp.puts "elif grep -i -q failed! #{outputFilename}"
588         outp.puts "then"
589         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
590         outp.puts "    " + plan.failCommand
591         outp.puts "else"
592         outp.puts "    " + plan.successCommand
593         outp.puts "fi"
594     }
595 end
596
597 # Error handler for tests that report error by saying "failed!", and are expected to
598 # fail. This is used by Mozilla tests.
599 def mozillaFailErrorHandler
600     Proc.new {
601         | outp, plan |
602         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
603
604         outp.puts "if test -e #{plan.failFile}"
605         outp.puts "then"
606         outp.puts "    " + plan.successCommand
607         outp.puts "elif grep -i -q failed! #{outputFilename}"
608         outp.puts "then"
609         outp.puts "    " + plan.successCommand
610         outp.puts "else"
611         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
612         outp.puts "    " + plan.failCommand
613         outp.puts "fi"
614     }
615 end
616
617 # Error handler for tests that report error by saying "failed!", and are expected to have
618 # an exit code of 3.
619 def mozillaExit3ErrorHandler
620     Proc.new {
621         | outp, plan |
622         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
623
624         outp.puts "if test -e #{plan.failFile}"
625         outp.puts "then"
626         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
627         outp.puts "    then"
628         outp.puts "        if grep -i -q failed! #{outputFilename}"
629         outp.puts "        then"
630         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
631         outp.puts "            " + plan.failCommand
632         outp.puts "        else"
633         outp.puts "            " + plan.successCommand
634         outp.puts "        fi"
635         outp.puts "    else"
636         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
637         outp.puts "        " + plan.failCommand
638         outp.puts "    fi"
639         outp.puts "else"
640         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
641         outp.puts "    " + plan.failCommand
642         outp.puts "fi"
643     }
644 end
645
646 # Error handler for tests that report success by saying "Passed" or error by saying "FAILED".
647 # This is used by Chakra tests.
648 def chakraPassFailErrorHandler
649     Proc.new {
650         | outp, plan |
651         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
652
653         outp.puts "if test -e #{plan.failFile}"
654         outp.puts "then"
655         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
656         outp.puts "    " + plan.failCommand
657         outp.puts "elif grep -i -q FAILED #{outputFilename}"
658         outp.puts "then"
659         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
660         outp.puts "    " + plan.failCommand
661         outp.puts "else"
662         outp.puts "    " + plan.successCommand
663         outp.puts "fi"
664     }
665 end
666
667 $runCommandOptions = {}
668
669 class Plan
670     attr_reader :directory, :arguments, :family, :name, :outputHandler, :errorHandler
671     attr_accessor :index
672     
673     def initialize(directory, arguments, family, name, outputHandler, errorHandler)
674         @directory = directory
675         @arguments = arguments
676         @family = family
677         @name = name
678         @outputHandler = outputHandler
679         @errorHandler = errorHandler
680         @isSlow = !!$runCommandOptions[:isSlow]
681     end
682     
683     def shellCommand
684         # It's important to remember that the test is actually run in a subshell, so if we change directory
685         # in the subshell when we return we will be in our original directory. This is nice because we don't
686         # have to bend over backwards to do things relative to the root.
687         script = "(cd ../#{Shellwords.shellescape(@directory.to_s)} && ("
688         $envVars.each { |var| script += "export " << var << "; " }
689         script += "\"$@\" " + escapeAll(@arguments) + "))"
690         return script
691     end
692     
693     def reproScriptCommand
694         # We have to find our way back to the .runner directory since that's where all of the relative
695         # paths assume they start out from.
696         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
697         script += "cd $CURRENT_DIR\n"
698         Pathname.new(@name).dirname.each_filename {
699             | pathComponent |
700             script += "cd ..\n"
701         }
702         script += "cd .runner\n"
703
704         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
705         script += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])}\n"
706         $envVars.each { |var| script += "export " << var << "\n" }
707         script += "#{shellCommand} || exit 1"
708         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
709     end
710     
711     def failCommand
712         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
713     end
714     
715     def successCommand
716         if $progressMeter or $verbosity >= 2
717             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
718         else
719             "rm -f #{failFile}"
720         end
721     end
722     
723     def failFile
724         "test_fail_#{@index}"
725     end
726     
727     def writeRunScript(filename)
728         File.open(filename, "w") {
729             | outp |
730             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
731             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
732             cmd += @outputHandler.call(@name)
733             if $verbosity >= 3
734                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
735             end
736             outp.puts cmd
737             @errorHandler.call(outp, self)
738         }
739     end
740 end
741
742 $uniqueFilenameCounter = 0
743 def uniqueFilename(extension)
744     payloadDir = $outputDir + "_payload"
745     Dir.mkdir payloadDir unless payloadDir.directory?
746     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
747     $uniqueFilenameCounter += 1
748     result
749 end
750
751 def baseOutputName(kind)
752     "#{$collectionName}/#{$benchmark}.#{kind}"
753 end
754
755 def addRunCommand(kind, command, outputHandler, errorHandler)
756     $didAddRunCommand = true
757     name = baseOutputName(kind)
758     if $filter and name !~ $filter
759         return
760     end
761     plan = Plan.new(
762         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
763         errorHandler)
764     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
765         $runlist.unshift plan
766     else
767         $runlist << plan
768     end
769 end
770
771 # Returns true if there were run commands found in the file ($benchmarkDirectory +
772 # $benchmark), in which case those run commands have already been executed. Otherwise
773 # returns false, in which case you're supposed to add your own run commands.
774 def parseRunCommands
775     oldDidAddRunCommand = $didAddRunCommand
776     $didAddRunCommand = false
777
778     Dir.chdir($outputDir) {
779         File.open($benchmarkDirectory + $benchmark) {
780             | inp |
781             inp.each_line {
782                 | line |
783                 begin
784                     doesMatch = line =~ /^\/\/@/
785                 rescue Exception => e
786                     # Apparently this happens in the case of some UTF8 stuff in some files, where
787                     # Ruby tries to be strict and throw exceptions.
788                     next
789                 end
790                 next unless doesMatch
791                 eval $~.post_match
792             }
793         }
794     }
795
796     result = $didAddRunCommand
797     $didAddRunCommand = result or oldDidAddRunCommand
798     result
799 end
800
801 def slow!
802     $runCommandOptions[:isSlow] = true
803 end
804
805 def runWithOutputHandler(kind, outputHandler, *options)
806     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
807 end
808
809 def run(kind, *options)
810     runWithOutputHandler(kind, silentOutputHandler, *options)
811 end
812
813 def runNoFTL(*optionalTestSpecificOptions)
814     run("no-ftl", *optionalTestSpecificOptions)
815 end
816
817 def runWithRAMSize(size, *optionalTestSpecificOptions)
818     run("ram-size-#{size}", "--forceRAMSize=#{size}", *optionalTestSpecificOptions)
819 end
820
821 def runOneLargeHeap(*optionalTestSpecificOptions)
822     if $memoryLimited
823         $didAddRunCommand = true
824         puts "Skipping #{$collectionName}/#{$benchmark}"
825     else
826         run("default", *optionalTestSpecificOptions)
827     end
828 end
829
830 def runNoJIT(*optionalTestSpecificOptions)
831     run("no-jit", "--useJIT=false", *optionalTestSpecificOptions)
832 end
833
834 def runNoLLInt(*optionalTestSpecificOptions)
835     if $jitTests
836         run("no-llint", "--useLLInt=false", *optionalTestSpecificOptions)
837     end
838 end
839
840 # NOTE: Tests rely on this using scribbleFreeCells.
841 def runNoCJITValidate(*optionalTestSpecificOptions)
842     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
843 end
844
845 def runNoCJITValidatePhases(*optionalTestSpecificOptions)
846     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
847 end
848
849 def runNoCJITCollectContinuously(*optionalTestSpecificOptions)
850     run("no-cjit-collect-continuously", *(NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
851 end
852
853 def runDefault(*optionalTestSpecificOptions)
854     run("default", *(FTL_OPTIONS + optionalTestSpecificOptions))
855 end
856
857 def runFTLNoCJIT(*optionalTestSpecificOptions)
858     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
859 end
860
861 def runFTLNoCJITB3O1(*optionalTestSpecificOptions)
862     run("ftl-no-cjit-b3o1", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
863 end
864
865 def runFTLNoCJITValidate(*optionalTestSpecificOptions)
866     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
867 end
868
869 def runFTLNoCJITNoPutStackValidate(*optionalTestSpecificOptions)
870     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
871 end
872
873 def runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
874     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
875 end
876
877 def runFTLNoCJITOSRValidation(*optionalTestSpecificOptions)
878     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
879 end
880
881 def runDFGEager(*optionalTestSpecificOptions)
882     run("dfg-eager", *(EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
883 end
884
885 def runDFGEagerNoCJITValidate(*optionalTestSpecificOptions)
886     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
887 end
888
889 def runFTLEager(*optionalTestSpecificOptions)
890     run("ftl-eager", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
891 end
892
893 def runFTLEagerNoCJITValidate(*optionalTestSpecificOptions)
894     run("ftl-eager-no-cjit", "--validateGraph=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
895 end
896
897 def runFTLEagerNoCJITB3O1(*optionalTestSpecificOptions)
898     run("ftl-eager-no-cjit-b3o1", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
899 end
900
901 def runFTLEagerNoCJITOSRValidation(*optionalTestSpecificOptions)
902     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
903 end
904
905 def runNoCJITNoASO(*optionalTestSpecificOptions)
906     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
907 end
908
909 def runNoCJITNoAccessInlining(*optionalTestSpecificOptions)
910     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
911 end
912
913 def runFTLNoCJITNoAccessInlining(*optionalTestSpecificOptions)
914     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
915 end
916
917 def runFTLNoCJITSmallPool(*optionalTestSpecificOptions)
918     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
919 end
920
921 def runNoCJIT(*optionalTestSpecificOptions)
922     run("no-cjit", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
923 end
924
925 def runDFGMaximalFlushPhase(*optionalTestSpecificOptions)
926     run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
927 end
928
929 def runShadowChicken(*optionalTestSpecificOptions)
930     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true", *optionalTestSpecificOptions)
931 end
932
933 def defaultRun
934     if $quickMode
935         defaultQuickRun
936     else
937         runDefault
938         if $jitTests
939             runNoLLInt
940             runNoCJITValidatePhases
941             runNoCJITCollectContinuously if shouldCollectContinuously?
942             runDFGEager
943             runDFGEagerNoCJITValidate
944             runDFGMaximalFlushPhase
945
946             return if !$isFTLPlatform
947
948             runNoFTL
949             runFTLNoCJITValidate
950             runFTLNoCJITB3O1
951             runFTLNoCJITNoPutStackValidate
952             runFTLNoCJITNoInlineValidate
953             runFTLEager
954             runFTLEagerNoCJITValidate
955             runFTLEagerNoCJITB3O1
956             runFTLNoCJITSmallPool
957         end
958     end
959 end
960
961 def defaultNoNoLLIntRun
962     if $quickMode
963         defaultQuickRun
964     else
965         runDefault
966         if $jitTests
967             runNoCJITValidatePhases
968             runNoCJITCollectContinuously if shouldCollectContinuously?
969             runDFGEager
970             runDFGEagerNoCJITValidate
971             runDFGMaximalFlushPhase
972
973             return if !$isFTLPlatform
974
975             runNoFTL
976             runFTLNoCJITValidate
977             runFTLNoCJITB3O1
978             runFTLNoCJITNoPutStackValidate
979             runFTLNoCJITNoInlineValidate
980             runFTLEager
981             runFTLEagerNoCJITValidate
982             runFTLNoCJITSmallPool
983         end
984     end
985 end
986
987 def defaultQuickRun
988     runDefault
989     if $jitTests
990         runNoCJITValidate
991
992         return if $isFTLPlatform
993
994         runNoFTL
995         runFTLNoCJITValidate
996     end
997 end
998
999 def defaultSpotCheckNoMaximalFlush
1000     defaultQuickRun
1001     runNoCJITNoAccessInlining
1002
1003     return if !$isFTLPlatform
1004
1005     runFTLNoCJITOSRValidation
1006     runFTLNoCJITNoAccessInlining
1007     runFTLNoCJITB3O1
1008 end
1009
1010 def defaultSpotCheck
1011     defaultSpotCheckNoMaximalFlush
1012     runDFGMaximalFlushPhase
1013 end
1014
1015 # This is expected to not do eager runs because eager runs can have a lot of recompilations
1016 # for reasons that don't arise in the real world. It's used for tests that assert convergence
1017 # by counting recompilations.
1018 def defaultNoEagerRun
1019     runDefault
1020     if $jitTests
1021         runNoLLInt
1022         runNoCJITValidatePhases
1023         runNoCJITCollectContinuously if shouldCollectContinuously?
1024
1025         return if !$isFTLPlatform
1026
1027         runNoFTL
1028         runFTLNoCJITValidate
1029         runFTLNoCJITNoInlineValidate
1030         runFTLNoCJITB3O1
1031     end
1032 end
1033
1034 def defaultNoSamplingProfilerRun
1035     runDefault
1036     if $jitTests
1037         runNoLLInt
1038         runNoCJITValidatePhases
1039         runNoCJITCollectContinuously if shouldCollectContinuously?
1040         runDFGEager
1041         runDFGEagerNoCJITValidate
1042         runDFGMaximalFlushPhase
1043
1044         return if !$isFTLPlatform
1045
1046         runNoFTL
1047         runFTLNoCJITNoPutStackValidate
1048         runFTLNoCJITNoInlineValidate
1049         runFTLEager
1050         runFTLEagerNoCJITValidate
1051         runFTLNoCJITSmallPool
1052     end
1053 end
1054
1055 def runProfiler
1056     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
1057         skip
1058         return
1059     end
1060
1061     profilerOutput = uniqueFilename(".json")
1062     if $canRunDisplayProfilerOutput
1063         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
1064     else
1065         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
1066         run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
1067     end
1068 end
1069
1070 def runExceptionFuzz
1071     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
1072     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
1073 end
1074
1075 def runExecutableAllocationFuzz(name, *options)
1076     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
1077     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
1078 end
1079
1080 def runTypeProfiler
1081     if !$jitTests
1082         return
1083     end
1084
1085     return if !$isFTLPlatform
1086
1087     run("ftl-no-cjit-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1088     run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
1089 end
1090
1091 def runControlFlowProfiler
1092     if !$jitTests
1093         return
1094     end
1095
1096     return if !$isFTLPlatform
1097
1098     run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1099 end
1100
1101 def runTest262(mode, exception, includeFiles, flags)
1102     failsWithException = exception != "NoException"
1103     isStrict = false
1104     isModule = false
1105     isAsync = false
1106
1107     flags.each {
1108         | flag |
1109         case flag
1110         when :strict
1111             isStrict = true
1112         when :module
1113             isModule = true
1114         when :async
1115             isAsync = true
1116         else
1117             raise "Invalid flag for runTest262, #{flag}"
1118         end
1119     }
1120
1121     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
1122
1123     args = [pathToVM.to_s] + BASE_OPTIONS
1124     args << "--exception=" + exception if failsWithException
1125     args << "--test262-async" if isAsync
1126     args += includeFiles
1127
1128     case mode
1129     when :normal
1130         errorHandler = simpleErrorHandler
1131         outputHandler = silentOutputHandler
1132     when :fail
1133         errorHandler = expectedFailErrorHandler
1134         outputHandler = noisyOutputHandler
1135     when :failDueToOutdatedOrBadTest
1136         errorHandler = expectedFailErrorHandler
1137         outputHandler = noisyOutputHandler
1138     else
1139         raise "Invalid mode: #{mode}"
1140     end
1141
1142     if isStrict
1143         kind = "default-strict"
1144         args << "--strict-file=#{$benchmark}"
1145     else
1146         kind = "default"
1147         if isModule
1148             args << "--module-file=#{$benchmark}"
1149         else
1150             args << $benchmark.to_s
1151         end
1152     end
1153
1154     addRunCommand(kind, args, outputHandler, errorHandler)
1155 end
1156
1157 def prepareTest262Fixture
1158     # This function is used to add the files used by Test262 modules tests.
1159     prepareExtraRelativeFiles([""], $collection)
1160 end
1161
1162 def runES6(mode)
1163     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
1164     case mode
1165     when :normal
1166         errorHandler = simpleErrorHandler
1167     when :fail
1168         errorHandler = expectedFailErrorHandler
1169     else
1170         raise "Invalid mode: #{mode}"
1171     end
1172     addRunCommand("default", args, noisyOutputHandler, errorHandler)
1173 end
1174
1175 def runModules
1176     run("default-modules", "-m")
1177
1178     if !$jitTests
1179         return
1180     end
1181
1182     run("no-llint-modules", "-m", "--useLLInt=false")
1183     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
1184     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
1185     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1186
1187     return if !$isFTLPlatform
1188
1189     run("default-ftl-modules", "-m", *FTL_OPTIONS)
1190     run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1191     run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1192     run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
1193     run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1194     run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1195 end
1196
1197 def runWebAssembly
1198     return if !$jitTests
1199     return if !$isFTLPlatform
1200     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1201     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1202     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1203     run("default-wasm", "-m", *FTL_OPTIONS)
1204     if !$quickMode
1205         run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1206         run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1207         run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1208         run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1209     end
1210 end
1211
1212 def runWebAssemblyEmscripten(mode)
1213     case mode
1214     when :skip
1215         return
1216     end
1217     return if !$jitTests
1218     return if !$isFTLPlatform
1219     wasm = $benchmark.to_s.sub! '.js', '.wasm'
1220     prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
1221     run("default-wasm", *FTL_OPTIONS)
1222     if !$quickMode
1223         run("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1224         run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1225         run("wasm-no-call-ic", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1226         run("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1227     end
1228 end
1229
1230 def runWebAssemblySpecTest(mode)
1231     case mode
1232     when :skip
1233         return
1234     end
1235
1236     return if !$jitTests
1237     return if !$isFTLPlatform
1238     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1239
1240     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1241     prepareExtraRelativeFiles(modules.map { |f| "../../" + f }, $collection)
1242
1243     harness = Dir[WASMTESTS_PATH + "spec-harness/" + "*.js"].map { |f| File.basename(f) }
1244     prepareExtraRelativeFiles(harness.map { |f| "../../spec-harness/" + f }, $collection)
1245
1246     runWithOutputHandler("default-wasm", noisyOutputHandler, "../spec-harness.js", *FTL_OPTIONS)
1247     runWithOutputHandler("wasm-no-cjit-yes-tls-context", noisyOutputHandler, "../spec-harness.js",  "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1248     runWithOutputHandler("wasm-eager-jettison", noisyOutputHandler, "../spec-harness.js", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1249     runWithOutputHandler("wasm-no-call-ic", noisyOutputHandler, "../spec-harness.js", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1250     runWithOutputHandler("wasm-no-tls-context", noisyOutputHandler, "../spec-harness.js", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1251 end
1252
1253 def runChakra(mode, exception, baselineFile, extraFiles)
1254     raise unless $benchmark.to_s =~ /\.js$/
1255     failsWithException = exception != "NoException"
1256     testName = $~.pre_match
1257
1258     prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
1259     prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)
1260
1261     args = [pathToVM.to_s] + BASE_OPTIONS
1262     args += FTL_OPTIONS if $isFTLPlatform
1263     args += EAGER_OPTIONS
1264     args << "--exception=" + exception if failsWithException
1265     args << "--dumpException" if failsWithException
1266     args += ["jsc-lib.js"]
1267
1268     case mode
1269     when :baseline
1270         prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
1271         errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
1272         outputHandler = noisyOutputHandler
1273     when :pass
1274         errorHandler = chakraPassFailErrorHandler
1275         outputHandler = noisyOutputHandler
1276     when :skip
1277         return
1278     else
1279         raise "Invalid mode: #{mode}"
1280     end
1281
1282     kind = "default"
1283     args << $benchmark.to_s
1284
1285     addRunCommand(kind, args, outputHandler, errorHandler)
1286 end
1287
1288 def runLayoutTest(kind, *options)
1289     raise unless $benchmark.to_s =~ /\.js$/
1290     testName = $~.pre_match
1291     if kind
1292         kind = "layout-" + kind
1293     else
1294         kind = "layout"
1295     end
1296
1297     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1298     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1299
1300     args = [pathToVM.to_s] + BASE_OPTIONS + options +
1301         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1302          $benchmark.to_s,
1303          (Pathname.new("resources") + "standalone-post.js").to_s]
1304     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1305 end
1306
1307 def runLayoutTestNoFTL
1308     runLayoutTest("no-ftl")
1309 end
1310
1311 def runLayoutTestNoLLInt
1312     runLayoutTest("no-llint", "--useLLInt=false")
1313 end
1314
1315 def runLayoutTestNoCJIT
1316     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1317 end
1318
1319 def runLayoutTestDFGEagerNoCJIT
1320     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1321 end
1322
1323 def runLayoutTestDefault
1324     runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
1325 end
1326
1327 def runLayoutTestFTLNoCJIT
1328     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1329 end
1330
1331 def runLayoutTestFTLEagerNoCJIT
1332     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1333 end
1334
1335 def runLayoutTestFTLEagerNoCJITB3O1
1336     runLayoutTest("ftl-eager-no-cjit-b3o1", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS))
1337 end
1338
1339 def noFTLRunLayoutTest
1340     if !$jitTests
1341         return
1342     end
1343
1344     runLayoutTestNoLLInt
1345     runLayoutTestNoCJIT
1346     runLayoutTestDFGEagerNoCJIT
1347 end
1348
1349 def defaultQuickRunLayoutTest
1350     runLayoutTestDefault
1351     if $jitTests
1352         if $isFTLPlatform
1353             runLayoutTestNoFTL
1354             runLayoutTestFTLNoCJIT
1355             runLayoutTestFTLEagerNoCJIT
1356         else
1357             noFTLRunLayoutTest
1358         end
1359     end
1360 end
1361
1362 def defaultRunLayoutTest
1363     if $quickMode
1364         defaultQuickRunLayoutTest
1365     else
1366         runLayoutTestDefault
1367         if $jitTests
1368             noFTLRunLayoutTest
1369
1370             return if !$isFTLPlatform
1371
1372             runLayoutTestNoFTL
1373             runLayoutTestFTLNoCJIT
1374             runLayoutTestFTLEagerNoCJIT
1375         end
1376     end
1377 end
1378
1379 def noEagerNoNoLLIntTestsRunLayoutTest
1380     runLayoutTestDefault
1381     if $jitTests
1382         runLayoutTestNoCJIT
1383
1384         return if !$isFTLPlatform
1385
1386         runLayoutTestNoFTL
1387         runLayoutTestFTLNoCJIT
1388     end
1389 end
1390
1391 def noNoLLIntRunLayoutTest
1392     runLayoutTestDefault
1393     if $jitTests
1394         runLayoutTestNoCJIT
1395         runLayoutTestDFGEagerNoCJIT
1396
1397         return if !$isFTLPlatform
1398
1399         runLayoutTestNoFTL
1400         runLayoutTestFTLNoCJIT
1401         runLayoutTestFTLEagerNoCJIT
1402     end
1403 end
1404
1405 def prepareExtraRelativeFiles(extraFiles, destination)
1406     Dir.chdir($outputDir) {
1407         extraFiles.each {
1408             | file |
1409             dest = destination + file
1410             FileUtils.mkdir_p(dest.dirname)
1411             FileUtils.cp $extraFilesBaseDir + file, dest
1412         }
1413     }
1414 end
1415
1416 def baseDirForCollection(collectionName)
1417     Pathname(".tests") + collectionName
1418 end
1419
1420 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1421     raise unless absoluteBase.absolute?
1422     Dir.chdir($outputDir) {
1423         collectionBaseDir = baseDirForCollection($collectionName)
1424         extraFiles.each {
1425             | file |
1426             destination = collectionBaseDir + file
1427             FileUtils.mkdir_p destination.dirname unless destination.directory?
1428             FileUtils.cp absoluteBase + file, destination
1429         }
1430     }
1431 end
1432
1433 def runMozillaTest(kind, mode, extraFiles, *options)
1434     if kind
1435         kind = "mozilla-" + kind
1436     else
1437         kind = "mozilla"
1438     end
1439     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1440     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1441     case mode
1442     when :normal
1443         errorHandler = mozillaErrorHandler
1444     when :negative
1445         errorHandler = mozillaExit3ErrorHandler
1446     when :fail
1447         errorHandler = mozillaFailErrorHandler
1448     when :skip
1449         return
1450     else
1451         raise "Invalid mode: #{mode}"
1452     end
1453     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1454 end
1455
1456 def runMozillaTestDefault(mode, *extraFiles)
1457     runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
1458 end
1459
1460 def runMozillaTestNoFTL(mode, *extraFiles)
1461     runMozillaTest("no-ftl", mode, extraFiles)
1462 end
1463
1464 def runMozillaTestLLInt(mode, *extraFiles)
1465     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1466 end
1467
1468 def runMozillaTestBaselineJIT(mode, *extraFiles)
1469     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1470 end
1471
1472 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1473     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1474 end
1475
1476 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1477     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1478 end
1479
1480 def defaultQuickRunMozillaTest(mode, *extraFiles)
1481     if $jitTests
1482         runMozillaTestDefault(mode, *extraFiles)
1483         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1484     else
1485         runMozillaTestNoFTL(mode, *extraFiles)
1486         if $jitTests
1487             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1488         end
1489     end
1490 end
1491
1492 def defaultRunMozillaTest(mode, *extraFiles)
1493     if $quickMode
1494         defaultQuickRunMozillaTest(mode, *extraFiles)
1495     else
1496         runMozillaTestNoFTL(mode, *extraFiles)
1497         if $jitTests
1498             runMozillaTestLLInt(mode, *extraFiles)
1499             runMozillaTestBaselineJIT(mode, *extraFiles)
1500             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1501             runMozillaTestDefault(mode, *extraFiles)
1502             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
1503         end
1504     end
1505 end
1506
1507 def runNoisyTest(kind, *options)
1508     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
1509 end
1510
1511 def runNoisyTestDefault
1512     runNoisyTest("default", *FTL_OPTIONS)
1513 end
1514
1515 def runNoisyTestNoFTL
1516     runNoisyTest("no-ftl")
1517 end
1518
1519 def runNoisyTestNoCJIT
1520     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1521 end
1522
1523 def runNoisyTestNoCJITB3O1
1524     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS))
1525 end
1526
1527 def runNoisyTestEagerNoCJIT
1528     runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1529 end
1530
1531 def defaultRunNoisyTest
1532     runNoisyTestDefault
1533     if $jitTests and $isFTLPlatform
1534         runNoisyTestNoFTL
1535         runNoisyTestNoCJIT
1536         runNoisyTestNoCJITB3O1
1537         runNoisyTestEagerNoCJIT
1538     end
1539 end
1540
1541 def skip
1542     $didAddRunCommand = true
1543     puts "Skipping #{$collectionName}/#{$benchmark}"
1544 end
1545
1546 def allJSFiles(path)
1547     if path.file?
1548         [path]
1549     else
1550         result = []
1551         Dir.foreach(path) {
1552             | filename |
1553             next unless filename =~ /\.js$/
1554             next unless (path + filename).file?
1555             result << path + filename
1556         }
1557         result
1558     end
1559 end
1560
1561 def uniqueifyName(names, name)
1562     result = name.to_s
1563     toAdd = 1
1564     while names[result]
1565         result = "#{name}-#{toAdd}"
1566         toAdd += 1
1567     end
1568     names[result] = true
1569     result
1570 end
1571
1572 def simplifyCollectionName(collectionPath)
1573     outerDir = collectionPath.dirname
1574     name = collectionPath.basename
1575     lastName = name
1576     if collectionPath.directory?
1577         while lastName.to_s =~ /test/
1578             lastName = outerDir.basename
1579             name = lastName + name
1580             outerDir = outerDir.dirname
1581         end
1582     end
1583     uniqueifyName($collectionNames, name)
1584 end
1585
1586 def prepareCollection(name)
1587     FileUtils.mkdir_p $outputDir + name
1588
1589     absoluteCollection = $collection.realpath
1590
1591     Dir.chdir($outputDir) {
1592         bundleDir = baseDirForCollection(name)
1593
1594         # Create the proper directory structures.
1595         FileUtils.mkdir_p bundleDir
1596         if bundleDir.basename == $collection.basename
1597             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1598             $collection = bundleDir
1599         else
1600             FileUtils.cp_r absoluteCollection, bundleDir
1601             $collection = bundleDir + $collection.basename
1602         end
1603
1604         $extraFilesBaseDir = absoluteCollection
1605     }
1606 end
1607
1608 $collectionNames = {}
1609
1610 def handleCollectionFile(collection)
1611     collectionName = simplifyCollectionName(collection)
1612    
1613     paths = {}
1614     subCollections = []
1615     YAML::load(IO::read(collection)).each {
1616         | entry |
1617         if entry["collection"]
1618             subCollections << entry["collection"]
1619             next
1620         end
1621         
1622         if Pathname.new(entry["path"]).absolute?
1623             raise "Absolute path: " + entry["path"] + " in #{collection}"
1624         end
1625         
1626         if paths[entry["path"]]
1627             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1628         end
1629         
1630         subCollection = collection.dirname + entry["path"]
1631         
1632         if subCollection.file?
1633             subCollectionName = Pathname.new(entry["path"]).dirname
1634         else
1635             subCollectionName = entry["path"]
1636         end
1637         
1638         $collection = subCollection
1639         $collectionName = Pathname.new(collectionName)
1640         Pathname.new(subCollectionName).each_filename {
1641             | filename |
1642             next if filename =~ /^\./
1643             $collectionName += filename
1644         }
1645         $collectionName = $collectionName.to_s
1646         
1647         prepareCollection($collectionName)
1648       
1649         Dir.chdir($outputDir) {
1650             pathsToSearch = [$collection]
1651             if entry["tests"]
1652                 if entry["tests"].is_a? Array
1653                     pathsToSearch = entry["tests"].map {
1654                         | testName |
1655                         pathsToSearch[0] + testName
1656                     }
1657                 else
1658                     pathsToSearch[0] += entry["tests"]
1659                 end
1660             end
1661             pathsToSearch.each {
1662                 | pathToSearch |
1663                 allJSFiles(pathToSearch).each {
1664                     | path |
1665                     
1666                     $benchmark = path.basename
1667                     $benchmarkDirectory = path.dirname
1668                     
1669                     $runCommandOptions = {}
1670                     eval entry["cmd"]
1671                 }
1672             }
1673         }
1674     }
1675     
1676     subCollections.each {
1677         | subCollection |
1678         handleCollection(collection.dirname + subCollection)
1679     }
1680 end
1681
1682 def handleCollectionDirectory(collection)
1683     collectionName = simplifyCollectionName(collection)
1684     
1685     $collection = collection
1686     $collectionName = collectionName
1687     prepareCollection(collectionName)
1688    
1689     Dir.chdir($outputDir) {
1690         $benchmarkDirectory = $collection
1691         allJSFiles($collection).each {
1692             | path |
1693             
1694             $benchmark = path.basename
1695             
1696             $runCommandOptions = {}
1697             defaultRun unless parseRunCommands
1698         }
1699     }
1700 end
1701
1702 def handleCollection(collection)
1703     collection = Pathname.new(collection)
1704     
1705     if collection.file?
1706         handleCollectionFile(collection)
1707     else
1708         handleCollectionDirectory(collection)
1709     end
1710 end
1711
1712 def appendFailure(plan)
1713     File.open($outputDir + "failed", "a") {
1714         | outp |
1715         outp.puts plan.name
1716     }
1717     $numFailures += 1
1718 end
1719
1720 def appendPass(plan)
1721     File.open($outputDir + "passed", "a") {
1722         | outp |
1723         outp.puts plan.name
1724     }
1725     $numPasses += 1
1726 end
1727
1728 def appendResult(plan, didPass)
1729     File.open($outputDir + "results", "a") {
1730         | outp |
1731         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1732     }
1733 end
1734
1735 def prepareBundle
1736     raise if $bundle
1737
1738     if $doNotMessWithVMPath
1739         if !$remote and !$tarball
1740             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1741             $jscPath = Pathname.new($jscPath).realpath
1742         else
1743             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1744         end
1745     else
1746         originalJSCPath = $jscPath
1747         vmDir = $outputDir + ".vm"
1748         FileUtils.mkdir_p vmDir
1749         
1750         frameworkPath = frameworkFromJSCPath($jscPath)
1751         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1752         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1753         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1754
1755         if frameworkPath
1756             source = frameworkPath
1757             destination = Pathname.new(".vm")
1758         else
1759             source = originalJSCPath
1760             destination = $jscPath
1761
1762             Dir.chdir($outputDir) {
1763                 FileUtils.mkdir_p $jscPath.dirname
1764             }
1765         end
1766
1767         Dir.chdir($outputDir) {
1768             if $copyVM
1769                 FileUtils.cp_r source, destination
1770             else
1771                 begin 
1772                     FileUtils.ln_s source, destination
1773                 rescue Exception
1774                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1775                     FileUtils.cp_r source, destination
1776                 end
1777             end
1778
1779             if $remote and $hostOS == "linux"
1780                 begin
1781                     dependencies = `ldd #{source}`
1782                     dependencies.split(/\n/).each {
1783                         | dependency |
1784                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1785                     }
1786                 rescue
1787                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1788                 end
1789             end
1790         }
1791     end
1792     
1793     Dir.chdir($outputDir) {
1794         FileUtils.cp_r HELPERS_PATH, ".helpers"
1795     }
1796
1797     ARGV.each {
1798         | collection |
1799         handleCollection(collection)
1800     }
1801
1802     puts
1803 end
1804
1805 def cleanOldResults
1806     raise unless $bundle
1807
1808     eachResultFile($outputDir) {
1809         | path |
1810         FileUtils.rm_f path
1811     }
1812 end
1813
1814 def cleanEmptyResultFiles
1815     eachResultFile($outputDir) {
1816         | path |
1817         next unless path.basename.to_s =~ /\.out$/
1818         next unless FileTest.size(path) == 0
1819         FileUtils.rm_f path
1820     }
1821 end
1822
1823 def eachResultFile(startingDir, &block)
1824     dirsToClean = [startingDir]
1825     until dirsToClean.empty?
1826         nextDir = dirsToClean.pop
1827         Dir.foreach(nextDir) {
1828             | entry |
1829             next if entry =~ /^\./
1830             path = nextDir + entry
1831             if path.directory?
1832                 dirsToClean.push(path)
1833             else
1834                 block.call(path)
1835             end
1836         }
1837     end
1838 end
1839
1840 def prepareTestRunner
1841     raise if $bundle
1842
1843     $runlist.each_with_index {
1844         | plan, index |
1845         plan.index = index
1846     }
1847
1848     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1849     toDelete = []
1850     Dir.foreach($runnerDir) {
1851         | filename |
1852         if filename =~ /^test_/
1853             toDelete << filename
1854         end
1855     }
1856     
1857     toDelete.each {
1858         | filename |
1859         File.unlink($runnerDir + filename)
1860     }
1861
1862     $runlist.each {
1863         | plan |
1864         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1865     }
1866
1867     case $testRunnerType
1868     when :make
1869         prepareMakeTestRunner
1870     when :shell
1871         prepareShellTestRunner
1872     else
1873         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1874     end
1875 end
1876
1877 def prepareShellTestRunner
1878     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
1879 end
1880
1881 def prepareMakeTestRunner
1882     # The goals of our parallel test runner are scalability and simplicity. The
1883     # simplicity part is particularly important. We don't want to have to have
1884     # a full-time contributor just philosophising about parallel testing.
1885     #
1886     # As such, we just pass off all of the hard work to 'make'. This creates a
1887     # dummy directory ("$outputDir/.runner") in which we create a dummy
1888     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
1889     # That is, for each test we know we will run, there is a rule in the
1890     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
1891     # Makefile results in 'make' doing all of the hard work:
1892     #
1893     # - Load balancing just works. Most systems have a great load balancer in
1894     #   'make'. If your system doesn't then just install a real 'make'.
1895     #
1896     # - Interruptions just work. For example Ctrl-C handling in 'make' is
1897     #   exactly right. You don't have to worry about zombie processes.
1898     #
1899     # We then do some tricks to make failure detection work and to make this
1900     # totally sound. If a test fails, we don't want the whole 'make' job to
1901     # stop. We also don't have any facility for makefile-escaping of path names.
1902     # We do have such a thing for shell-escaping, though. We fix both problems
1903     # by having the actual work for each of the test rules be done in a shell
1904     # script on the side. There is one such script per test. The script responds
1905     # to failure by printing something on the console and then touching a
1906     # failure file for that test, but then still returns 0. This makes 'make'
1907     # continue past that failure and complete all the tests anyway.
1908     #
1909     # In the end, this script collects all of the failures by searching for
1910     # files in the .runner directory whose name matches /^test_fail_/, where
1911     # the thing after the 'fail_' is the test index. Those are the files that
1912     # would be created by the test scripts if they detect failure. We're
1913     # basically using the filesystem as a concurrent database of test failures.
1914     # Even if two tests fail at the same time, since they're touching different
1915     # files we won't miss any failures.
1916     runIndices = []
1917     $runlist.each {
1918         | plan |
1919         runIndices << plan.index
1920     }
1921     
1922     File.open($runnerDir + "Makefile", "w") {
1923         | outp |
1924         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1925         runIndices.each {
1926             | index |
1927             plan = $runlist[index]
1928             outp.puts "test_done_#{index}:"
1929             outp.puts "\tsh test_script_#{plan.index}"
1930         }
1931     }
1932 end
1933
1934 def cleanRunnerDirectory
1935     raise unless $bundle
1936     Dir.foreach($runnerDir) {
1937         | filename |
1938         next unless filename =~ /^test_fail/
1939         FileUtils.rm_f $runnerDir + filename
1940     }
1941 end
1942
1943 def sshRead(cmd)
1944     raise unless $remote
1945
1946     result = ""
1947     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1948       | inp |
1949       inp.each_line {
1950         | line |
1951         result += line
1952       }
1953     }
1954     raise "#{$?}" unless $?.success?
1955     result
1956 end
1957
1958 def runCommandOnTester(cmd)
1959     if $remote
1960         result = sshRead(cmd)
1961     else
1962         result = `#{cmd}`
1963     end
1964 end
1965
1966 def numberOfProcessors
1967     if $hostOS == "windows"
1968         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1969     else
1970         begin
1971             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1972         rescue
1973             numProcessors = 0
1974         end
1975
1976         if numProcessors == 0
1977             begin
1978                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1979             rescue
1980                 numProcessors == 0
1981             end
1982         end
1983     end
1984
1985     if numProcessors == 0
1986         numProcessors = 1
1987     end
1988     return numProcessors
1989 end
1990
1991 def runAndMonitorTestRunnerCommand(*cmd)
1992     numberOfTests = 0
1993     Dir.chdir($runnerDir) {
1994         # -1 for the runscript, and -2 for '..' and '.'
1995         numberOfTests = Dir.entries(".").count - 3
1996     }
1997     unless $progressMeter
1998         mysys(cmd.join(' '))
1999     else
2000        running = {}
2001        didRun = {}
2002        didFail = {}
2003        blankLine = true
2004        prevStringLength = 0
2005        IO.popen(cmd.join(' '), mode="r") {
2006            | inp |
2007            inp.each_line {
2008                | line |
2009                line = line.scrub.chomp
2010                if line =~ /^Running /
2011                    running[$~.post_match] = true
2012                elsif line =~ /^PASS: /
2013                    didRun[$~.post_match] = true
2014                elsif line =~ /^FAIL: /
2015                    didRun[$~.post_match] = true
2016                    didFail[$~.post_match] = true
2017                else
2018                    unless blankLine
2019                        print("\r" + " " * prevStringLength + "\r")
2020                    end
2021                    puts line
2022                    blankLine = true
2023                end
2024
2025                def lpad(str, chars)
2026                    str = str.to_s
2027                    if str.length > chars
2028                        str
2029                    else
2030                       "%#{chars}s"%(str)
2031                    end
2032                end
2033
2034                string  = ""
2035                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
2036                unless didFail.empty?
2037                    string += " (failed #{didFail.size})"
2038                end
2039                string += " "
2040                (running.size - didRun.size).times {
2041                    string += "."
2042                }
2043                if string.length < prevStringLength
2044                    print string
2045                    print(" " * (prevStringLength - string.length))
2046                end
2047                print string
2048                prevStringLength = string.length
2049                blankLine = false
2050                $stdout.flush
2051            }
2052        }
2053        puts
2054        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
2055     end
2056 end
2057
2058 def runTestRunner
2059     case $testRunnerType
2060     when :shell
2061         testRunnerCommand = "sh runscript"
2062     when :make
2063         testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
2064     else
2065         raise "Unknown test runner type: #{$testRunnerType.to_s}"
2066     end
2067
2068     if $remote
2069         if !$remoteDirectory
2070             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
2071         end
2072         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
2073         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
2074         remoteScript = "\""
2075         remoteScript += "cd #{$remoteDirectory} && "
2076         remoteScript += "rm -rf #{$outputDir.basename} && "
2077         remoteScript += "tar xzf #{$tarFileName} && "
2078         remoteScript += "cd #{$outputDir.basename}/.runner && "
2079         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
2080         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
2081         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
2082         $envVars.each { |var| remoteScript += "export " << var << "\n" }
2083         remoteScript += "#{testRunnerCommand}\""
2084         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
2085     else
2086         Dir.chdir($runnerDir) {
2087             runAndMonitorTestRunnerCommand(testRunnerCommand)
2088         }
2089     end
2090 end
2091
2092 def detectFailures
2093     raise if $bundle
2094
2095     failures = []
2096     
2097     if $remote
2098         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
2099         output.split(/\n/).each {
2100             | line |
2101             next unless line =~ /test_fail_/
2102             failures << $~.post_match.to_i
2103         }
2104     else
2105         Dir.foreach($runnerDir) {
2106             | filename |
2107             next unless filename =~ /test_fail_/
2108             failures << $~.post_match.to_i
2109         }
2110     end
2111
2112     failureSet = {}
2113
2114     failures.each {
2115         | failure | 
2116         appendFailure($runlist[failure])
2117         failureSet[failure] = true
2118     }
2119
2120     familyMap = {}
2121     $runlist.each_with_index {
2122         | plan, index |
2123         unless familyMap[plan.family]
2124             familyMap[plan.family] = []
2125         end
2126         if failureSet[index]
2127             appendResult(plan, false)
2128             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
2129             next
2130         else
2131             appendResult(plan, true)
2132             familyMap[plan.family] << {:result => "PASS", :plan => plan};
2133         end
2134         appendPass(plan)
2135     }
2136
2137     File.open($outputDir + "resultsByFamily", "w") {
2138         | outp |
2139         first = true
2140         familyMap.keys.sort.each {
2141             | familyName |
2142             if first
2143                 first = false
2144             else
2145                 outp.puts
2146             end
2147             
2148             outp.print "#{familyName}:"
2149
2150             numPassed = 0
2151             familyMap[familyName].each {
2152                 | entry |
2153                 if entry[:result] == "PASS"
2154                     numPassed += 1
2155                 end
2156             }
2157
2158             if numPassed == familyMap[familyName].size
2159                 outp.puts " PASSED"
2160             elsif numPassed == 0
2161                 outp.puts " FAILED"
2162             else
2163                 outp.puts
2164                 familyMap[familyName].each {
2165                     | entry |
2166                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
2167                 }
2168             end
2169         }
2170     }
2171 end
2172
2173 def compressBundle
2174     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
2175     $stderr.puts ">> #{cmd}" if $verbosity >= 2
2176     raise unless system(cmd)
2177 end
2178
2179 def clean(file)
2180     FileUtils.rm_rf file unless $bundle
2181 end
2182
2183 clean($outputDir + "failed")
2184 clean($outputDir + "passed")
2185 clean($outputDir + "results")
2186 clean($outputDir + "resultsByFamily")
2187 clean($outputDir + ".vm")
2188 clean($outputDir + ".helpers")
2189 clean($outputDir + ".runner")
2190 clean($outputDir + ".tests")
2191 clean($outputDir + "_payload")
2192
2193 Dir.mkdir($outputDir) unless $outputDir.directory?
2194
2195 $outputDir = $outputDir.realpath
2196 $runnerDir = $outputDir + ".runner"
2197
2198 if !$numChildProcesses
2199     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
2200         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
2201     else
2202         $numChildProcesses = numberOfProcessors
2203     end
2204 end
2205
2206 if ENV["JSCTEST_timeout"]
2207     # In the worst case, the processors just interfere with each other.
2208     # Increase the timeout proportionally to the number of processors.
2209     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
2210 end
2211
2212 def runBundle
2213     raise unless $bundle
2214
2215     cleanRunnerDirectory
2216     cleanOldResults
2217     runTestRunner
2218     cleanEmptyResultFiles
2219 end
2220
2221 def runNormal
2222     raise if $bundle or $tarball
2223
2224     prepareBundle
2225     prepareTestRunner
2226     runTestRunner
2227     cleanEmptyResultFiles
2228     detectFailures
2229 end
2230
2231 def runTarball
2232     raise unless $tarball
2233
2234     prepareBundle 
2235     prepareTestRunner
2236     compressBundle
2237 end
2238
2239 def runRemote
2240     raise unless $remote
2241
2242     prepareBundle
2243     prepareTestRunner
2244     compressBundle
2245     runTestRunner
2246     detectFailures
2247 end
2248
2249 puts
2250 if $bundle
2251     runBundle
2252 elsif $remote
2253     runRemote
2254 elsif $tarball
2255     runTarball
2256 else
2257     runNormal
2258 end