Unreviewed, rolling out r218512.
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2016 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
47 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
48 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
49 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
50 WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
51 CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
52 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
53 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
54
55 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
56
57 begin
58     require 'shellwords'
59 rescue Exception => e
60     $stderr.puts "Warning: did not find shellwords, not running any tests."
61     exit 0
62 end
63
64 $canRunDisplayProfilerOutput = false
65
66 begin
67     require 'rubygems'
68     require 'json'
69     require 'highline'
70     $canRunDisplayProfilerOutput = true
71 rescue Exception => e
72     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
73     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
74     $stderr.puts "Error: #{e.inspect}"
75 end
76
77 def printCommandArray(*cmd)
78     begin
79         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
80     rescue
81         commandArray = cmd.join(' ')
82     end
83     $stderr.puts ">> #{commandArray}"
84 end
85
86 def mysys(*cmd)
87     printCommandArray(*cmd) if $verbosity >= 1
88     raise "Command failed: #{$?.inspect}" unless system(*cmd)
89 end
90
91 def escapeAll(array)
92     array.map {
93         | v |
94         raise "Detected a non-string in #{inspect}" unless v.is_a? String
95         Shellwords.shellescape(v)
96     }.join(' ')
97 end
98
99
100 $jscPath = nil
101 $doNotMessWithVMPath = false
102 $jitTests = true
103 $memoryLimited = false
104 $outputDir = Pathname.new("results")
105 $verbosity = 0
106 $bundle = nil
107 $tarball = false
108 $tarFileName = "payload.tar.gz"
109 $copyVM = false
110 $testRunnerType = nil
111 $remoteUser = nil
112 $remoteHost = nil
113 $remotePort = nil
114 $remoteDirectory = nil
115 $architecture = nil
116 $hostOS = nil
117 $filter = nil
118 $envVars = []
119 $quickMode = false
120 $buildType = "release"
121 $forceCollectContinuously = false
122
123 def usage
124     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
125     puts
126     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
127     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
128     puts "                            --jsc specifies an already present JavaScriptCore to test."
129     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
130     puts "                            Skip tests tagged with //@skip if $memoryLimited"
131     puts "--no-jit                    Do not run JIT specific tests."
132     puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
133     puts "                            platform."
134     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
135     puts "--verbose            (-v)   Print more things while running."
136     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
137     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
138     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
139     puts "                            e.g. x86, x86_64, arm."
140     puts "--os                        Specify os instead of determining from JavaScriptCore build."
141     puts "                            e.g. darwin, linux & windows."
142     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
143     puts "                            In general the shell runner is slower than the make runner."
144     puts "--make-runner               Uses the faster make-based runner."
145     puts "--remote                    Specify a remote host on which to run tests from command line argument."
146     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
147     puts "--child-processes    (-c)   Specify the number of child processes."
148     puts "--filter                    Only run tests whose name matches the given regular expression."
149     puts "--help               (-h)   Print this message."
150     puts "--env-vars                  Add a list of environment variables to set before running jsc."
151     puts "                            Each environment variable should be separated by a space."
152     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
153     puts "                            it will override the default value."
154     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
155     exit 1
156 end
157
158 jscArg = nil
159
160 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
161                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
162                ['--no-copy', GetoptLong::NO_ARGUMENT],
163                ['--memory-limited', GetoptLong::NO_ARGUMENT],
164                ['--no-jit', GetoptLong::NO_ARGUMENT],
165                ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
166                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
167                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
168                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
169                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
170                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
171                ['--os', GetoptLong::REQUIRED_ARGUMENT],
172                ['--shell-runner', GetoptLong::NO_ARGUMENT],
173                ['--make-runner', GetoptLong::NO_ARGUMENT],
174                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
175                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
176                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
177                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
178                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
179                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
180                ['--debug', GetoptLong::NO_ARGUMENT],
181                ['--release', GetoptLong::NO_ARGUMENT],
182                ['--quick', '-q', GetoptLong::NO_ARGUMENT]).each {
183     | opt, arg |
184     case opt
185     when '--help'
186         usage
187     when '--jsc'
188         jscArg = arg
189     when '--no-copy'
190         $doNotMessWithVMPath = true
191     when '--output-dir'
192         $outputDir = Pathname.new(arg)
193     when '--memory-limited'
194         $memoryLimited = true
195     when '--no-jit'
196         $jitTests = false
197     when '--force-collectContinuously'
198         $forceCollectContinuously = true;
199     when '--verbose'
200         $verbosity += 1
201     when '--run-bundle'
202         $bundle = Pathname.new(arg)
203     when '--tarball'
204         $tarball = true
205         $copyVM = true
206         $tarFileName = arg unless arg == ''
207     when '--force-vm-copy'
208         $copyVM = true
209     when '--shell-runner'
210         $testRunnerType = :shell
211     when '--make-runner'
212         $testRunnerType = :make
213     when '--remote'
214         $copyVM = true
215         $tarball = true
216         $remote = true
217         uri = URI("ssh://" + arg)
218         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
219     when '--remote-config-file'
220         $remoteConfigFile = arg
221     when '--child-processes'
222         $numChildProcesses = arg.to_i
223     when '--filter'
224         $filter = Regexp.new(arg)
225     when '--arch'
226         $architecture = arg
227     when '--os'
228         $hostOS = arg
229     when '--env-vars'
230         $envVars = arg.gsub(/\s+/, ' ').split(' ')
231     when '--quick'
232         $quickMode = true
233     when '--debug'
234         $buildType = "debug"
235     when '--release'
236         $buildType = "release"
237     end
238 }
239
240 if $remoteConfigFile
241     file = File.read($remoteConfigFile)
242     config = JSON.parse(file)
243
244     if !$remote and config['remote']
245         $copyVM = true
246         $tarball = true
247         $remote = true
248         uri = URI("ssh://" + config['remote'])
249         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
250     end
251
252     if config['remoteDirectory']
253         $remoteDirectory = config['remoteDirectory']
254     end
255 end
256
257 unless jscArg
258     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
259     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
260     command += ($buildType == "release") ? " --release" : " --debug"
261     command += " --executablePath"
262
263     output = `#{command}`.split("\n")
264     if !output.length
265         $stderr.puts "Error: must specify --jsc <path>"
266         exit 1
267     end
268
269     output = output[0]
270     jscArg = Pathname.new(output).join("jsc")
271     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
272     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
273     if !File.file?(jscArg)
274         $stderr.puts "Error: must specify --jsc <path>"
275         exit 1
276     end
277
278     puts "Using the following jsc path: #{jscArg}"
279 end
280
281 if $doNotMessWithVMPath
282     $jscPath = Pathname.new(jscArg)
283 else
284     $jscPath = Pathname.new(jscArg).realpath
285 end
286
287 $progressMeter = ($verbosity == 0 and $stdout.tty?)
288
289 if $bundle
290     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
291     $outputDir = $bundle
292 end
293
294 # Try to determine architecture. Return nil on failure.
295 def machOArchitectureCode
296     begin 
297         otoolLines = `otool -afh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
298         otoolLines.each_with_index {
299             | value, index |
300             if value =~ /magic/ and value =~ /cputype/
301                 return otoolLines[index + 1].split[1].to_i
302             end
303         }
304     rescue
305         $stderr.puts "Warning: unable to execute otool."
306     end
307     $stderr.puts "Warning: unable to determine architecture."
308     nil
309 end
310
311 def determineArchitectureFromMachOBinary
312     code = machOArchitectureCode
313     return nil unless code
314     is64BitFlag = 0x01000000
315     case code
316     when 7
317         "x86"
318     when 7 | is64BitFlag
319         "x86-64"
320     when 12
321         "arm"
322     when 12 | is64BitFlag
323         "arm64"
324     else
325         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
326         nil
327     end
328 end
329
330 def determineArchitectureFromELFBinary
331     f = File.open($jscPath.to_s)
332     data = f.read(19)
333
334     if !(data[0,4] == "\x7F\x45\x4C\x46")
335         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
336         return nil
337     end
338
339     code = data[18].ord
340     case code
341     when 3
342         "x86"
343     when 8
344         "mips"
345     when 62
346         "x86-64"
347     when 40
348         "arm"
349     when 183
350         "arm64"
351     else
352         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
353         nil
354     end
355 end
356
357 def determineArchitectureFromPEBinary
358     f = File.open($jscPath.to_s)
359     data = f.read(1024)
360
361     if !(data[0, 2] == "MZ")
362         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
363         return nil
364     end
365
366     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
367
368     if !(data[peHeaderAddr, 4] == "PE\0\0")
369         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
370         return nil
371     end
372
373     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
374
375     case machine
376     when 0x014c
377         "x86"
378     when 0x8664
379         "x86-64"
380     else
381         $stderr.puts "Warning: unsupported machine type: #{machine}"
382         nil
383     end
384 end
385
386 def determineArchitecture
387     case $hostOS
388     when "darwin"
389         determineArchitectureFromMachOBinary
390     when "linux"
391         determineArchitectureFromELFBinary
392     when "windows"
393         determineArchitectureFromPEBinary
394     else
395         $stderr.puts "Warning: unable to determine architecture on this platform."
396         nil
397     end
398 end
399
400 def determineOS
401     case RbConfig::CONFIG["host_os"]
402     when /darwin/i
403         "darwin"
404     when /linux/i
405         "linux"
406     when /mswin|mingw|cygwin/
407         "windows"
408     else
409         $stderr.puts "Warning: unable to determine host operating system"
410         nil
411     end
412 end
413
414 $hostOS = determineOS unless $hostOS
415 $architecture = determineArchitecture unless $architecture
416 $isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $architecture == "mips" || $hostOS == "windows")
417
418 if !$testRunnerType
419     if $remote and $hostOS == "darwin"
420         $testRunnerType = :shell
421     else
422         $testRunnerType = :make
423     end
424 end
425
426 $numFailures = 0
427 $numPasses = 0
428
429 # We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
430 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--maxPerThreadStackUsage=1572864"]
431 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
432 # NOTE: Tests rely on this using scribbleFreeCells.
433 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
434 B3O1_OPTIONS = ["--defaultB3OptLevel=1"]
435 FTL_OPTIONS = ["--useFTLJIT=true"]
436
437 def shouldCollectContinuously?
438     $buildType == "release" or $forceCollectContinuously
439 end
440
441 COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false"] : []
442
443 $runlist = []
444
445 def frameworkFromJSCPath(jscPath)
446     parentDirectory = jscPath.dirname
447     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
448         parentDirectory.dirname
449     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
450         jscPath.dirname + "JavaScriptCore.framework"
451     else
452         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
453         nil
454     end
455 end
456
457 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
458     dir = Pathname.new(".")
459     $benchmarkDirectory.each_filename {
460         | pathComponent |
461         dir += ".."
462     }
463     dir + resourcePath
464 end
465
466 def pathToVM
467     pathToBundleResourceFromBenchmarkDirectory($jscPath)
468 end
469
470 def pathToHelpers
471     pathToBundleResourceFromBenchmarkDirectory(".helpers")
472 end
473
474 def prefixCommand(prefix)
475     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
476 end
477
478 def redirectAndPrefixCommand(prefix)
479     prefixCommand(prefix) + " 2>&1"
480 end
481
482 def pipeAndPrefixCommand(outputFilename, prefix)
483     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
484 end
485
486 # Output handler for tests that are expected to be silent.
487 def silentOutputHandler
488     Proc.new {
489         | name |
490         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
491     }
492 end
493
494 # Output handler for tests that are expected to produce meaningful output.
495 def noisyOutputHandler
496     Proc.new {
497         | name |
498         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
499     }
500 end
501
502 # Error handler for tests that fail exactly when they return non-zero exit status.
503 # This is useful when a test is expected to fail.
504 def simpleErrorHandler
505     Proc.new {
506         | outp, plan |
507         outp.puts "if test -e #{plan.failFile}"
508         outp.puts "then"
509         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
510         outp.puts "    " + plan.failCommand
511         outp.puts "else"
512         outp.puts "    " + plan.successCommand
513         outp.puts "fi"
514     }
515 end
516
517 # Error handler for tests that fail exactly when they return zero exit status.
518 def expectedFailErrorHandler
519     Proc.new {
520         | outp, plan |
521         outp.puts "if test -e #{plan.failFile}"
522         outp.puts "then"
523         outp.puts "    " + plan.successCommand
524         outp.puts "else"
525         outp.puts "    (echo ERROR: Unexpected exit code: 0) | " + redirectAndPrefixCommand(plan.name)
526         outp.puts "    " + plan.failCommand
527         outp.puts "fi"
528     }
529 end
530
531 # Error handler for tests that fail exactly when they return non-zero exit status and produce
532 # lots of spew. This will echo that spew when the test fails.
533 def noisyErrorHandler
534     Proc.new {
535         | outp, plan |
536         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
537     
538         outp.puts "if test -e #{plan.failFile}"
539         outp.puts "then"
540         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
541         outp.puts "    " + plan.failCommand
542         outp.puts "else"
543         outp.puts "    " + plan.successCommand
544         outp.puts "fi"
545     }
546 end
547
548 # Error handler for tests that diff their output with some expectation.
549 def diffErrorHandler(expectedFilename)
550     Proc.new {
551         | outp, plan |
552         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
553         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
554         
555         outp.puts "if test -e #{plan.failFile}"
556         outp.puts "then"
557         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
558         outp.puts "    " + plan.failCommand
559         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
560         outp.puts "then"
561         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
562         outp.puts "    if [ $? -eq 0 ]"
563         outp.puts "    then"
564         outp.puts "    " + plan.successCommand
565         outp.puts "    else"
566         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
567         outp.puts "        " + plan.failCommand
568         outp.puts "    fi"
569         outp.puts "else"
570         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
571         outp.puts "    " + plan.failCommand
572         outp.puts "fi"
573     }
574 end
575
576 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
577 # tests.
578 def mozillaErrorHandler
579     Proc.new {
580         | outp, plan |
581         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
582
583         outp.puts "if test -e #{plan.failFile}"
584         outp.puts "then"
585         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
586         outp.puts "    " + plan.failCommand
587         outp.puts "elif grep -i -q failed! #{outputFilename}"
588         outp.puts "then"
589         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
590         outp.puts "    " + plan.failCommand
591         outp.puts "else"
592         outp.puts "    " + plan.successCommand
593         outp.puts "fi"
594     }
595 end
596
597 # Error handler for tests that report error by saying "failed!", and are expected to
598 # fail. This is used by Mozilla tests.
599 def mozillaFailErrorHandler
600     Proc.new {
601         | outp, plan |
602         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
603
604         outp.puts "if test -e #{plan.failFile}"
605         outp.puts "then"
606         outp.puts "    " + plan.successCommand
607         outp.puts "elif grep -i -q failed! #{outputFilename}"
608         outp.puts "then"
609         outp.puts "    " + plan.successCommand
610         outp.puts "else"
611         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
612         outp.puts "    " + plan.failCommand
613         outp.puts "fi"
614     }
615 end
616
617 # Error handler for tests that report error by saying "failed!", and are expected to have
618 # an exit code of 3.
619 def mozillaExit3ErrorHandler
620     Proc.new {
621         | outp, plan |
622         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
623
624         outp.puts "if test -e #{plan.failFile}"
625         outp.puts "then"
626         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
627         outp.puts "    then"
628         outp.puts "        if grep -i -q failed! #{outputFilename}"
629         outp.puts "        then"
630         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
631         outp.puts "            " + plan.failCommand
632         outp.puts "        else"
633         outp.puts "            " + plan.successCommand
634         outp.puts "        fi"
635         outp.puts "    else"
636         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
637         outp.puts "        " + plan.failCommand
638         outp.puts "    fi"
639         outp.puts "else"
640         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
641         outp.puts "    " + plan.failCommand
642         outp.puts "fi"
643     }
644 end
645
646 # Error handler for tests that report success by saying "Passed" or error by saying "FAILED".
647 # This is used by Chakra tests.
648 def chakraPassFailErrorHandler
649     Proc.new {
650         | outp, plan |
651         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
652
653         outp.puts "if test -e #{plan.failFile}"
654         outp.puts "then"
655         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
656         outp.puts "    " + plan.failCommand
657         outp.puts "elif grep -i -q FAILED #{outputFilename}"
658         outp.puts "then"
659         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
660         outp.puts "    " + plan.failCommand
661         outp.puts "else"
662         outp.puts "    " + plan.successCommand
663         outp.puts "fi"
664     }
665 end
666
667 $runCommandOptions = {}
668
669 class Plan
670     attr_reader :directory, :arguments, :family, :name, :outputHandler, :errorHandler
671     attr_accessor :index
672     
673     def initialize(directory, arguments, family, name, outputHandler, errorHandler)
674         @directory = directory
675         @arguments = arguments
676         @family = family
677         @name = name
678         @outputHandler = outputHandler
679         @errorHandler = errorHandler
680         @isSlow = !!$runCommandOptions[:isSlow]
681     end
682     
683     def shellCommand
684         # It's important to remember that the test is actually run in a subshell, so if we change directory
685         # in the subshell when we return we will be in our original directory. This is nice because we don't
686         # have to bend over backwards to do things relative to the root.
687         script = "(cd ../#{Shellwords.shellescape(@directory.to_s)} && ("
688         $envVars.each { |var| script += "export " << var << "; " }
689         script += "\"$@\" " + escapeAll(@arguments) + "))"
690         return script
691     end
692     
693     def reproScriptCommand
694         # We have to find our way back to the .runner directory since that's where all of the relative
695         # paths assume they start out from.
696         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
697         script += "cd $CURRENT_DIR\n"
698         Pathname.new(@name).dirname.each_filename {
699             | pathComponent |
700             script += "cd ..\n"
701         }
702         script += "cd .runner\n"
703
704         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
705         script += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])}\n"
706         $envVars.each { |var| script += "export " << var << "\n" }
707         script += "#{shellCommand} || exit 1"
708         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
709     end
710     
711     def failCommand
712         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
713     end
714     
715     def successCommand
716         if $progressMeter or $verbosity >= 2
717             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
718         else
719             "rm -f #{failFile}"
720         end
721     end
722     
723     def failFile
724         "test_fail_#{@index}"
725     end
726     
727     def writeRunScript(filename)
728         File.open(filename, "w") {
729             | outp |
730             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
731             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
732             cmd += @outputHandler.call(@name)
733             if $verbosity >= 3
734                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
735             end
736             outp.puts cmd
737             @errorHandler.call(outp, self)
738         }
739     end
740 end
741
742 $uniqueFilenameCounter = 0
743 def uniqueFilename(extension)
744     payloadDir = $outputDir + "_payload"
745     Dir.mkdir payloadDir unless payloadDir.directory?
746     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
747     $uniqueFilenameCounter += 1
748     result
749 end
750
751 def baseOutputName(kind)
752     "#{$collectionName}/#{$benchmark}.#{kind}"
753 end
754
755 def addRunCommand(kind, command, outputHandler, errorHandler)
756     $didAddRunCommand = true
757     name = baseOutputName(kind)
758     if $filter and name !~ $filter
759         return
760     end
761     plan = Plan.new(
762         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
763         errorHandler)
764     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
765         $runlist.unshift plan
766     else
767         $runlist << plan
768     end
769 end
770
771 # Returns true if there were run commands found in the file ($benchmarkDirectory +
772 # $benchmark), in which case those run commands have already been executed. Otherwise
773 # returns false, in which case you're supposed to add your own run commands.
774 def parseRunCommands
775     oldDidAddRunCommand = $didAddRunCommand
776     $didAddRunCommand = false
777
778     Dir.chdir($outputDir) {
779         File.open($benchmarkDirectory + $benchmark) {
780             | inp |
781             inp.each_line {
782                 | line |
783                 begin
784                     doesMatch = line =~ /^\/\/@/
785                 rescue Exception => e
786                     # Apparently this happens in the case of some UTF8 stuff in some files, where
787                     # Ruby tries to be strict and throw exceptions.
788                     next
789                 end
790                 next unless doesMatch
791                 eval $~.post_match
792             }
793         }
794     }
795
796     result = $didAddRunCommand
797     $didAddRunCommand = result or oldDidAddRunCommand
798     result
799 end
800
801 def slow!
802     $runCommandOptions[:isSlow] = true
803 end
804
805 def runWithOutputHandler(kind, outputHandler, *options)
806     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
807 end
808
809 def run(kind, *options)
810     runWithOutputHandler(kind, silentOutputHandler, *options)
811 end
812
813 def runNoFTL(*optionalTestSpecificOptions)
814     run("no-ftl", *optionalTestSpecificOptions)
815 end
816
817 def runWithRAMSize(size, *optionalTestSpecificOptions)
818     run("ram-size-#{size}", "--forceRAMSize=#{size}", *optionalTestSpecificOptions)
819 end
820
821 def runOneLargeHeap(*optionalTestSpecificOptions)
822     if $memoryLimited
823         $didAddRunCommand = true
824         puts "Skipping #{$collectionName}/#{$benchmark}"
825     else
826         run("default", *optionalTestSpecificOptions)
827     end
828 end
829
830 def runNoJIT(*optionalTestSpecificOptions)
831     run("no-jit", "--useJIT=false", *optionalTestSpecificOptions)
832 end
833
834 def runNoLLInt(*optionalTestSpecificOptions)
835     if $jitTests
836         run("no-llint", "--useLLInt=false", *optionalTestSpecificOptions)
837     end
838 end
839
840 # NOTE: Tests rely on this using scribbleFreeCells.
841 def runNoCJITValidate(*optionalTestSpecificOptions)
842     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
843 end
844
845 def runNoCJITValidatePhases(*optionalTestSpecificOptions)
846     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
847 end
848
849 def runNoCJITCollectContinuously(*optionalTestSpecificOptions)
850     run("no-cjit-collect-continuously", *(NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
851 end
852
853 def runDefault(*optionalTestSpecificOptions)
854     run("default", *(FTL_OPTIONS + optionalTestSpecificOptions))
855 end
856
857 def runFTLNoCJIT(*optionalTestSpecificOptions)
858     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
859 end
860
861 def runFTLNoCJITB3O1(*optionalTestSpecificOptions)
862     run("ftl-no-cjit-b3o1", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
863 end
864
865 def runFTLNoCJITValidate(*optionalTestSpecificOptions)
866     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
867 end
868
869 def runFTLNoCJITNoPutStackValidate(*optionalTestSpecificOptions)
870     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
871 end
872
873 def runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
874     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
875 end
876
877 def runFTLNoCJITOSRValidation(*optionalTestSpecificOptions)
878     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
879 end
880
881 def runDFGEager(*optionalTestSpecificOptions)
882     run("dfg-eager", *(EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
883 end
884
885 def runDFGEagerNoCJITValidate(*optionalTestSpecificOptions)
886     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
887 end
888
889 def runFTLEager(*optionalTestSpecificOptions)
890     run("ftl-eager", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
891 end
892
893 def runFTLEagerWatchdog(*optionalTestSpecificOptions)
894     timeout = rand(100)
895     run("ftl-eager-watchdog-#{timeout}", "--watchdog=#{timeout}", "--watchdog-exception-ok", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
896 end
897
898 def runFTLEagerNoCJITValidate(*optionalTestSpecificOptions)
899     run("ftl-eager-no-cjit", "--validateGraph=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
900 end
901
902 def runFTLEagerNoCJITB3O1(*optionalTestSpecificOptions)
903     run("ftl-eager-no-cjit-b3o1", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
904 end
905
906 def runFTLEagerNoCJITOSRValidation(*optionalTestSpecificOptions)
907     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
908 end
909
910 def runNoCJITNoASO(*optionalTestSpecificOptions)
911     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
912 end
913
914 def runNoCJITNoAccessInlining(*optionalTestSpecificOptions)
915     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
916 end
917
918 def runFTLNoCJITNoAccessInlining(*optionalTestSpecificOptions)
919     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
920 end
921
922 def runFTLNoCJITSmallPool(*optionalTestSpecificOptions)
923     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
924 end
925
926 def runNoCJIT(*optionalTestSpecificOptions)
927     run("no-cjit", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
928 end
929
930 def runDFGMaximalFlushPhase(*optionalTestSpecificOptions)
931     run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
932 end
933
934 def runShadowChicken(*optionalTestSpecificOptions)
935     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true", *optionalTestSpecificOptions)
936 end
937
938 def defaultRun
939     if $quickMode
940         defaultQuickRun
941     else
942         runDefault
943         if $jitTests
944             runNoLLInt
945             runNoCJITValidatePhases
946             runNoCJITCollectContinuously if shouldCollectContinuously?
947             runDFGEager
948             runDFGEagerNoCJITValidate
949             runDFGMaximalFlushPhase
950
951             return if !$isFTLPlatform
952
953             runNoFTL
954             runFTLNoCJITValidate
955             runFTLNoCJITB3O1
956             runFTLNoCJITNoPutStackValidate
957             runFTLNoCJITNoInlineValidate
958             runFTLEager
959             runFTLEagerNoCJITValidate
960             runFTLEagerNoCJITB3O1
961             runFTLNoCJITSmallPool
962         end
963     end
964 end
965
966 def defaultNoNoLLIntRun
967     if $quickMode
968         defaultQuickRun
969     else
970         runDefault
971         if $jitTests
972             runNoCJITValidatePhases
973             runNoCJITCollectContinuously if shouldCollectContinuously?
974             runDFGEager
975             runDFGEagerNoCJITValidate
976             runDFGMaximalFlushPhase
977
978             return if !$isFTLPlatform
979
980             runNoFTL
981             runFTLNoCJITValidate
982             runFTLNoCJITB3O1
983             runFTLNoCJITNoPutStackValidate
984             runFTLNoCJITNoInlineValidate
985             runFTLEager
986             runFTLEagerNoCJITValidate
987             runFTLNoCJITSmallPool
988         end
989     end
990 end
991
992 def defaultQuickRun
993     runDefault
994     if $jitTests
995         runNoCJITValidate
996
997         return if $isFTLPlatform
998
999         runNoFTL
1000         runFTLNoCJITValidate
1001     end
1002 end
1003
1004 def defaultSpotCheckNoMaximalFlush
1005     defaultQuickRun
1006     runNoCJITNoAccessInlining
1007
1008     return if !$isFTLPlatform
1009
1010     runFTLNoCJITOSRValidation
1011     runFTLNoCJITNoAccessInlining
1012     runFTLNoCJITB3O1
1013 end
1014
1015 def defaultSpotCheck
1016     defaultSpotCheckNoMaximalFlush
1017     runDFGMaximalFlushPhase
1018 end
1019
1020 # This is expected to not do eager runs because eager runs can have a lot of recompilations
1021 # for reasons that don't arise in the real world. It's used for tests that assert convergence
1022 # by counting recompilations.
1023 def defaultNoEagerRun
1024     runDefault
1025     if $jitTests
1026         runNoLLInt
1027         runNoCJITValidatePhases
1028         runNoCJITCollectContinuously if shouldCollectContinuously?
1029
1030         return if !$isFTLPlatform
1031
1032         runNoFTL
1033         runFTLNoCJITValidate
1034         runFTLNoCJITNoInlineValidate
1035         runFTLNoCJITB3O1
1036     end
1037 end
1038
1039 def defaultNoSamplingProfilerRun
1040     runDefault
1041     if $jitTests
1042         runNoLLInt
1043         runNoCJITValidatePhases
1044         runNoCJITCollectContinuously if shouldCollectContinuously?
1045         runDFGEager
1046         runDFGEagerNoCJITValidate
1047         runDFGMaximalFlushPhase
1048
1049         return if !$isFTLPlatform
1050
1051         runNoFTL
1052         runFTLNoCJITNoPutStackValidate
1053         runFTLNoCJITNoInlineValidate
1054         runFTLEager
1055         runFTLEagerNoCJITValidate
1056         runFTLNoCJITSmallPool
1057     end
1058 end
1059
1060 def runProfiler
1061     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
1062         skip
1063         return
1064     end
1065
1066     profilerOutput = uniqueFilename(".json")
1067     if $canRunDisplayProfilerOutput
1068         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
1069     else
1070         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
1071         run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
1072     end
1073 end
1074
1075 def runExceptionFuzz
1076     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
1077     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
1078 end
1079
1080 def runExecutableAllocationFuzz(name, *options)
1081     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
1082     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
1083 end
1084
1085 def runTypeProfiler
1086     if !$jitTests
1087         return
1088     end
1089
1090     return if !$isFTLPlatform
1091
1092     run("ftl-no-cjit-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1093     run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
1094 end
1095
1096 def runControlFlowProfiler
1097     if !$jitTests
1098         return
1099     end
1100
1101     return if !$isFTLPlatform
1102
1103     run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1104 end
1105
1106 def runTest262(mode, exception, includeFiles, flags)
1107     failsWithException = exception != "NoException"
1108     isStrict = false
1109     isModule = false
1110     isAsync = false
1111
1112     flags.each {
1113         | flag |
1114         case flag
1115         when :strict
1116             isStrict = true
1117         when :module
1118             isModule = true
1119         when :async
1120             isAsync = true
1121         else
1122             raise "Invalid flag for runTest262, #{flag}"
1123         end
1124     }
1125
1126     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
1127
1128     args = [pathToVM.to_s] + BASE_OPTIONS
1129     args << "--exception=" + exception if failsWithException
1130     args << "--test262-async" if isAsync
1131     args += includeFiles
1132
1133     case mode
1134     when :normal
1135         errorHandler = simpleErrorHandler
1136         outputHandler = silentOutputHandler
1137     when :fail
1138         errorHandler = expectedFailErrorHandler
1139         outputHandler = noisyOutputHandler
1140     when :failDueToOutdatedOrBadTest
1141         errorHandler = expectedFailErrorHandler
1142         outputHandler = noisyOutputHandler
1143     else
1144         raise "Invalid mode: #{mode}"
1145     end
1146
1147     if isStrict
1148         kind = "default-strict"
1149         args << "--strict-file=#{$benchmark}"
1150     else
1151         kind = "default"
1152         if isModule
1153             args << "--module-file=#{$benchmark}"
1154         else
1155             args << $benchmark.to_s
1156         end
1157     end
1158
1159     addRunCommand(kind, args, outputHandler, errorHandler)
1160 end
1161
1162 def prepareTest262Fixture
1163     # This function is used to add the files used by Test262 modules tests.
1164     prepareExtraRelativeFiles([""], $collection)
1165 end
1166
1167 def runES6(mode)
1168     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
1169     case mode
1170     when :normal
1171         errorHandler = simpleErrorHandler
1172     when :fail
1173         errorHandler = expectedFailErrorHandler
1174     when :failDueToOutdatedOrBadTest
1175         errorHandler = expectedFailErrorHandler
1176     else
1177         raise "Invalid mode: #{mode}"
1178     end
1179     addRunCommand("default", args, noisyOutputHandler, errorHandler)
1180 end
1181
1182 def runModules
1183     run("default-modules", "-m")
1184
1185     if !$jitTests
1186         return
1187     end
1188
1189     run("no-llint-modules", "-m", "--useLLInt=false")
1190     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
1191     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
1192     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1193
1194     return if !$isFTLPlatform
1195
1196     run("default-ftl-modules", "-m", *FTL_OPTIONS)
1197     run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1198     run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1199     run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
1200     run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1201     run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1202 end
1203
1204 def runWebAssembly
1205     return if !$jitTests
1206     return if !$isFTLPlatform
1207     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1208     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1209     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1210     run("default-wasm", "-m", *FTL_OPTIONS)
1211     if !$quickMode
1212         run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1213         run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1214         run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1215         run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1216     end
1217 end
1218
1219 def runWebAssemblyEmscripten(mode)
1220     case mode
1221     when :skip
1222         return
1223     end
1224     return if !$jitTests
1225     return if !$isFTLPlatform
1226     wasm = $benchmark.to_s.sub! '.js', '.wasm'
1227     prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
1228     run("default-wasm", *FTL_OPTIONS)
1229     if !$quickMode
1230         run("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1231         run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1232         run("wasm-no-call-ic", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1233         run("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1234     end
1235 end
1236
1237 def runWebAssemblySpecTest(mode)
1238     case mode
1239     when :skip
1240         return
1241     end
1242     return if !$jitTests
1243     return if !$isFTLPlatform
1244     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1245
1246     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1247     prepareExtraRelativeFiles(modules.map { |f| "../../" + f }, $collection)
1248
1249     harness = Dir[WASMTESTS_PATH + "spec-harness/" + "*.js"].map { |f| File.basename(f) }
1250     prepareExtraRelativeFiles(harness.map { |f| "../../spec-harness/" + f }, $collection)
1251
1252     runWithOutputHandler("default-wasm", noisyOutputHandler, "../spec-harness.js", *FTL_OPTIONS)
1253     if !$quickMode
1254       runWithOutputHandler("wasm-no-cjit-yes-tls-context", noisyOutputHandler, "../spec-harness.js",  "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1255       runWithOutputHandler("wasm-eager-jettison", noisyOutputHandler, "../spec-harness.js", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1256       runWithOutputHandler("wasm-no-call-ic", noisyOutputHandler, "../spec-harness.js", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1257       runWithOutputHandler("wasm-no-tls-context", noisyOutputHandler, "../spec-harness.js", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1258     end
1259 end
1260
1261 def runWebAssemblyLowExecutableMemory(*optionalTestSpecificOptions)
1262     return if !$jitTests
1263     return if !$isFTLPlatform
1264     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1265     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1266     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1267     # Only let WebAssembly get executable memory.
1268     run("default-wasm", "--useConcurrentGC=0" , "--useConcurrentJIT=0", "--jitMemoryReservationSize=15000", "--useBaselineJIT=0", "--useDFGJIT=0", "--useFTLJIT=0", "-m")
1269 end
1270
1271 def runChakra(mode, exception, baselineFile, extraFiles)
1272     raise unless $benchmark.to_s =~ /\.js$/
1273     failsWithException = exception != "NoException"
1274     testName = $~.pre_match
1275
1276     prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
1277     prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)
1278
1279     args = [pathToVM.to_s] + BASE_OPTIONS
1280     args += FTL_OPTIONS if $isFTLPlatform
1281     args += EAGER_OPTIONS
1282     args << "--exception=" + exception if failsWithException
1283     args << "--dumpException" if failsWithException
1284     args += ["jsc-lib.js"]
1285
1286     case mode
1287     when :baseline
1288         prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
1289         errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
1290         outputHandler = noisyOutputHandler
1291     when :pass
1292         errorHandler = chakraPassFailErrorHandler
1293         outputHandler = noisyOutputHandler
1294     when :skipDueToOutdatedOrBadTest
1295         return
1296     when :skip
1297         return
1298     else
1299         raise "Invalid mode: #{mode}"
1300     end
1301
1302     kind = "default"
1303     args << $benchmark.to_s
1304
1305     addRunCommand(kind, args, outputHandler, errorHandler)
1306 end
1307
1308 def runLayoutTest(kind, *options)
1309     raise unless $benchmark.to_s =~ /\.js$/
1310     testName = $~.pre_match
1311     if kind
1312         kind = "layout-" + kind
1313     else
1314         kind = "layout"
1315     end
1316
1317     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1318     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1319
1320     args = [pathToVM.to_s] + BASE_OPTIONS + options +
1321         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1322          $benchmark.to_s,
1323          (Pathname.new("resources") + "standalone-post.js").to_s]
1324     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1325 end
1326
1327 def runLayoutTestNoFTL
1328     runLayoutTest("no-ftl")
1329 end
1330
1331 def runLayoutTestNoLLInt
1332     runLayoutTest("no-llint", "--useLLInt=false")
1333 end
1334
1335 def runLayoutTestNoCJIT
1336     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1337 end
1338
1339 def runLayoutTestDFGEagerNoCJIT
1340     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1341 end
1342
1343 def runLayoutTestDefault
1344     runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
1345 end
1346
1347 def runLayoutTestFTLNoCJIT
1348     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1349 end
1350
1351 def runLayoutTestFTLEagerNoCJIT
1352     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1353 end
1354
1355 def runLayoutTestFTLEagerNoCJITB3O1
1356     runLayoutTest("ftl-eager-no-cjit-b3o1", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS))
1357 end
1358
1359 def noFTLRunLayoutTest
1360     if !$jitTests
1361         return
1362     end
1363
1364     runLayoutTestNoLLInt
1365     runLayoutTestNoCJIT
1366     runLayoutTestDFGEagerNoCJIT
1367 end
1368
1369 def defaultQuickRunLayoutTest
1370     runLayoutTestDefault
1371     if $jitTests
1372         if $isFTLPlatform
1373             runLayoutTestNoFTL
1374             runLayoutTestFTLNoCJIT
1375             runLayoutTestFTLEagerNoCJIT
1376         else
1377             noFTLRunLayoutTest
1378         end
1379     end
1380 end
1381
1382 def defaultRunLayoutTest
1383     if $quickMode
1384         defaultQuickRunLayoutTest
1385     else
1386         runLayoutTestDefault
1387         if $jitTests
1388             noFTLRunLayoutTest
1389
1390             return if !$isFTLPlatform
1391
1392             runLayoutTestNoFTL
1393             runLayoutTestFTLNoCJIT
1394             runLayoutTestFTLEagerNoCJIT
1395         end
1396     end
1397 end
1398
1399 def noEagerNoNoLLIntTestsRunLayoutTest
1400     runLayoutTestDefault
1401     if $jitTests
1402         runLayoutTestNoCJIT
1403
1404         return if !$isFTLPlatform
1405
1406         runLayoutTestNoFTL
1407         runLayoutTestFTLNoCJIT
1408     end
1409 end
1410
1411 def noNoLLIntRunLayoutTest
1412     runLayoutTestDefault
1413     if $jitTests
1414         runLayoutTestNoCJIT
1415         runLayoutTestDFGEagerNoCJIT
1416
1417         return if !$isFTLPlatform
1418
1419         runLayoutTestNoFTL
1420         runLayoutTestFTLNoCJIT
1421         runLayoutTestFTLEagerNoCJIT
1422     end
1423 end
1424
1425 def prepareExtraRelativeFiles(extraFiles, destination)
1426     Dir.chdir($outputDir) {
1427         extraFiles.each {
1428             | file |
1429             dest = destination + file
1430             FileUtils.mkdir_p(dest.dirname)
1431             FileUtils.cp $extraFilesBaseDir + file, dest
1432         }
1433     }
1434 end
1435
1436 def baseDirForCollection(collectionName)
1437     Pathname(".tests") + collectionName
1438 end
1439
1440 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1441     raise unless absoluteBase.absolute?
1442     Dir.chdir($outputDir) {
1443         collectionBaseDir = baseDirForCollection($collectionName)
1444         extraFiles.each {
1445             | file |
1446             destination = collectionBaseDir + file
1447             FileUtils.mkdir_p destination.dirname unless destination.directory?
1448             FileUtils.cp absoluteBase + file, destination
1449         }
1450     }
1451 end
1452
1453 def runMozillaTest(kind, mode, extraFiles, *options)
1454     if kind
1455         kind = "mozilla-" + kind
1456     else
1457         kind = "mozilla"
1458     end
1459     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1460     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1461     case mode
1462     when :normal
1463         errorHandler = mozillaErrorHandler
1464     when :negative
1465         errorHandler = mozillaExit3ErrorHandler
1466     when :fail
1467         errorHandler = mozillaFailErrorHandler
1468     when :skip
1469         return
1470     else
1471         raise "Invalid mode: #{mode}"
1472     end
1473     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1474 end
1475
1476 def runMozillaTestDefault(mode, *extraFiles)
1477     runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
1478 end
1479
1480 def runMozillaTestNoFTL(mode, *extraFiles)
1481     runMozillaTest("no-ftl", mode, extraFiles)
1482 end
1483
1484 def runMozillaTestLLInt(mode, *extraFiles)
1485     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1486 end
1487
1488 def runMozillaTestBaselineJIT(mode, *extraFiles)
1489     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1490 end
1491
1492 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1493     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1494 end
1495
1496 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1497     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1498 end
1499
1500 def defaultQuickRunMozillaTest(mode, *extraFiles)
1501     if $jitTests
1502         runMozillaTestDefault(mode, *extraFiles)
1503         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1504     else
1505         runMozillaTestNoFTL(mode, *extraFiles)
1506         if $jitTests
1507             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1508         end
1509     end
1510 end
1511
1512 def defaultRunMozillaTest(mode, *extraFiles)
1513     if $quickMode
1514         defaultQuickRunMozillaTest(mode, *extraFiles)
1515     else
1516         runMozillaTestNoFTL(mode, *extraFiles)
1517         if $jitTests
1518             runMozillaTestLLInt(mode, *extraFiles)
1519             runMozillaTestBaselineJIT(mode, *extraFiles)
1520             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1521             runMozillaTestDefault(mode, *extraFiles)
1522             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
1523         end
1524     end
1525 end
1526
1527 def runNoisyTest(kind, *options)
1528     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
1529 end
1530
1531 def runNoisyTestDefault
1532     runNoisyTest("default", *FTL_OPTIONS)
1533 end
1534
1535 def runNoisyTestNoFTL
1536     runNoisyTest("no-ftl")
1537 end
1538
1539 def runNoisyTestNoCJIT
1540     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1541 end
1542
1543 def runNoisyTestNoCJITB3O1
1544     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS))
1545 end
1546
1547 def runNoisyTestEagerNoCJIT
1548     runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1549 end
1550
1551 def defaultRunNoisyTest
1552     runNoisyTestDefault
1553     if $jitTests and $isFTLPlatform
1554         runNoisyTestNoFTL
1555         runNoisyTestNoCJIT
1556         runNoisyTestNoCJITB3O1
1557         runNoisyTestEagerNoCJIT
1558     end
1559 end
1560
1561 def skip
1562     $didAddRunCommand = true
1563     puts "Skipping #{$collectionName}/#{$benchmark}"
1564 end
1565
1566 def allJSFiles(path)
1567     if path.file?
1568         [path]
1569     else
1570         result = []
1571         Dir.foreach(path) {
1572             | filename |
1573             next unless filename =~ /\.js$/
1574             next unless (path + filename).file?
1575             result << path + filename
1576         }
1577         result
1578     end
1579 end
1580
1581 def uniqueifyName(names, name)
1582     result = name.to_s
1583     toAdd = 1
1584     while names[result]
1585         result = "#{name}-#{toAdd}"
1586         toAdd += 1
1587     end
1588     names[result] = true
1589     result
1590 end
1591
1592 def simplifyCollectionName(collectionPath)
1593     outerDir = collectionPath.dirname
1594     name = collectionPath.basename
1595     lastName = name
1596     if collectionPath.directory?
1597         while lastName.to_s =~ /test/
1598             lastName = outerDir.basename
1599             name = lastName + name
1600             outerDir = outerDir.dirname
1601         end
1602     end
1603     uniqueifyName($collectionNames, name)
1604 end
1605
1606 def prepareCollection(name)
1607     FileUtils.mkdir_p $outputDir + name
1608
1609     absoluteCollection = $collection.realpath
1610
1611     Dir.chdir($outputDir) {
1612         bundleDir = baseDirForCollection(name)
1613
1614         # Create the proper directory structures.
1615         FileUtils.mkdir_p bundleDir
1616         if bundleDir.basename == $collection.basename
1617             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1618             $collection = bundleDir
1619         else
1620             FileUtils.cp_r absoluteCollection, bundleDir
1621             $collection = bundleDir + $collection.basename
1622         end
1623
1624         $extraFilesBaseDir = absoluteCollection
1625     }
1626 end
1627
1628 $collectionNames = {}
1629
1630 def handleCollectionFile(collection)
1631     collectionName = simplifyCollectionName(collection)
1632    
1633     paths = {}
1634     subCollections = []
1635     YAML::load(IO::read(collection)).each {
1636         | entry |
1637         if entry["collection"]
1638             subCollections << entry["collection"]
1639             next
1640         end
1641         
1642         if Pathname.new(entry["path"]).absolute?
1643             raise "Absolute path: " + entry["path"] + " in #{collection}"
1644         end
1645         
1646         if paths[entry["path"]]
1647             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1648         end
1649         
1650         subCollection = collection.dirname + entry["path"]
1651         
1652         if subCollection.file?
1653             subCollectionName = Pathname.new(entry["path"]).dirname
1654         else
1655             subCollectionName = entry["path"]
1656         end
1657         
1658         $collection = subCollection
1659         $collectionName = Pathname.new(collectionName)
1660         Pathname.new(subCollectionName).each_filename {
1661             | filename |
1662             next if filename =~ /^\./
1663             $collectionName += filename
1664         }
1665         $collectionName = $collectionName.to_s
1666         
1667         prepareCollection($collectionName)
1668       
1669         Dir.chdir($outputDir) {
1670             pathsToSearch = [$collection]
1671             if entry["tests"]
1672                 if entry["tests"].is_a? Array
1673                     pathsToSearch = entry["tests"].map {
1674                         | testName |
1675                         pathsToSearch[0] + testName
1676                     }
1677                 else
1678                     pathsToSearch[0] += entry["tests"]
1679                 end
1680             end
1681             pathsToSearch.each {
1682                 | pathToSearch |
1683                 allJSFiles(pathToSearch).each {
1684                     | path |
1685                     
1686                     $benchmark = path.basename
1687                     $benchmarkDirectory = path.dirname
1688                     
1689                     $runCommandOptions = {}
1690                     eval entry["cmd"]
1691                 }
1692             }
1693         }
1694     }
1695     
1696     subCollections.each {
1697         | subCollection |
1698         handleCollection(collection.dirname + subCollection)
1699     }
1700 end
1701
1702 def handleCollectionDirectory(collection)
1703     collectionName = simplifyCollectionName(collection)
1704     
1705     $collection = collection
1706     $collectionName = collectionName
1707     prepareCollection(collectionName)
1708    
1709     Dir.chdir($outputDir) {
1710         $benchmarkDirectory = $collection
1711         allJSFiles($collection).each {
1712             | path |
1713             
1714             $benchmark = path.basename
1715             
1716             $runCommandOptions = {}
1717             defaultRun unless parseRunCommands
1718         }
1719     }
1720 end
1721
1722 def handleCollection(collection)
1723     collection = Pathname.new(collection)
1724     
1725     if collection.file?
1726         handleCollectionFile(collection)
1727     else
1728         handleCollectionDirectory(collection)
1729     end
1730 end
1731
1732 def appendFailure(plan)
1733     File.open($outputDir + "failed", "a") {
1734         | outp |
1735         outp.puts plan.name
1736     }
1737     $numFailures += 1
1738 end
1739
1740 def appendPass(plan)
1741     File.open($outputDir + "passed", "a") {
1742         | outp |
1743         outp.puts plan.name
1744     }
1745     $numPasses += 1
1746 end
1747
1748 def appendResult(plan, didPass)
1749     File.open($outputDir + "results", "a") {
1750         | outp |
1751         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1752     }
1753 end
1754
1755 def prepareBundle
1756     raise if $bundle
1757
1758     if $doNotMessWithVMPath
1759         if !$remote and !$tarball
1760             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1761             $jscPath = Pathname.new($jscPath).realpath
1762         else
1763             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1764         end
1765     else
1766         originalJSCPath = $jscPath
1767         vmDir = $outputDir + ".vm"
1768         FileUtils.mkdir_p vmDir
1769         
1770         frameworkPath = frameworkFromJSCPath($jscPath)
1771         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1772         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1773         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1774
1775         if frameworkPath
1776             source = frameworkPath
1777             destination = Pathname.new(".vm")
1778         else
1779             source = originalJSCPath
1780             destination = $jscPath
1781
1782             Dir.chdir($outputDir) {
1783                 FileUtils.mkdir_p $jscPath.dirname
1784             }
1785         end
1786
1787         Dir.chdir($outputDir) {
1788             if $copyVM
1789                 FileUtils.cp_r source, destination
1790             else
1791                 begin 
1792                     FileUtils.ln_s source, destination
1793                 rescue Exception
1794                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1795                     FileUtils.cp_r source, destination
1796                 end
1797             end
1798
1799             if $remote and $hostOS == "linux"
1800                 begin
1801                     dependencies = `ldd #{source}`
1802                     dependencies.split(/\n/).each {
1803                         | dependency |
1804                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1805                     }
1806                 rescue
1807                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1808                 end
1809             end
1810         }
1811     end
1812     
1813     Dir.chdir($outputDir) {
1814         FileUtils.cp_r HELPERS_PATH, ".helpers"
1815     }
1816
1817     ARGV.each {
1818         | collection |
1819         handleCollection(collection)
1820     }
1821
1822     puts
1823 end
1824
1825 def cleanOldResults
1826     raise unless $bundle
1827
1828     eachResultFile($outputDir) {
1829         | path |
1830         FileUtils.rm_f path
1831     }
1832 end
1833
1834 def cleanEmptyResultFiles
1835     eachResultFile($outputDir) {
1836         | path |
1837         next unless path.basename.to_s =~ /\.out$/
1838         next unless FileTest.size(path) == 0
1839         FileUtils.rm_f path
1840     }
1841 end
1842
1843 def eachResultFile(startingDir, &block)
1844     dirsToClean = [startingDir]
1845     until dirsToClean.empty?
1846         nextDir = dirsToClean.pop
1847         Dir.foreach(nextDir) {
1848             | entry |
1849             next if entry =~ /^\./
1850             path = nextDir + entry
1851             if path.directory?
1852                 dirsToClean.push(path)
1853             else
1854                 block.call(path)
1855             end
1856         }
1857     end
1858 end
1859
1860 def prepareTestRunner
1861     raise if $bundle
1862
1863     $runlist.each_with_index {
1864         | plan, index |
1865         plan.index = index
1866     }
1867
1868     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1869     toDelete = []
1870     Dir.foreach($runnerDir) {
1871         | filename |
1872         if filename =~ /^test_/
1873             toDelete << filename
1874         end
1875     }
1876     
1877     toDelete.each {
1878         | filename |
1879         File.unlink($runnerDir + filename)
1880     }
1881
1882     $runlist.each {
1883         | plan |
1884         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1885     }
1886
1887     case $testRunnerType
1888     when :make
1889         prepareMakeTestRunner
1890     when :shell
1891         prepareShellTestRunner
1892     else
1893         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1894     end
1895 end
1896
1897 def prepareShellTestRunner
1898     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
1899 end
1900
1901 def prepareMakeTestRunner
1902     # The goals of our parallel test runner are scalability and simplicity. The
1903     # simplicity part is particularly important. We don't want to have to have
1904     # a full-time contributor just philosophising about parallel testing.
1905     #
1906     # As such, we just pass off all of the hard work to 'make'. This creates a
1907     # dummy directory ("$outputDir/.runner") in which we create a dummy
1908     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
1909     # That is, for each test we know we will run, there is a rule in the
1910     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
1911     # Makefile results in 'make' doing all of the hard work:
1912     #
1913     # - Load balancing just works. Most systems have a great load balancer in
1914     #   'make'. If your system doesn't then just install a real 'make'.
1915     #
1916     # - Interruptions just work. For example Ctrl-C handling in 'make' is
1917     #   exactly right. You don't have to worry about zombie processes.
1918     #
1919     # We then do some tricks to make failure detection work and to make this
1920     # totally sound. If a test fails, we don't want the whole 'make' job to
1921     # stop. We also don't have any facility for makefile-escaping of path names.
1922     # We do have such a thing for shell-escaping, though. We fix both problems
1923     # by having the actual work for each of the test rules be done in a shell
1924     # script on the side. There is one such script per test. The script responds
1925     # to failure by printing something on the console and then touching a
1926     # failure file for that test, but then still returns 0. This makes 'make'
1927     # continue past that failure and complete all the tests anyway.
1928     #
1929     # In the end, this script collects all of the failures by searching for
1930     # files in the .runner directory whose name matches /^test_fail_/, where
1931     # the thing after the 'fail_' is the test index. Those are the files that
1932     # would be created by the test scripts if they detect failure. We're
1933     # basically using the filesystem as a concurrent database of test failures.
1934     # Even if two tests fail at the same time, since they're touching different
1935     # files we won't miss any failures.
1936     runIndices = []
1937     $runlist.each {
1938         | plan |
1939         runIndices << plan.index
1940     }
1941     
1942     File.open($runnerDir + "Makefile", "w") {
1943         | outp |
1944         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1945         runIndices.each {
1946             | index |
1947             plan = $runlist[index]
1948             outp.puts "test_done_#{index}:"
1949             outp.puts "\tsh test_script_#{plan.index}"
1950         }
1951     }
1952 end
1953
1954 def cleanRunnerDirectory
1955     raise unless $bundle
1956     Dir.foreach($runnerDir) {
1957         | filename |
1958         next unless filename =~ /^test_fail/
1959         FileUtils.rm_f $runnerDir + filename
1960     }
1961 end
1962
1963 def sshRead(cmd)
1964     raise unless $remote
1965
1966     result = ""
1967     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1968       | inp |
1969       inp.each_line {
1970         | line |
1971         result += line
1972       }
1973     }
1974     raise "#{$?}" unless $?.success?
1975     result
1976 end
1977
1978 def runCommandOnTester(cmd)
1979     if $remote
1980         result = sshRead(cmd)
1981     else
1982         result = `#{cmd}`
1983     end
1984 end
1985
1986 def numberOfProcessors
1987     if $hostOS == "windows"
1988         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1989     else
1990         begin
1991             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1992         rescue
1993             numProcessors = 0
1994         end
1995
1996         if numProcessors == 0
1997             begin
1998                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1999             rescue
2000                 numProcessors == 0
2001             end
2002         end
2003     end
2004
2005     if numProcessors == 0
2006         numProcessors = 1
2007     end
2008     return numProcessors
2009 end
2010
2011 def runAndMonitorTestRunnerCommand(*cmd)
2012     numberOfTests = 0
2013     Dir.chdir($runnerDir) {
2014         # -1 for the runscript, and -2 for '..' and '.'
2015         numberOfTests = Dir.entries(".").count - 3
2016     }
2017     unless $progressMeter
2018         mysys(cmd.join(' '))
2019     else
2020        running = {}
2021        didRun = {}
2022        didFail = {}
2023        blankLine = true
2024        prevStringLength = 0
2025        IO.popen(cmd.join(' '), mode="r") {
2026            | inp |
2027            inp.each_line {
2028                | line |
2029                line = line.scrub.chomp
2030                if line =~ /^Running /
2031                    running[$~.post_match] = true
2032                elsif line =~ /^PASS: /
2033                    didRun[$~.post_match] = true
2034                elsif line =~ /^FAIL: /
2035                    didRun[$~.post_match] = true
2036                    didFail[$~.post_match] = true
2037                else
2038                    unless blankLine
2039                        print("\r" + " " * prevStringLength + "\r")
2040                    end
2041                    puts line
2042                    blankLine = true
2043                end
2044
2045                def lpad(str, chars)
2046                    str = str.to_s
2047                    if str.length > chars
2048                        str
2049                    else
2050                       "%#{chars}s"%(str)
2051                    end
2052                end
2053
2054                string  = ""
2055                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
2056                unless didFail.empty?
2057                    string += " (failed #{didFail.size})"
2058                end
2059                string += " "
2060                (running.size - didRun.size).times {
2061                    string += "."
2062                }
2063                if string.length < prevStringLength
2064                    print string
2065                    print(" " * (prevStringLength - string.length))
2066                end
2067                print string
2068                prevStringLength = string.length
2069                blankLine = false
2070                $stdout.flush
2071            }
2072        }
2073        puts
2074        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
2075     end
2076 end
2077
2078 def runTestRunner
2079     case $testRunnerType
2080     when :shell
2081         testRunnerCommand = "sh runscript"
2082     when :make
2083         testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
2084     else
2085         raise "Unknown test runner type: #{$testRunnerType.to_s}"
2086     end
2087
2088     if $remote
2089         if !$remoteDirectory
2090             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
2091         end
2092         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
2093         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
2094         remoteScript = "\""
2095         remoteScript += "cd #{$remoteDirectory} && "
2096         remoteScript += "rm -rf #{$outputDir.basename} && "
2097         remoteScript += "tar xzf #{$tarFileName} && "
2098         remoteScript += "cd #{$outputDir.basename}/.runner && "
2099         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
2100         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
2101         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
2102         $envVars.each { |var| remoteScript += "export " << var << "\n" }
2103         remoteScript += "#{testRunnerCommand}\""
2104         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
2105     else
2106         Dir.chdir($runnerDir) {
2107             runAndMonitorTestRunnerCommand(testRunnerCommand)
2108         }
2109     end
2110 end
2111
2112 def detectFailures
2113     raise if $bundle
2114
2115     failures = []
2116     
2117     if $remote
2118         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
2119         output.split(/\n/).each {
2120             | line |
2121             next unless line =~ /test_fail_/
2122             failures << $~.post_match.to_i
2123         }
2124     else
2125         Dir.foreach($runnerDir) {
2126             | filename |
2127             next unless filename =~ /test_fail_/
2128             failures << $~.post_match.to_i
2129         }
2130     end
2131
2132     failureSet = {}
2133
2134     failures.each {
2135         | failure | 
2136         appendFailure($runlist[failure])
2137         failureSet[failure] = true
2138     }
2139
2140     familyMap = {}
2141     $runlist.each_with_index {
2142         | plan, index |
2143         unless familyMap[plan.family]
2144             familyMap[plan.family] = []
2145         end
2146         if failureSet[index]
2147             appendResult(plan, false)
2148             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
2149             next
2150         else
2151             appendResult(plan, true)
2152             familyMap[plan.family] << {:result => "PASS", :plan => plan};
2153         end
2154         appendPass(plan)
2155     }
2156
2157     File.open($outputDir + "resultsByFamily", "w") {
2158         | outp |
2159         first = true
2160         familyMap.keys.sort.each {
2161             | familyName |
2162             if first
2163                 first = false
2164             else
2165                 outp.puts
2166             end
2167             
2168             outp.print "#{familyName}:"
2169
2170             numPassed = 0
2171             familyMap[familyName].each {
2172                 | entry |
2173                 if entry[:result] == "PASS"
2174                     numPassed += 1
2175                 end
2176             }
2177
2178             if numPassed == familyMap[familyName].size
2179                 outp.puts " PASSED"
2180             elsif numPassed == 0
2181                 outp.puts " FAILED"
2182             else
2183                 outp.puts
2184                 familyMap[familyName].each {
2185                     | entry |
2186                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
2187                 }
2188             end
2189         }
2190     }
2191 end
2192
2193 def compressBundle
2194     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
2195     $stderr.puts ">> #{cmd}" if $verbosity >= 2
2196     raise unless system(cmd)
2197 end
2198
2199 def clean(file)
2200     FileUtils.rm_rf file unless $bundle
2201 end
2202
2203 clean($outputDir + "failed")
2204 clean($outputDir + "passed")
2205 clean($outputDir + "results")
2206 clean($outputDir + "resultsByFamily")
2207 clean($outputDir + ".vm")
2208 clean($outputDir + ".helpers")
2209 clean($outputDir + ".runner")
2210 clean($outputDir + ".tests")
2211 clean($outputDir + "_payload")
2212
2213 Dir.mkdir($outputDir) unless $outputDir.directory?
2214
2215 $outputDir = $outputDir.realpath
2216 $runnerDir = $outputDir + ".runner"
2217
2218 if !$numChildProcesses
2219     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
2220         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
2221     else
2222         $numChildProcesses = numberOfProcessors
2223     end
2224 end
2225
2226 if ENV["JSCTEST_timeout"]
2227     # In the worst case, the processors just interfere with each other.
2228     # Increase the timeout proportionally to the number of processors.
2229     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
2230 end
2231
2232 def runBundle
2233     raise unless $bundle
2234
2235     cleanRunnerDirectory
2236     cleanOldResults
2237     runTestRunner
2238     cleanEmptyResultFiles
2239 end
2240
2241 def runNormal
2242     raise if $bundle or $tarball
2243
2244     prepareBundle
2245     prepareTestRunner
2246     runTestRunner
2247     cleanEmptyResultFiles
2248     detectFailures
2249 end
2250
2251 def runTarball
2252     raise unless $tarball
2253
2254     prepareBundle 
2255     prepareTestRunner
2256     compressBundle
2257 end
2258
2259 def runRemote
2260     raise unless $remote
2261
2262     prepareBundle
2263     prepareTestRunner
2264     compressBundle
2265     runTestRunner
2266     detectFailures
2267 end
2268
2269 puts
2270 if $bundle
2271     runBundle
2272 elsif $remote
2273     runRemote
2274 elsif $tarball
2275     runTarball
2276 else
2277     runNormal
2278 end