Bmalloc and GC should put auxiliaries (butterflies, typed array backing stores) in...
[WebKit.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2016 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
47 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
48 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
49 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
50 WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
51 CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
52 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
53 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
54
55 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
56
57 begin
58     require 'shellwords'
59 rescue Exception => e
60     $stderr.puts "Warning: did not find shellwords, not running any tests."
61     exit 0
62 end
63
64 $canRunDisplayProfilerOutput = false
65
66 begin
67     require 'rubygems'
68     require 'json'
69     require 'highline'
70     $canRunDisplayProfilerOutput = true
71 rescue Exception => e
72     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
73     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
74     $stderr.puts "Error: #{e.inspect}"
75 end
76
77 def printCommandArray(*cmd)
78     begin
79         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
80     rescue
81         commandArray = cmd.join(' ')
82     end
83     $stderr.puts ">> #{commandArray}"
84 end
85
86 def mysys(*cmd)
87     printCommandArray(*cmd) if $verbosity >= 1
88     raise "Command failed: #{$?.inspect}" unless system(*cmd)
89 end
90
91 def escapeAll(array)
92     array.map {
93         | v |
94         raise "Detected a non-string in #{inspect}" unless v.is_a? String
95         Shellwords.shellescape(v)
96     }.join(' ')
97 end
98
99
100 $jscPath = nil
101 $doNotMessWithVMPath = false
102 $jitTests = true
103 $memoryLimited = false
104 $outputDir = Pathname.new("results")
105 $verbosity = 0
106 $bundle = nil
107 $tarball = false
108 $tarFileName = "payload.tar.gz"
109 $copyVM = false
110 $testRunnerType = nil
111 $remoteUser = nil
112 $remoteHost = nil
113 $remotePort = nil
114 $remoteDirectory = nil
115 $architecture = nil
116 $hostOS = nil
117 $filter = nil
118 $envVars = []
119 $quickMode = false
120 $buildType = "release"
121 $forceCollectContinuously = false
122
123 def usage
124     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
125     puts
126     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
127     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
128     puts "                            --jsc specifies an already present JavaScriptCore to test."
129     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
130     puts "                            Skip tests tagged with //@skip if $memoryLimited"
131     puts "--no-jit                    Do not run JIT specific tests."
132     puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
133     puts "                            platform."
134     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
135     puts "--verbose            (-v)   Print more things while running."
136     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
137     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
138     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
139     puts "                            e.g. x86, x86_64, arm."
140     puts "--os                        Specify os instead of determining from JavaScriptCore build."
141     puts "                            e.g. darwin, linux & windows."
142     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
143     puts "                            In general the shell runner is slower than the make runner."
144     puts "--make-runner               Uses the faster make-based runner."
145     puts "--remote                    Specify a remote host on which to run tests from command line argument."
146     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
147     puts "--child-processes    (-c)   Specify the number of child processes."
148     puts "--filter                    Only run tests whose name matches the given regular expression."
149     puts "--help               (-h)   Print this message."
150     puts "--env-vars                  Add a list of environment variables to set before running jsc."
151     puts "                            Each environment variable should be separated by a space."
152     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
153     puts "                            it will override the default value."
154     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
155     exit 1
156 end
157
158 jscArg = nil
159
160 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
161                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
162                ['--no-copy', GetoptLong::NO_ARGUMENT],
163                ['--memory-limited', GetoptLong::NO_ARGUMENT],
164                ['--no-jit', GetoptLong::NO_ARGUMENT],
165                ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
166                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
167                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
168                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
169                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
170                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
171                ['--os', GetoptLong::REQUIRED_ARGUMENT],
172                ['--shell-runner', GetoptLong::NO_ARGUMENT],
173                ['--make-runner', GetoptLong::NO_ARGUMENT],
174                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
175                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
176                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
177                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
178                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
179                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
180                ['--debug', GetoptLong::NO_ARGUMENT],
181                ['--release', GetoptLong::NO_ARGUMENT],
182                ['--quick', '-q', GetoptLong::NO_ARGUMENT]).each {
183     | opt, arg |
184     case opt
185     when '--help'
186         usage
187     when '--jsc'
188         jscArg = arg
189     when '--no-copy'
190         $doNotMessWithVMPath = true
191     when '--output-dir'
192         $outputDir = Pathname.new(arg)
193     when '--memory-limited'
194         $memoryLimited = true
195     when '--no-jit'
196         $jitTests = false
197     when '--force-collectContinuously'
198         $forceCollectContinuously = true;
199     when '--verbose'
200         $verbosity += 1
201     when '--run-bundle'
202         $bundle = Pathname.new(arg)
203     when '--tarball'
204         $tarball = true
205         $copyVM = true
206         $tarFileName = arg unless arg == ''
207     when '--force-vm-copy'
208         $copyVM = true
209     when '--shell-runner'
210         $testRunnerType = :shell
211     when '--make-runner'
212         $testRunnerType = :make
213     when '--remote'
214         $copyVM = true
215         $tarball = true
216         $remote = true
217         uri = URI("ssh://" + arg)
218         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
219     when '--remote-config-file'
220         $remoteConfigFile = arg
221     when '--child-processes'
222         $numChildProcesses = arg.to_i
223     when '--filter'
224         $filter = Regexp.new(arg)
225     when '--arch'
226         $architecture = arg
227     when '--os'
228         $hostOS = arg
229     when '--env-vars'
230         $envVars = arg.gsub(/\s+/, ' ').split(' ')
231     when '--quick'
232         $quickMode = true
233     when '--debug'
234         $buildType = "debug"
235     when '--release'
236         $buildType = "release"
237     end
238 }
239
240 if $remoteConfigFile
241     file = File.read($remoteConfigFile)
242     config = JSON.parse(file)
243
244     if !$remote and config['remote']
245         $copyVM = true
246         $tarball = true
247         $remote = true
248         uri = URI("ssh://" + config['remote'])
249         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
250     end
251
252     if config['remoteDirectory']
253         $remoteDirectory = config['remoteDirectory']
254     end
255 end
256
257 unless jscArg
258     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
259     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
260     command += ($buildType == "release") ? " --release" : " --debug"
261     command += " --executablePath"
262
263     output = `#{command}`.split("\n")
264     if !output.length
265         $stderr.puts "Error: must specify --jsc <path>"
266         exit 1
267     end
268
269     output = output[0]
270     jscArg = Pathname.new(output).join("jsc")
271     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
272     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
273     if !File.file?(jscArg)
274         $stderr.puts "Error: must specify --jsc <path>"
275         exit 1
276     end
277
278     puts "Using the following jsc path: #{jscArg}"
279 end
280
281 if $doNotMessWithVMPath
282     $jscPath = Pathname.new(jscArg)
283 else
284     $jscPath = Pathname.new(jscArg).realpath
285 end
286
287 $progressMeter = ($verbosity == 0 and $stdout.tty?)
288
289 if $bundle
290     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
291     $outputDir = $bundle
292 end
293
294 # Try to determine architecture. Return nil on failure.
295 def machOArchitectureCode
296     begin 
297         otoolLines = `otool -afh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
298         otoolLines.each_with_index {
299             | value, index |
300             if value =~ /magic/ and value =~ /cputype/
301                 return otoolLines[index + 1].split[1].to_i
302             end
303         }
304     rescue
305         $stderr.puts "Warning: unable to execute otool."
306     end
307     $stderr.puts "Warning: unable to determine architecture."
308     nil
309 end
310
311 def determineArchitectureFromMachOBinary
312     code = machOArchitectureCode
313     return nil unless code
314     is64BitFlag = 0x01000000
315     case code
316     when 7
317         "x86"
318     when 7 | is64BitFlag
319         "x86-64"
320     when 12
321         "arm"
322     when 12 | is64BitFlag
323         "arm64"
324     else
325         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
326         nil
327     end
328 end
329
330 def determineArchitectureFromELFBinary
331     f = File.open($jscPath.to_s)
332     data = f.read(19)
333
334     if !(data[0,4] == "\x7F\x45\x4C\x46")
335         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
336         return nil
337     end
338
339     code = data[18].ord
340     case code
341     when 3
342         "x86"
343     when 8
344         "mips"
345     when 62
346         "x86-64"
347     when 40
348         "arm"
349     when 183
350         "arm64"
351     else
352         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
353         nil
354     end
355 end
356
357 def determineArchitectureFromPEBinary
358     f = File.open($jscPath.to_s)
359     data = f.read(1024)
360
361     if !(data[0, 2] == "MZ")
362         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
363         return nil
364     end
365
366     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
367
368     if !(data[peHeaderAddr, 4] == "PE\0\0")
369         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
370         return nil
371     end
372
373     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
374
375     case machine
376     when 0x014c
377         "x86"
378     when 0x8664
379         "x86-64"
380     else
381         $stderr.puts "Warning: unsupported machine type: #{machine}"
382         nil
383     end
384 end
385
386 def determineArchitecture
387     case $hostOS
388     when "darwin"
389         determineArchitectureFromMachOBinary
390     when "linux"
391         determineArchitectureFromELFBinary
392     when "windows"
393         determineArchitectureFromPEBinary
394     else
395         $stderr.puts "Warning: unable to determine architecture on this platform."
396         nil
397     end
398 end
399
400 def determineOS
401     case RbConfig::CONFIG["host_os"]
402     when /darwin/i
403         "darwin"
404     when /linux/i
405         "linux"
406     when /mswin|mingw|cygwin/
407         "windows"
408     else
409         $stderr.puts "Warning: unable to determine host operating system"
410         nil
411     end
412 end
413
414 $hostOS = determineOS unless $hostOS
415 $architecture = determineArchitecture unless $architecture
416 $isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $architecture == "mips" || $hostOS == "windows")
417
418 if !$testRunnerType
419     if $remote and $hostOS == "darwin"
420         $testRunnerType = :shell
421     else
422         $testRunnerType = :make
423     end
424 end
425
426 $numFailures = 0
427 $numPasses = 0
428
429 # We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
430 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--maxPerThreadStackUsage=1572864"]
431 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
432 # NOTE: Tests rely on this using scribbleFreeCells.
433 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
434 B3O1_OPTIONS = ["--defaultB3OptLevel=1"]
435 FTL_OPTIONS = ["--useFTLJIT=true"]
436
437 def shouldCollectContinuously?
438     $buildType == "release" or $forceCollectContinuously
439 end
440
441 COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false"] : []
442
443 $runlist = []
444
445 def frameworkFromJSCPath(jscPath)
446     parentDirectory = jscPath.dirname
447     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
448         parentDirectory.dirname
449     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
450         jscPath.dirname + "JavaScriptCore.framework"
451     else
452         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
453         nil
454     end
455 end
456
457 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
458     dir = Pathname.new(".")
459     $benchmarkDirectory.each_filename {
460         | pathComponent |
461         dir += ".."
462     }
463     dir + resourcePath
464 end
465
466 def pathToVM
467     pathToBundleResourceFromBenchmarkDirectory($jscPath)
468 end
469
470 def pathToHelpers
471     pathToBundleResourceFromBenchmarkDirectory(".helpers")
472 end
473
474 def prefixCommand(prefix)
475     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
476 end
477
478 def redirectAndPrefixCommand(prefix)
479     prefixCommand(prefix) + " 2>&1"
480 end
481
482 def pipeAndPrefixCommand(outputFilename, prefix)
483     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
484 end
485
486 # Output handler for tests that are expected to be silent.
487 def silentOutputHandler
488     Proc.new {
489         | name |
490         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
491     }
492 end
493
494 # Output handler for tests that are expected to produce meaningful output.
495 def noisyOutputHandler
496     Proc.new {
497         | name |
498         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
499     }
500 end
501
502 # Error handler for tests that fail exactly when they return non-zero exit status.
503 # This is useful when a test is expected to fail.
504 def simpleErrorHandler
505     Proc.new {
506         | outp, plan |
507         outp.puts "if test -e #{plan.failFile}"
508         outp.puts "then"
509         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
510         outp.puts "    " + plan.failCommand
511         outp.puts "else"
512         outp.puts "    " + plan.successCommand
513         outp.puts "fi"
514     }
515 end
516
517 # Error handler for tests that fail exactly when they return zero exit status.
518 def expectedFailErrorHandler
519     Proc.new {
520         | outp, plan |
521         outp.puts "if test -e #{plan.failFile}"
522         outp.puts "then"
523         outp.puts "    " + plan.successCommand
524         outp.puts "else"
525         outp.puts "    (echo ERROR: Unexpected exit code: 0) | " + redirectAndPrefixCommand(plan.name)
526         outp.puts "    " + plan.failCommand
527         outp.puts "fi"
528     }
529 end
530
531 # Error handler for tests that fail exactly when they return non-zero exit status and produce
532 # lots of spew. This will echo that spew when the test fails.
533 def noisyErrorHandler
534     Proc.new {
535         | outp, plan |
536         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
537     
538         outp.puts "if test -e #{plan.failFile}"
539         outp.puts "then"
540         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
541         outp.puts "    " + plan.failCommand
542         outp.puts "else"
543         outp.puts "    " + plan.successCommand
544         outp.puts "fi"
545     }
546 end
547
548 # Error handler for tests that diff their output with some expectation.
549 def diffErrorHandler(expectedFilename)
550     Proc.new {
551         | outp, plan |
552         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
553         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
554         
555         outp.puts "if test -e #{plan.failFile}"
556         outp.puts "then"
557         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
558         outp.puts "    " + plan.failCommand
559         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
560         outp.puts "then"
561         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
562         outp.puts "    if [ $? -eq 0 ]"
563         outp.puts "    then"
564         outp.puts "    " + plan.successCommand
565         outp.puts "    else"
566         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
567         outp.puts "        " + plan.failCommand
568         outp.puts "    fi"
569         outp.puts "else"
570         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
571         outp.puts "    " + plan.failCommand
572         outp.puts "fi"
573     }
574 end
575
576 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
577 # tests.
578 def mozillaErrorHandler
579     Proc.new {
580         | outp, plan |
581         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
582
583         outp.puts "if test -e #{plan.failFile}"
584         outp.puts "then"
585         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
586         outp.puts "    " + plan.failCommand
587         outp.puts "elif grep -i -q failed! #{outputFilename}"
588         outp.puts "then"
589         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
590         outp.puts "    " + plan.failCommand
591         outp.puts "else"
592         outp.puts "    " + plan.successCommand
593         outp.puts "fi"
594     }
595 end
596
597 # Error handler for tests that report error by saying "failed!", and are expected to
598 # fail. This is used by Mozilla tests.
599 def mozillaFailErrorHandler
600     Proc.new {
601         | outp, plan |
602         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
603
604         outp.puts "if test -e #{plan.failFile}"
605         outp.puts "then"
606         outp.puts "    " + plan.successCommand
607         outp.puts "elif grep -i -q failed! #{outputFilename}"
608         outp.puts "then"
609         outp.puts "    " + plan.successCommand
610         outp.puts "else"
611         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
612         outp.puts "    " + plan.failCommand
613         outp.puts "fi"
614     }
615 end
616
617 # Error handler for tests that report error by saying "failed!", and are expected to have
618 # an exit code of 3.
619 def mozillaExit3ErrorHandler
620     Proc.new {
621         | outp, plan |
622         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
623
624         outp.puts "if test -e #{plan.failFile}"
625         outp.puts "then"
626         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
627         outp.puts "    then"
628         outp.puts "        if grep -i -q failed! #{outputFilename}"
629         outp.puts "        then"
630         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
631         outp.puts "            " + plan.failCommand
632         outp.puts "        else"
633         outp.puts "            " + plan.successCommand
634         outp.puts "        fi"
635         outp.puts "    else"
636         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
637         outp.puts "        " + plan.failCommand
638         outp.puts "    fi"
639         outp.puts "else"
640         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
641         outp.puts "    " + plan.failCommand
642         outp.puts "fi"
643     }
644 end
645
646 # Error handler for tests that report success by saying "Passed" or error by saying "FAILED".
647 # This is used by Chakra tests.
648 def chakraPassFailErrorHandler
649     Proc.new {
650         | outp, plan |
651         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
652
653         outp.puts "if test -e #{plan.failFile}"
654         outp.puts "then"
655         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
656         outp.puts "    " + plan.failCommand
657         outp.puts "elif grep -i -q FAILED #{outputFilename}"
658         outp.puts "then"
659         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
660         outp.puts "    " + plan.failCommand
661         outp.puts "else"
662         outp.puts "    " + plan.successCommand
663         outp.puts "fi"
664     }
665 end
666
667 $runCommandOptions = {}
668
669 class Plan
670     attr_reader :directory, :arguments, :family, :name, :outputHandler, :errorHandler
671     attr_accessor :index
672     
673     def initialize(directory, arguments, family, name, outputHandler, errorHandler)
674         @directory = directory
675         @arguments = arguments
676         @family = family
677         @name = name
678         @outputHandler = outputHandler
679         @errorHandler = errorHandler
680         @isSlow = !!$runCommandOptions[:isSlow]
681     end
682     
683     def shellCommand
684         # It's important to remember that the test is actually run in a subshell, so if we change directory
685         # in the subshell when we return we will be in our original directory. This is nice because we don't
686         # have to bend over backwards to do things relative to the root.
687         script = "(cd ../#{Shellwords.shellescape(@directory.to_s)} && ("
688         $envVars.each { |var| script += "export " << var << "; " }
689         script += "\"$@\" " + escapeAll(@arguments) + "))"
690         return script
691     end
692     
693     def reproScriptCommand
694         # We have to find our way back to the .runner directory since that's where all of the relative
695         # paths assume they start out from.
696         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
697         script += "cd $CURRENT_DIR\n"
698         Pathname.new(@name).dirname.each_filename {
699             | pathComponent |
700             script += "cd ..\n"
701         }
702         script += "cd .runner\n"
703
704         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
705         script += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])}\n"
706         $envVars.each { |var| script += "export " << var << "\n" }
707         script += "#{shellCommand} || exit 1"
708         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
709     end
710     
711     def failCommand
712         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
713     end
714     
715     def successCommand
716         if $progressMeter or $verbosity >= 2
717             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
718         else
719             "rm -f #{failFile}"
720         end
721     end
722     
723     def failFile
724         "test_fail_#{@index}"
725     end
726     
727     def writeRunScript(filename)
728         File.open(filename, "w") {
729             | outp |
730             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
731             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
732             cmd += @outputHandler.call(@name)
733             if $verbosity >= 3
734                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
735             end
736             outp.puts cmd
737             @errorHandler.call(outp, self)
738         }
739     end
740 end
741
742 $uniqueFilenameCounter = 0
743 def uniqueFilename(extension)
744     payloadDir = $outputDir + "_payload"
745     Dir.mkdir payloadDir unless payloadDir.directory?
746     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
747     $uniqueFilenameCounter += 1
748     result
749 end
750
751 def baseOutputName(kind)
752     "#{$collectionName}/#{$benchmark}.#{kind}"
753 end
754
755 def addRunCommand(kind, command, outputHandler, errorHandler)
756     $didAddRunCommand = true
757     name = baseOutputName(kind)
758     if $filter and name !~ $filter
759         return
760     end
761     plan = Plan.new(
762         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
763         errorHandler)
764     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
765         $runlist.unshift plan
766     else
767         $runlist << plan
768     end
769 end
770
771 # Returns true if there were run commands found in the file ($benchmarkDirectory +
772 # $benchmark), in which case those run commands have already been executed. Otherwise
773 # returns false, in which case you're supposed to add your own run commands.
774 def parseRunCommands
775     oldDidAddRunCommand = $didAddRunCommand
776     $didAddRunCommand = false
777
778     Dir.chdir($outputDir) {
779         File.open($benchmarkDirectory + $benchmark) {
780             | inp |
781             inp.each_line {
782                 | line |
783                 begin
784                     doesMatch = line =~ /^\/\/@/
785                 rescue Exception => e
786                     # Apparently this happens in the case of some UTF8 stuff in some files, where
787                     # Ruby tries to be strict and throw exceptions.
788                     next
789                 end
790                 next unless doesMatch
791                 eval $~.post_match
792             }
793         }
794     }
795
796     result = $didAddRunCommand
797     $didAddRunCommand = result or oldDidAddRunCommand
798     result
799 end
800
801 def slow!
802     $runCommandOptions[:isSlow] = true
803 end
804
805 def runWithOutputHandler(kind, outputHandler, *options)
806     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
807 end
808
809 def run(kind, *options)
810     runWithOutputHandler(kind, silentOutputHandler, *options)
811 end
812
813 def runNoFTL(*optionalTestSpecificOptions)
814     run("no-ftl", *optionalTestSpecificOptions)
815 end
816
817 def runWithRAMSize(size, *optionalTestSpecificOptions)
818     run("ram-size-#{size}", "--forceRAMSize=#{size}", *optionalTestSpecificOptions)
819 end
820
821 def runOneLargeHeap(*optionalTestSpecificOptions)
822     if $memoryLimited
823         $didAddRunCommand = true
824         puts "Skipping #{$collectionName}/#{$benchmark}"
825     else
826         run("default", *optionalTestSpecificOptions)
827     end
828 end
829
830 def runNoJIT(*optionalTestSpecificOptions)
831     run("no-jit", "--useJIT=false", *optionalTestSpecificOptions)
832 end
833
834 def runNoLLInt(*optionalTestSpecificOptions)
835     if $jitTests
836         run("no-llint", "--useLLInt=false", *optionalTestSpecificOptions)
837     end
838 end
839
840 # NOTE: Tests rely on this using scribbleFreeCells.
841 def runNoCJITValidate(*optionalTestSpecificOptions)
842     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
843 end
844
845 def runNoCJITValidatePhases(*optionalTestSpecificOptions)
846     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
847 end
848
849 def runNoCJITCollectContinuously(*optionalTestSpecificOptions)
850     run("no-cjit-collect-continuously", *(NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
851 end
852
853 def runDefault(*optionalTestSpecificOptions)
854     run("default", *(FTL_OPTIONS + optionalTestSpecificOptions))
855 end
856
857 def runFTLNoCJIT(*optionalTestSpecificOptions)
858     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
859 end
860
861 def runFTLNoCJITB3O1(*optionalTestSpecificOptions)
862     run("ftl-no-cjit-b3o1", "--useArrayAllocationProfiling=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
863 end
864
865 def runFTLNoCJITValidate(*optionalTestSpecificOptions)
866     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
867 end
868
869 def runFTLNoCJITNoPutStackValidate(*optionalTestSpecificOptions)
870     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
871 end
872
873 def runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
874     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
875 end
876
877 def runFTLNoCJITOSRValidation(*optionalTestSpecificOptions)
878     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
879 end
880
881 def runDFGEager(*optionalTestSpecificOptions)
882     run("dfg-eager", *(EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
883 end
884
885 def runDFGEagerNoCJITValidate(*optionalTestSpecificOptions)
886     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
887 end
888
889 def runFTLEager(*optionalTestSpecificOptions)
890     run("ftl-eager", "--airForceBriggsAllocator=true", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
891 end
892
893 def runFTLEagerWatchdog(*optionalTestSpecificOptions)
894     timeout = rand(100)
895     run("ftl-eager-watchdog-#{timeout}", "--watchdog=#{timeout}", "--watchdog-exception-ok", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
896 end
897
898 def runFTLEagerNoCJITValidate(*optionalTestSpecificOptions)
899     run("ftl-eager-no-cjit", "--validateGraph=true", "--airForceIRCAllocator=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
900 end
901
902 def runFTLEagerNoCJITB3O1(*optionalTestSpecificOptions)
903     run("ftl-eager-no-cjit-b3o1", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS + optionalTestSpecificOptions))
904 end
905
906 def runFTLEagerNoCJITOSRValidation(*optionalTestSpecificOptions)
907     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
908 end
909
910 def runNoCJITNoASO(*optionalTestSpecificOptions)
911     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
912 end
913
914 def runNoCJITNoAccessInlining(*optionalTestSpecificOptions)
915     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
916 end
917
918 def runFTLNoCJITNoAccessInlining(*optionalTestSpecificOptions)
919     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
920 end
921
922 def runFTLNoCJITSmallPool(*optionalTestSpecificOptions)
923     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
924 end
925
926 def runNoCJIT(*optionalTestSpecificOptions)
927     run("no-cjit", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
928 end
929
930 def runDFGMaximalFlushPhase(*optionalTestSpecificOptions)
931     run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
932 end
933
934 def runShadowChicken(*optionalTestSpecificOptions)
935     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true", *optionalTestSpecificOptions)
936 end
937
938 def defaultRun
939     if $quickMode
940         defaultQuickRun
941     else
942         runDefault
943         if $jitTests
944             runNoLLInt
945             runNoCJITValidatePhases
946             runNoCJITCollectContinuously if shouldCollectContinuously?
947             runDFGEager
948             runDFGEagerNoCJITValidate
949             runDFGMaximalFlushPhase
950
951             return if !$isFTLPlatform
952
953             runNoFTL
954             runFTLNoCJITValidate
955             runFTLNoCJITB3O1
956             runFTLNoCJITNoPutStackValidate
957             runFTLNoCJITNoInlineValidate
958             runFTLEager
959             runFTLEagerNoCJITValidate
960             runFTLEagerNoCJITB3O1
961             runFTLNoCJITSmallPool
962         end
963     end
964 end
965
966 def defaultNoNoLLIntRun
967     if $quickMode
968         defaultQuickRun
969     else
970         runDefault
971         if $jitTests
972             runNoCJITValidatePhases
973             runNoCJITCollectContinuously if shouldCollectContinuously?
974             runDFGEager
975             runDFGEagerNoCJITValidate
976             runDFGMaximalFlushPhase
977
978             return if !$isFTLPlatform
979
980             runNoFTL
981             runFTLNoCJITValidate
982             runFTLNoCJITB3O1
983             runFTLNoCJITNoPutStackValidate
984             runFTLNoCJITNoInlineValidate
985             runFTLEager
986             runFTLEagerNoCJITValidate
987             runFTLNoCJITSmallPool
988         end
989     end
990 end
991
992 def defaultQuickRun
993     runDefault
994     if $jitTests
995         runNoCJITValidate
996
997         return if $isFTLPlatform
998
999         runNoFTL
1000         runFTLNoCJITValidate
1001     end
1002 end
1003
1004 def defaultSpotCheckNoMaximalFlush
1005     defaultQuickRun
1006     runNoCJITNoAccessInlining
1007
1008     return if !$isFTLPlatform
1009
1010     runFTLNoCJITOSRValidation
1011     runFTLNoCJITNoAccessInlining
1012     runFTLNoCJITB3O1
1013 end
1014
1015 def defaultSpotCheck
1016     defaultSpotCheckNoMaximalFlush
1017     runDFGMaximalFlushPhase
1018 end
1019
1020 # This is expected to not do eager runs because eager runs can have a lot of recompilations
1021 # for reasons that don't arise in the real world. It's used for tests that assert convergence
1022 # by counting recompilations.
1023 def defaultNoEagerRun
1024     runDefault
1025     if $jitTests
1026         runNoLLInt
1027         runNoCJITValidatePhases
1028         runNoCJITCollectContinuously if shouldCollectContinuously?
1029
1030         return if !$isFTLPlatform
1031
1032         runNoFTL
1033         runFTLNoCJITValidate
1034         runFTLNoCJITNoInlineValidate
1035         runFTLNoCJITB3O1
1036     end
1037 end
1038
1039 def defaultNoSamplingProfilerRun
1040     runDefault
1041     if $jitTests
1042         runNoLLInt
1043         runNoCJITValidatePhases
1044         runNoCJITCollectContinuously if shouldCollectContinuously?
1045         runDFGEager
1046         runDFGEagerNoCJITValidate
1047         runDFGMaximalFlushPhase
1048
1049         return if !$isFTLPlatform
1050
1051         runNoFTL
1052         runFTLNoCJITNoPutStackValidate
1053         runFTLNoCJITNoInlineValidate
1054         runFTLEager
1055         runFTLEagerNoCJITValidate
1056         runFTLNoCJITSmallPool
1057     end
1058 end
1059
1060 def runProfiler
1061     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
1062         skip
1063         return
1064     end
1065
1066     profilerOutput = uniqueFilename(".json")
1067     if $canRunDisplayProfilerOutput
1068         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
1069     else
1070         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
1071         run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
1072     end
1073 end
1074
1075 def runExceptionFuzz
1076     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
1077     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
1078 end
1079
1080 def runExecutableAllocationFuzz(name, *options)
1081     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
1082     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
1083 end
1084
1085 def runTypeProfiler
1086     if !$jitTests
1087         return
1088     end
1089
1090     return if !$isFTLPlatform
1091
1092     run("ftl-no-cjit-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1093     run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
1094 end
1095
1096 def runControlFlowProfiler
1097     if !$jitTests
1098         return
1099     end
1100
1101     return if !$isFTLPlatform
1102
1103     run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1104 end
1105
1106 def runTest262(mode, exception, includeFiles, flags)
1107     failsWithException = exception != "NoException"
1108     isStrict = false
1109     isModule = false
1110     isAsync = false
1111
1112     flags.each {
1113         | flag |
1114         case flag
1115         when :strict
1116             isStrict = true
1117         when :module
1118             isModule = true
1119         when :async
1120             isAsync = true
1121         else
1122             raise "Invalid flag for runTest262, #{flag}"
1123         end
1124     }
1125
1126     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
1127
1128     args = [pathToVM.to_s] + BASE_OPTIONS
1129     args << "--exception=" + exception if failsWithException
1130     args << "--test262-async" if isAsync
1131     args += includeFiles
1132
1133     case mode
1134     when :normal
1135         errorHandler = simpleErrorHandler
1136         outputHandler = silentOutputHandler
1137     when :fail
1138         errorHandler = expectedFailErrorHandler
1139         outputHandler = noisyOutputHandler
1140     when :failDueToOutdatedOrBadTest
1141         errorHandler = expectedFailErrorHandler
1142         outputHandler = noisyOutputHandler
1143     else
1144         raise "Invalid mode: #{mode}"
1145     end
1146
1147     if isStrict
1148         kind = "default-strict"
1149         args << "--strict-file=#{$benchmark}"
1150     else
1151         kind = "default"
1152         if isModule
1153             args << "--module-file=#{$benchmark}"
1154         else
1155             args << $benchmark.to_s
1156         end
1157     end
1158
1159     addRunCommand(kind, args, outputHandler, errorHandler)
1160 end
1161
1162 def prepareTest262Fixture
1163     # This function is used to add the files used by Test262 modules tests.
1164     prepareExtraRelativeFiles([""], $collection)
1165 end
1166
1167 def runES6(mode)
1168     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
1169     case mode
1170     when :normal
1171         errorHandler = simpleErrorHandler
1172     when :fail
1173         errorHandler = expectedFailErrorHandler
1174     when :failDueToOutdatedOrBadTest
1175         errorHandler = expectedFailErrorHandler
1176     else
1177         raise "Invalid mode: #{mode}"
1178     end
1179     addRunCommand("default", args, noisyOutputHandler, errorHandler)
1180 end
1181
1182 def runModules
1183     run("default-modules", "-m")
1184
1185     if !$jitTests
1186         return
1187     end
1188
1189     run("no-llint-modules", "-m", "--useLLInt=false")
1190     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
1191     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
1192     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1193
1194     return if !$isFTLPlatform
1195
1196     run("default-ftl-modules", "-m", *FTL_OPTIONS)
1197     run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1198     run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1199     run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
1200     run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1201     run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1202 end
1203
1204 def runWebAssembly
1205     return if !$jitTests
1206     return if !$isFTLPlatform
1207     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1208     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1209     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1210     run("default-wasm", "-m", *FTL_OPTIONS)
1211     if !$quickMode
1212         run("wasm-no-cjit-yes-tls-context", "-m", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1213         run("wasm-eager-jettison", "-m", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1214         run("wasm-no-call-ic", "-m", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1215         run("wasm-no-tls-context", "-m", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1216         run("wasm-slow-memory", "-m", "--useWebAssemblyFastMemory=false", *FTL_OPTIONS)
1217     end
1218 end
1219
1220 def runWebAssemblyEmscripten(mode)
1221     case mode
1222     when :skip
1223         return
1224     end
1225     return if !$jitTests
1226     return if !$isFTLPlatform
1227     wasm = $benchmark.to_s.sub! '.js', '.wasm'
1228     prepareExtraRelativeFiles([Pathname('..') + wasm], $collection)
1229     run("default-wasm", *FTL_OPTIONS)
1230     if !$quickMode
1231         run("wasm-no-cjit-yes-tls-context", "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1232         run("wasm-eager-jettison", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1233         run("wasm-no-call-ic", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1234         run("wasm-no-tls-context", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1235     end
1236 end
1237
1238 def runWebAssemblySpecTest(mode)
1239     case mode
1240     when :skip
1241         return
1242     end
1243     return if !$jitTests
1244     return if !$isFTLPlatform
1245     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1246
1247     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1248     prepareExtraRelativeFiles(modules.map { |f| "../../" + f }, $collection)
1249
1250     harness = Dir[WASMTESTS_PATH + "spec-harness/" + "*.js"].map { |f| File.basename(f) }
1251     prepareExtraRelativeFiles(harness.map { |f| "../../spec-harness/" + f }, $collection)
1252
1253     runWithOutputHandler("default-wasm", noisyOutputHandler, "../spec-harness.js", *FTL_OPTIONS)
1254     if !$quickMode
1255       runWithOutputHandler("wasm-no-cjit-yes-tls-context", noisyOutputHandler, "../spec-harness.js",  "--useFastTLSForWasmContext=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1256       runWithOutputHandler("wasm-eager-jettison", noisyOutputHandler, "../spec-harness.js", "--forceCodeBlockToJettisonDueToOldAge=true", *FTL_OPTIONS)
1257       runWithOutputHandler("wasm-no-call-ic", noisyOutputHandler, "../spec-harness.js", "--useCallICsForWebAssemblyToJSCalls=false", *FTL_OPTIONS)
1258       runWithOutputHandler("wasm-no-tls-context", noisyOutputHandler, "../spec-harness.js", "--useFastTLSForWasmContext=false", *FTL_OPTIONS)
1259     end
1260 end
1261
1262 def runWebAssemblyLowExecutableMemory(*optionalTestSpecificOptions)
1263     return if !$jitTests
1264     return if !$isFTLPlatform
1265     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1266     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1267     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1268     # Only let WebAssembly get executable memory.
1269     run("default-wasm", "--useConcurrentGC=0" , "--useConcurrentJIT=0", "--jitMemoryReservationSize=15000", "--useBaselineJIT=0", "--useDFGJIT=0", "--useFTLJIT=0", "-m")
1270 end
1271
1272 def runChakra(mode, exception, baselineFile, extraFiles)
1273     raise unless $benchmark.to_s =~ /\.js$/
1274     failsWithException = exception != "NoException"
1275     testName = $~.pre_match
1276
1277     prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
1278     prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)
1279
1280     args = [pathToVM.to_s] + BASE_OPTIONS
1281     args += FTL_OPTIONS if $isFTLPlatform
1282     args += EAGER_OPTIONS
1283     args << "--exception=" + exception if failsWithException
1284     args << "--dumpException" if failsWithException
1285     args += ["jsc-lib.js"]
1286
1287     case mode
1288     when :baseline
1289         prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
1290         errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
1291         outputHandler = noisyOutputHandler
1292     when :pass
1293         errorHandler = chakraPassFailErrorHandler
1294         outputHandler = noisyOutputHandler
1295     when :skipDueToOutdatedOrBadTest
1296         return
1297     when :skip
1298         return
1299     else
1300         raise "Invalid mode: #{mode}"
1301     end
1302
1303     kind = "default"
1304     args << $benchmark.to_s
1305
1306     addRunCommand(kind, args, outputHandler, errorHandler)
1307 end
1308
1309 def runLayoutTest(kind, *options)
1310     raise unless $benchmark.to_s =~ /\.js$/
1311     testName = $~.pre_match
1312     if kind
1313         kind = "layout-" + kind
1314     else
1315         kind = "layout"
1316     end
1317
1318     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1319     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1320
1321     args = [pathToVM.to_s] + BASE_OPTIONS + options +
1322         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1323          $benchmark.to_s,
1324          (Pathname.new("resources") + "standalone-post.js").to_s]
1325     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1326 end
1327
1328 def runLayoutTestNoFTL
1329     runLayoutTest("no-ftl")
1330 end
1331
1332 def runLayoutTestNoLLInt
1333     runLayoutTest("no-llint", "--useLLInt=false")
1334 end
1335
1336 def runLayoutTestNoCJIT
1337     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1338 end
1339
1340 def runLayoutTestDFGEagerNoCJIT
1341     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1342 end
1343
1344 def runLayoutTestDefault
1345     runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
1346 end
1347
1348 def runLayoutTestFTLNoCJIT
1349     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1350 end
1351
1352 def runLayoutTestFTLEagerNoCJIT
1353     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1354 end
1355
1356 def runLayoutTestFTLEagerNoCJITB3O1
1357     runLayoutTest("ftl-eager-no-cjit-b3o1", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + B3O1_OPTIONS))
1358 end
1359
1360 def noFTLRunLayoutTest
1361     if !$jitTests
1362         return
1363     end
1364
1365     runLayoutTestNoLLInt
1366     runLayoutTestNoCJIT
1367     runLayoutTestDFGEagerNoCJIT
1368 end
1369
1370 def defaultQuickRunLayoutTest
1371     runLayoutTestDefault
1372     if $jitTests
1373         if $isFTLPlatform
1374             runLayoutTestNoFTL
1375             runLayoutTestFTLNoCJIT
1376             runLayoutTestFTLEagerNoCJIT
1377         else
1378             noFTLRunLayoutTest
1379         end
1380     end
1381 end
1382
1383 def defaultRunLayoutTest
1384     if $quickMode
1385         defaultQuickRunLayoutTest
1386     else
1387         runLayoutTestDefault
1388         if $jitTests
1389             noFTLRunLayoutTest
1390
1391             return if !$isFTLPlatform
1392
1393             runLayoutTestNoFTL
1394             runLayoutTestFTLNoCJIT
1395             runLayoutTestFTLEagerNoCJIT
1396         end
1397     end
1398 end
1399
1400 def noEagerNoNoLLIntTestsRunLayoutTest
1401     runLayoutTestDefault
1402     if $jitTests
1403         runLayoutTestNoCJIT
1404
1405         return if !$isFTLPlatform
1406
1407         runLayoutTestNoFTL
1408         runLayoutTestFTLNoCJIT
1409     end
1410 end
1411
1412 def noNoLLIntRunLayoutTest
1413     runLayoutTestDefault
1414     if $jitTests
1415         runLayoutTestNoCJIT
1416         runLayoutTestDFGEagerNoCJIT
1417
1418         return if !$isFTLPlatform
1419
1420         runLayoutTestNoFTL
1421         runLayoutTestFTLNoCJIT
1422         runLayoutTestFTLEagerNoCJIT
1423     end
1424 end
1425
1426 def prepareExtraRelativeFiles(extraFiles, destination)
1427     Dir.chdir($outputDir) {
1428         extraFiles.each {
1429             | file |
1430             dest = destination + file
1431             FileUtils.mkdir_p(dest.dirname)
1432             FileUtils.cp $extraFilesBaseDir + file, dest
1433         }
1434     }
1435 end
1436
1437 def baseDirForCollection(collectionName)
1438     Pathname(".tests") + collectionName
1439 end
1440
1441 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1442     raise unless absoluteBase.absolute?
1443     Dir.chdir($outputDir) {
1444         collectionBaseDir = baseDirForCollection($collectionName)
1445         extraFiles.each {
1446             | file |
1447             destination = collectionBaseDir + file
1448             FileUtils.mkdir_p destination.dirname unless destination.directory?
1449             FileUtils.cp absoluteBase + file, destination
1450         }
1451     }
1452 end
1453
1454 def runMozillaTest(kind, mode, extraFiles, *options)
1455     if kind
1456         kind = "mozilla-" + kind
1457     else
1458         kind = "mozilla"
1459     end
1460     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1461     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1462     case mode
1463     when :normal
1464         errorHandler = mozillaErrorHandler
1465     when :negative
1466         errorHandler = mozillaExit3ErrorHandler
1467     when :fail
1468         errorHandler = mozillaFailErrorHandler
1469     when :skip
1470         return
1471     else
1472         raise "Invalid mode: #{mode}"
1473     end
1474     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1475 end
1476
1477 def runMozillaTestDefault(mode, *extraFiles)
1478     runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
1479 end
1480
1481 def runMozillaTestNoFTL(mode, *extraFiles)
1482     runMozillaTest("no-ftl", mode, extraFiles)
1483 end
1484
1485 def runMozillaTestLLInt(mode, *extraFiles)
1486     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1487 end
1488
1489 def runMozillaTestBaselineJIT(mode, *extraFiles)
1490     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1491 end
1492
1493 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1494     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1495 end
1496
1497 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1498     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1499 end
1500
1501 def defaultQuickRunMozillaTest(mode, *extraFiles)
1502     if $jitTests
1503         runMozillaTestDefault(mode, *extraFiles)
1504         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1505     else
1506         runMozillaTestNoFTL(mode, *extraFiles)
1507         if $jitTests
1508             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1509         end
1510     end
1511 end
1512
1513 def defaultRunMozillaTest(mode, *extraFiles)
1514     if $quickMode
1515         defaultQuickRunMozillaTest(mode, *extraFiles)
1516     else
1517         runMozillaTestNoFTL(mode, *extraFiles)
1518         if $jitTests
1519             runMozillaTestLLInt(mode, *extraFiles)
1520             runMozillaTestBaselineJIT(mode, *extraFiles)
1521             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1522             runMozillaTestDefault(mode, *extraFiles)
1523             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
1524         end
1525     end
1526 end
1527
1528 def runNoisyTest(kind, *options)
1529     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
1530 end
1531
1532 def runNoisyTestDefault
1533     runNoisyTest("default", *FTL_OPTIONS)
1534 end
1535
1536 def runNoisyTestNoFTL
1537     runNoisyTest("no-ftl")
1538 end
1539
1540 def runNoisyTestNoCJIT
1541     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1542 end
1543
1544 def runNoisyTestNoCJITB3O1
1545     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + B3O1_OPTIONS))
1546 end
1547
1548 def runNoisyTestEagerNoCJIT
1549     runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1550 end
1551
1552 def defaultRunNoisyTest
1553     runNoisyTestDefault
1554     if $jitTests and $isFTLPlatform
1555         runNoisyTestNoFTL
1556         runNoisyTestNoCJIT
1557         runNoisyTestNoCJITB3O1
1558         runNoisyTestEagerNoCJIT
1559     end
1560 end
1561
1562 def skip
1563     $didAddRunCommand = true
1564     puts "Skipping #{$collectionName}/#{$benchmark}"
1565 end
1566
1567 def allJSFiles(path)
1568     if path.file?
1569         [path]
1570     else
1571         result = []
1572         Dir.foreach(path) {
1573             | filename |
1574             next unless filename =~ /\.js$/
1575             next unless (path + filename).file?
1576             result << path + filename
1577         }
1578         result
1579     end
1580 end
1581
1582 def uniqueifyName(names, name)
1583     result = name.to_s
1584     toAdd = 1
1585     while names[result]
1586         result = "#{name}-#{toAdd}"
1587         toAdd += 1
1588     end
1589     names[result] = true
1590     result
1591 end
1592
1593 def simplifyCollectionName(collectionPath)
1594     outerDir = collectionPath.dirname
1595     name = collectionPath.basename
1596     lastName = name
1597     if collectionPath.directory?
1598         while lastName.to_s =~ /test/
1599             lastName = outerDir.basename
1600             name = lastName + name
1601             outerDir = outerDir.dirname
1602         end
1603     end
1604     uniqueifyName($collectionNames, name)
1605 end
1606
1607 def prepareCollection(name)
1608     FileUtils.mkdir_p $outputDir + name
1609
1610     absoluteCollection = $collection.realpath
1611
1612     Dir.chdir($outputDir) {
1613         bundleDir = baseDirForCollection(name)
1614
1615         # Create the proper directory structures.
1616         FileUtils.mkdir_p bundleDir
1617         if bundleDir.basename == $collection.basename
1618             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1619             $collection = bundleDir
1620         else
1621             FileUtils.cp_r absoluteCollection, bundleDir
1622             $collection = bundleDir + $collection.basename
1623         end
1624
1625         $extraFilesBaseDir = absoluteCollection
1626     }
1627 end
1628
1629 $collectionNames = {}
1630
1631 def handleCollectionFile(collection)
1632     collectionName = simplifyCollectionName(collection)
1633    
1634     paths = {}
1635     subCollections = []
1636     YAML::load(IO::read(collection)).each {
1637         | entry |
1638         if entry["collection"]
1639             subCollections << entry["collection"]
1640             next
1641         end
1642         
1643         if Pathname.new(entry["path"]).absolute?
1644             raise "Absolute path: " + entry["path"] + " in #{collection}"
1645         end
1646         
1647         if paths[entry["path"]]
1648             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1649         end
1650         
1651         subCollection = collection.dirname + entry["path"]
1652         
1653         if subCollection.file?
1654             subCollectionName = Pathname.new(entry["path"]).dirname
1655         else
1656             subCollectionName = entry["path"]
1657         end
1658         
1659         $collection = subCollection
1660         $collectionName = Pathname.new(collectionName)
1661         Pathname.new(subCollectionName).each_filename {
1662             | filename |
1663             next if filename =~ /^\./
1664             $collectionName += filename
1665         }
1666         $collectionName = $collectionName.to_s
1667         
1668         prepareCollection($collectionName)
1669       
1670         Dir.chdir($outputDir) {
1671             pathsToSearch = [$collection]
1672             if entry["tests"]
1673                 if entry["tests"].is_a? Array
1674                     pathsToSearch = entry["tests"].map {
1675                         | testName |
1676                         pathsToSearch[0] + testName
1677                     }
1678                 else
1679                     pathsToSearch[0] += entry["tests"]
1680                 end
1681             end
1682             pathsToSearch.each {
1683                 | pathToSearch |
1684                 allJSFiles(pathToSearch).each {
1685                     | path |
1686                     
1687                     $benchmark = path.basename
1688                     $benchmarkDirectory = path.dirname
1689                     
1690                     $runCommandOptions = {}
1691                     eval entry["cmd"]
1692                 }
1693             }
1694         }
1695     }
1696     
1697     subCollections.each {
1698         | subCollection |
1699         handleCollection(collection.dirname + subCollection)
1700     }
1701 end
1702
1703 def handleCollectionDirectory(collection)
1704     collectionName = simplifyCollectionName(collection)
1705     
1706     $collection = collection
1707     $collectionName = collectionName
1708     prepareCollection(collectionName)
1709    
1710     Dir.chdir($outputDir) {
1711         $benchmarkDirectory = $collection
1712         allJSFiles($collection).each {
1713             | path |
1714             
1715             $benchmark = path.basename
1716             
1717             $runCommandOptions = {}
1718             defaultRun unless parseRunCommands
1719         }
1720     }
1721 end
1722
1723 def handleCollection(collection)
1724     collection = Pathname.new(collection)
1725     
1726     if collection.file?
1727         handleCollectionFile(collection)
1728     else
1729         handleCollectionDirectory(collection)
1730     end
1731 end
1732
1733 def appendFailure(plan)
1734     File.open($outputDir + "failed", "a") {
1735         | outp |
1736         outp.puts plan.name
1737     }
1738     $numFailures += 1
1739 end
1740
1741 def appendPass(plan)
1742     File.open($outputDir + "passed", "a") {
1743         | outp |
1744         outp.puts plan.name
1745     }
1746     $numPasses += 1
1747 end
1748
1749 def appendResult(plan, didPass)
1750     File.open($outputDir + "results", "a") {
1751         | outp |
1752         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1753     }
1754 end
1755
1756 def prepareBundle
1757     raise if $bundle
1758
1759     if $doNotMessWithVMPath
1760         if !$remote and !$tarball
1761             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1762             $jscPath = Pathname.new($jscPath).realpath
1763         else
1764             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1765         end
1766     else
1767         originalJSCPath = $jscPath
1768         vmDir = $outputDir + ".vm"
1769         FileUtils.mkdir_p vmDir
1770         
1771         frameworkPath = frameworkFromJSCPath($jscPath)
1772         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1773         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1774         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1775
1776         if frameworkPath
1777             source = frameworkPath
1778             destination = Pathname.new(".vm")
1779         else
1780             source = originalJSCPath
1781             destination = $jscPath
1782
1783             Dir.chdir($outputDir) {
1784                 FileUtils.mkdir_p $jscPath.dirname
1785             }
1786         end
1787
1788         Dir.chdir($outputDir) {
1789             if $copyVM
1790                 FileUtils.cp_r source, destination
1791             else
1792                 begin 
1793                     FileUtils.ln_s source, destination
1794                 rescue Exception
1795                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1796                     FileUtils.cp_r source, destination
1797                 end
1798             end
1799
1800             if $remote and $hostOS == "linux"
1801                 begin
1802                     dependencies = `ldd #{source}`
1803                     dependencies.split(/\n/).each {
1804                         | dependency |
1805                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1806                     }
1807                 rescue
1808                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1809                 end
1810             end
1811         }
1812     end
1813     
1814     Dir.chdir($outputDir) {
1815         FileUtils.cp_r HELPERS_PATH, ".helpers"
1816     }
1817
1818     ARGV.each {
1819         | collection |
1820         handleCollection(collection)
1821     }
1822
1823     puts
1824 end
1825
1826 def cleanOldResults
1827     raise unless $bundle
1828
1829     eachResultFile($outputDir) {
1830         | path |
1831         FileUtils.rm_f path
1832     }
1833 end
1834
1835 def cleanEmptyResultFiles
1836     eachResultFile($outputDir) {
1837         | path |
1838         next unless path.basename.to_s =~ /\.out$/
1839         next unless FileTest.size(path) == 0
1840         FileUtils.rm_f path
1841     }
1842 end
1843
1844 def eachResultFile(startingDir, &block)
1845     dirsToClean = [startingDir]
1846     until dirsToClean.empty?
1847         nextDir = dirsToClean.pop
1848         Dir.foreach(nextDir) {
1849             | entry |
1850             next if entry =~ /^\./
1851             path = nextDir + entry
1852             if path.directory?
1853                 dirsToClean.push(path)
1854             else
1855                 block.call(path)
1856             end
1857         }
1858     end
1859 end
1860
1861 def prepareTestRunner
1862     raise if $bundle
1863
1864     $runlist.each_with_index {
1865         | plan, index |
1866         plan.index = index
1867     }
1868
1869     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1870     toDelete = []
1871     Dir.foreach($runnerDir) {
1872         | filename |
1873         if filename =~ /^test_/
1874             toDelete << filename
1875         end
1876     }
1877     
1878     toDelete.each {
1879         | filename |
1880         File.unlink($runnerDir + filename)
1881     }
1882
1883     $runlist.each {
1884         | plan |
1885         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1886     }
1887
1888     case $testRunnerType
1889     when :make
1890         prepareMakeTestRunner
1891     when :shell
1892         prepareShellTestRunner
1893     else
1894         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1895     end
1896 end
1897
1898 def prepareShellTestRunner
1899     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
1900 end
1901
1902 def prepareMakeTestRunner
1903     # The goals of our parallel test runner are scalability and simplicity. The
1904     # simplicity part is particularly important. We don't want to have to have
1905     # a full-time contributor just philosophising about parallel testing.
1906     #
1907     # As such, we just pass off all of the hard work to 'make'. This creates a
1908     # dummy directory ("$outputDir/.runner") in which we create a dummy
1909     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
1910     # That is, for each test we know we will run, there is a rule in the
1911     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
1912     # Makefile results in 'make' doing all of the hard work:
1913     #
1914     # - Load balancing just works. Most systems have a great load balancer in
1915     #   'make'. If your system doesn't then just install a real 'make'.
1916     #
1917     # - Interruptions just work. For example Ctrl-C handling in 'make' is
1918     #   exactly right. You don't have to worry about zombie processes.
1919     #
1920     # We then do some tricks to make failure detection work and to make this
1921     # totally sound. If a test fails, we don't want the whole 'make' job to
1922     # stop. We also don't have any facility for makefile-escaping of path names.
1923     # We do have such a thing for shell-escaping, though. We fix both problems
1924     # by having the actual work for each of the test rules be done in a shell
1925     # script on the side. There is one such script per test. The script responds
1926     # to failure by printing something on the console and then touching a
1927     # failure file for that test, but then still returns 0. This makes 'make'
1928     # continue past that failure and complete all the tests anyway.
1929     #
1930     # In the end, this script collects all of the failures by searching for
1931     # files in the .runner directory whose name matches /^test_fail_/, where
1932     # the thing after the 'fail_' is the test index. Those are the files that
1933     # would be created by the test scripts if they detect failure. We're
1934     # basically using the filesystem as a concurrent database of test failures.
1935     # Even if two tests fail at the same time, since they're touching different
1936     # files we won't miss any failures.
1937     runIndices = []
1938     $runlist.each {
1939         | plan |
1940         runIndices << plan.index
1941     }
1942     
1943     File.open($runnerDir + "Makefile", "w") {
1944         | outp |
1945         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1946         runIndices.each {
1947             | index |
1948             plan = $runlist[index]
1949             outp.puts "test_done_#{index}:"
1950             outp.puts "\tsh test_script_#{plan.index}"
1951         }
1952     }
1953 end
1954
1955 def cleanRunnerDirectory
1956     raise unless $bundle
1957     Dir.foreach($runnerDir) {
1958         | filename |
1959         next unless filename =~ /^test_fail/
1960         FileUtils.rm_f $runnerDir + filename
1961     }
1962 end
1963
1964 def sshRead(cmd)
1965     raise unless $remote
1966
1967     result = ""
1968     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1969       | inp |
1970       inp.each_line {
1971         | line |
1972         result += line
1973       }
1974     }
1975     raise "#{$?}" unless $?.success?
1976     result
1977 end
1978
1979 def runCommandOnTester(cmd)
1980     if $remote
1981         result = sshRead(cmd)
1982     else
1983         result = `#{cmd}`
1984     end
1985 end
1986
1987 def numberOfProcessors
1988     if $hostOS == "windows"
1989         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1990     else
1991         begin
1992             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1993         rescue
1994             numProcessors = 0
1995         end
1996
1997         if numProcessors == 0
1998             begin
1999                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
2000             rescue
2001                 numProcessors == 0
2002             end
2003         end
2004     end
2005
2006     if numProcessors == 0
2007         numProcessors = 1
2008     end
2009     return numProcessors
2010 end
2011
2012 def runAndMonitorTestRunnerCommand(*cmd)
2013     numberOfTests = 0
2014     Dir.chdir($runnerDir) {
2015         # -1 for the runscript, and -2 for '..' and '.'
2016         numberOfTests = Dir.entries(".").count - 3
2017     }
2018     unless $progressMeter
2019         mysys(cmd.join(' '))
2020     else
2021        running = {}
2022        didRun = {}
2023        didFail = {}
2024        blankLine = true
2025        prevStringLength = 0
2026        IO.popen(cmd.join(' '), mode="r") {
2027            | inp |
2028            inp.each_line {
2029                | line |
2030                line = line.scrub.chomp
2031                if line =~ /^Running /
2032                    running[$~.post_match] = true
2033                elsif line =~ /^PASS: /
2034                    didRun[$~.post_match] = true
2035                elsif line =~ /^FAIL: /
2036                    didRun[$~.post_match] = true
2037                    didFail[$~.post_match] = true
2038                else
2039                    unless blankLine
2040                        print("\r" + " " * prevStringLength + "\r")
2041                    end
2042                    puts line
2043                    blankLine = true
2044                end
2045
2046                def lpad(str, chars)
2047                    str = str.to_s
2048                    if str.length > chars
2049                        str
2050                    else
2051                       "%#{chars}s"%(str)
2052                    end
2053                end
2054
2055                string  = ""
2056                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
2057                unless didFail.empty?
2058                    string += " (failed #{didFail.size})"
2059                end
2060                string += " "
2061                (running.size - didRun.size).times {
2062                    string += "."
2063                }
2064                if string.length < prevStringLength
2065                    print string
2066                    print(" " * (prevStringLength - string.length))
2067                end
2068                print string
2069                prevStringLength = string.length
2070                blankLine = false
2071                $stdout.flush
2072            }
2073        }
2074        puts
2075        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
2076     end
2077 end
2078
2079 def runTestRunner
2080     case $testRunnerType
2081     when :shell
2082         testRunnerCommand = "sh runscript"
2083     when :make
2084         testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
2085     else
2086         raise "Unknown test runner type: #{$testRunnerType.to_s}"
2087     end
2088
2089     if $remote
2090         if !$remoteDirectory
2091             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
2092         end
2093         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
2094         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
2095         remoteScript = "\""
2096         remoteScript += "cd #{$remoteDirectory} && "
2097         remoteScript += "rm -rf #{$outputDir.basename} && "
2098         remoteScript += "tar xzf #{$tarFileName} && "
2099         remoteScript += "cd #{$outputDir.basename}/.runner && "
2100         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
2101         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
2102         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
2103         $envVars.each { |var| remoteScript += "export " << var << "\n" }
2104         remoteScript += "#{testRunnerCommand}\""
2105         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
2106     else
2107         Dir.chdir($runnerDir) {
2108             runAndMonitorTestRunnerCommand(testRunnerCommand)
2109         }
2110     end
2111 end
2112
2113 def detectFailures
2114     raise if $bundle
2115
2116     failures = []
2117     
2118     if $remote
2119         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
2120         output.split(/\n/).each {
2121             | line |
2122             next unless line =~ /test_fail_/
2123             failures << $~.post_match.to_i
2124         }
2125     else
2126         Dir.foreach($runnerDir) {
2127             | filename |
2128             next unless filename =~ /test_fail_/
2129             failures << $~.post_match.to_i
2130         }
2131     end
2132
2133     failureSet = {}
2134
2135     failures.each {
2136         | failure | 
2137         appendFailure($runlist[failure])
2138         failureSet[failure] = true
2139     }
2140
2141     familyMap = {}
2142     $runlist.each_with_index {
2143         | plan, index |
2144         unless familyMap[plan.family]
2145             familyMap[plan.family] = []
2146         end
2147         if failureSet[index]
2148             appendResult(plan, false)
2149             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
2150             next
2151         else
2152             appendResult(plan, true)
2153             familyMap[plan.family] << {:result => "PASS", :plan => plan};
2154         end
2155         appendPass(plan)
2156     }
2157
2158     File.open($outputDir + "resultsByFamily", "w") {
2159         | outp |
2160         first = true
2161         familyMap.keys.sort.each {
2162             | familyName |
2163             if first
2164                 first = false
2165             else
2166                 outp.puts
2167             end
2168             
2169             outp.print "#{familyName}:"
2170
2171             numPassed = 0
2172             familyMap[familyName].each {
2173                 | entry |
2174                 if entry[:result] == "PASS"
2175                     numPassed += 1
2176                 end
2177             }
2178
2179             if numPassed == familyMap[familyName].size
2180                 outp.puts " PASSED"
2181             elsif numPassed == 0
2182                 outp.puts " FAILED"
2183             else
2184                 outp.puts
2185                 familyMap[familyName].each {
2186                     | entry |
2187                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
2188                 }
2189             end
2190         }
2191     }
2192 end
2193
2194 def compressBundle
2195     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
2196     $stderr.puts ">> #{cmd}" if $verbosity >= 2
2197     raise unless system(cmd)
2198 end
2199
2200 def clean(file)
2201     FileUtils.rm_rf file unless $bundle
2202 end
2203
2204 clean($outputDir + "failed")
2205 clean($outputDir + "passed")
2206 clean($outputDir + "results")
2207 clean($outputDir + "resultsByFamily")
2208 clean($outputDir + ".vm")
2209 clean($outputDir + ".helpers")
2210 clean($outputDir + ".runner")
2211 clean($outputDir + ".tests")
2212 clean($outputDir + "_payload")
2213
2214 Dir.mkdir($outputDir) unless $outputDir.directory?
2215
2216 $outputDir = $outputDir.realpath
2217 $runnerDir = $outputDir + ".runner"
2218
2219 if !$numChildProcesses
2220     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
2221         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
2222     else
2223         $numChildProcesses = numberOfProcessors
2224     end
2225 end
2226
2227 if ENV["JSCTEST_timeout"]
2228     # In the worst case, the processors just interfere with each other.
2229     # Increase the timeout proportionally to the number of processors.
2230     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
2231 end
2232
2233 def runBundle
2234     raise unless $bundle
2235
2236     cleanRunnerDirectory
2237     cleanOldResults
2238     runTestRunner
2239     cleanEmptyResultFiles
2240 end
2241
2242 def runNormal
2243     raise if $bundle or $tarball
2244
2245     prepareBundle
2246     prepareTestRunner
2247     runTestRunner
2248     cleanEmptyResultFiles
2249     detectFailures
2250 end
2251
2252 def runTarball
2253     raise unless $tarball
2254
2255     prepareBundle 
2256     prepareTestRunner
2257     compressBundle
2258 end
2259
2260 def runRemote
2261     raise unless $remote
2262
2263     prepareBundle
2264     prepareTestRunner
2265     compressBundle
2266     runTestRunner
2267     detectFailures
2268 end
2269
2270 puts
2271 if $bundle
2272     runBundle
2273 elsif $remote
2274     runRemote
2275 elsif $tarball
2276     runTarball
2277 else
2278     runNormal
2279 end