The collector thread should only start when the mutator doesn't have heap access
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2016 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
47 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
48 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
49 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
50 WASMTESTS_PATH = WEBKIT_PATH + "JSTests/wasm"
51 CHAKRATESTS_PATH = WEBKIT_PATH + "JSTests/ChakraCore/test"
52 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
53 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
54
55 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
56
57 begin
58     require 'shellwords'
59 rescue Exception => e
60     $stderr.puts "Warning: did not find shellwords, not running any tests."
61     exit 0
62 end
63
64 $canRunDisplayProfilerOutput = false
65
66 begin
67     require 'rubygems'
68     require 'json'
69     require 'highline'
70     $canRunDisplayProfilerOutput = true
71 rescue Exception => e
72     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
73     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
74     $stderr.puts "Error: #{e.inspect}"
75 end
76
77 def printCommandArray(*cmd)
78     begin
79         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
80     rescue
81         commandArray = cmd.join(' ')
82     end
83     $stderr.puts ">> #{commandArray}"
84 end
85
86 def mysys(*cmd)
87     printCommandArray(*cmd) if $verbosity >= 1
88     raise "Command failed: #{$?.inspect}" unless system(*cmd)
89 end
90
91 def escapeAll(array)
92     array.map {
93         | v |
94         raise "Detected a non-string in #{inspect}" unless v.is_a? String
95         Shellwords.shellescape(v)
96     }.join(' ')
97 end
98
99
100 $jscPath = nil
101 $doNotMessWithVMPath = false
102 $jitTests = true
103 $memoryLimited = false
104 $outputDir = Pathname.new("results")
105 $verbosity = 0
106 $bundle = nil
107 $tarball = false
108 $tarFileName = "payload.tar.gz"
109 $copyVM = false
110 $testRunnerType = nil
111 $remoteUser = nil
112 $remoteHost = nil
113 $remotePort = nil
114 $remoteDirectory = nil
115 $architecture = nil
116 $hostOS = nil
117 $filter = nil
118 $envVars = []
119 $quickMode = false
120 $buildType = "release"
121 $forceCollectContinuously = false
122
123 def usage
124     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
125     puts
126     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
127     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
128     puts "                            --jsc specifies an already present JavaScriptCore to test."
129     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
130     puts "                            Skip tests tagged with //@skip if $memoryLimited"
131     puts "--no-jit                    Do not run JIT specific tests."
132     puts "--force-collectContinuously Enable the collectContinuously mode even if disabled on this"
133     puts "                            platform."
134     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
135     puts "--verbose            (-v)   Print more things while running."
136     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
137     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
138     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
139     puts "                            e.g. x86, x86_64, arm."
140     puts "--os                        Specify os instead of determining from JavaScriptCore build."
141     puts "                            e.g. darwin, linux & windows."
142     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
143     puts "                            In general the shell runner is slower than the make runner."
144     puts "--make-runner               Uses the faster make-based runner."
145     puts "--remote                    Specify a remote host on which to run tests from command line argument."
146     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
147     puts "--child-processes    (-c)   Specify the number of child processes."
148     puts "--filter                    Only run tests whose name matches the given regular expression."
149     puts "--help               (-h)   Print this message."
150     puts "--env-vars                  Add a list of environment variables to set before running jsc."
151     puts "                            Each environment variable should be separated by a space."
152     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
153     puts "                            it will override the default value."
154     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
155     exit 1
156 end
157
158 jscArg = nil
159
160 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
161                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
162                ['--no-copy', GetoptLong::NO_ARGUMENT],
163                ['--memory-limited', GetoptLong::NO_ARGUMENT],
164                ['--no-jit', GetoptLong::NO_ARGUMENT],
165                ['--force-collectContinuously', GetoptLong::NO_ARGUMENT],
166                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
167                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
168                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
169                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
170                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
171                ['--os', GetoptLong::REQUIRED_ARGUMENT],
172                ['--shell-runner', GetoptLong::NO_ARGUMENT],
173                ['--make-runner', GetoptLong::NO_ARGUMENT],
174                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
175                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
176                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
177                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
178                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
179                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
180                ['--debug', GetoptLong::NO_ARGUMENT],
181                ['--release', GetoptLong::NO_ARGUMENT],
182                ['--quick', '-q', GetoptLong::NO_ARGUMENT]).each {
183     | opt, arg |
184     case opt
185     when '--help'
186         usage
187     when '--jsc'
188         jscArg = arg
189     when '--no-copy'
190         $doNotMessWithVMPath = true
191     when '--output-dir'
192         $outputDir = Pathname.new(arg)
193     when '--memory-limited'
194         $memoryLimited = true
195     when '--no-jit'
196         $jitTests = false
197     when '--force-collectContinuously'
198         $forceCollectContinuously = true;
199     when '--verbose'
200         $verbosity += 1
201     when '--run-bundle'
202         $bundle = Pathname.new(arg)
203     when '--tarball'
204         $tarball = true
205         $copyVM = true
206         $tarFileName = arg unless arg == ''
207     when '--force-vm-copy'
208         $copyVM = true
209     when '--shell-runner'
210         $testRunnerType = :shell
211     when '--make-runner'
212         $testRunnerType = :make
213     when '--remote'
214         $copyVM = true
215         $tarball = true
216         $remote = true
217         uri = URI("ssh://" + arg)
218         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
219     when '--remote-config-file'
220         $remoteConfigFile = arg
221     when '--child-processes'
222         $numChildProcesses = arg.to_i
223     when '--filter'
224         $filter = Regexp.new(arg)
225     when '--arch'
226         $architecture = arg
227     when '--os'
228         $hostOS = arg
229     when '--env-vars'
230         $envVars = arg.gsub(/\s+/, ' ').split(' ')
231     when '--quick'
232         $quickMode = true
233     when '--debug'
234         $buildType = "debug"
235     when '--release'
236         $buildType = "release"
237     end
238 }
239
240 if $remoteConfigFile
241     file = File.read($remoteConfigFile)
242     config = JSON.parse(file)
243
244     if !$remote and config['remote']
245         $copyVM = true
246         $tarball = true
247         $remote = true
248         uri = URI("ssh://" + config['remote'])
249         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
250     end
251
252     if config['remoteDirectory']
253         $remoteDirectory = config['remoteDirectory']
254     end
255 end
256
257 unless jscArg
258     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
259     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
260     command += ($buildType == "release") ? " --release" : " --debug"
261     command += " --executablePath"
262
263     output = `#{command}`.split("\n")
264     if !output.length
265         $stderr.puts "Error: must specify --jsc <path>"
266         exit 1
267     end
268
269     output = output[0]
270     jscArg = Pathname.new(output).join("jsc")
271     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
272     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
273     if !File.file?(jscArg)
274         $stderr.puts "Error: must specify --jsc <path>"
275         exit 1
276     end
277
278     puts "Using the following jsc path: #{jscArg}"
279 end
280
281 if $doNotMessWithVMPath
282     $jscPath = Pathname.new(jscArg)
283 else
284     $jscPath = Pathname.new(jscArg).realpath
285 end
286
287 $progressMeter = ($verbosity == 0 and $stdout.tty?)
288
289 if $bundle
290     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
291     $outputDir = $bundle
292 end
293
294 # Try to determine architecture. Return nil on failure.
295 def machOArchitectureCode
296     begin 
297         otoolLines = `otool -aSfh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
298         otoolLines.each_with_index {
299             | value, index |
300             if value =~ /magic/ and value =~ /cputype/
301                 return otoolLines[index + 1].split[1].to_i
302             end
303         }
304     rescue
305         $stderr.puts "Warning: unable to execute otool."
306     end
307     $stderr.puts "Warning: unable to determine architecture."
308     nil
309 end
310
311 def determineArchitectureFromMachOBinary
312     code = machOArchitectureCode
313     return nil unless code
314     is64BitFlag = 0x01000000
315     case code
316     when 7
317         "x86"
318     when 7 | is64BitFlag
319         "x86-64"
320     when 12
321         "arm"
322     when 12 | is64BitFlag
323         "arm64"
324     else
325         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
326         nil
327     end
328 end
329
330 def determineArchitectureFromELFBinary
331     f = File.open($jscPath.to_s)
332     data = f.read(19)
333
334     if !(data[0,4] == "\x7F\x45\x4C\x46")
335         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
336         return nil
337     end
338
339     code = data[18].ord
340     case code
341     when 3
342         "x86"
343     when 62
344         "x86-64"
345     when 40
346         "arm"
347     when 183
348         "arm64"
349     else
350         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
351         nil
352     end
353 end
354
355 def determineArchitectureFromPEBinary
356     f = File.open($jscPath.to_s)
357     data = f.read(1024)
358
359     if !(data[0, 2] == "MZ")
360         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
361         return nil
362     end
363
364     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
365
366     if !(data[peHeaderAddr, 4] == "PE\0\0")
367         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
368         return nil
369     end
370
371     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
372
373     case machine
374     when 0x014c
375         "x86"
376     when 0x8664
377         "x86-64"
378     else
379         $stderr.puts "Warning: unsupported machine type: #{machine}"
380         nil
381     end
382 end
383
384 def determineArchitecture
385     case $hostOS
386     when "darwin"
387         determineArchitectureFromMachOBinary
388     when "linux"
389         determineArchitectureFromELFBinary
390     when "windows"
391         determineArchitectureFromPEBinary
392     else
393         $stderr.puts "Warning: unable to determine architecture on this platform."
394         nil
395     end
396 end
397
398 def determineOS
399     case RbConfig::CONFIG["host_os"]
400     when /darwin/i
401         "darwin"
402     when /linux/i
403         "linux"
404     when /mswin|mingw|cygwin/
405         "windows"
406     else
407         $stderr.puts "Warning: unable to determine host operating system"
408         nil
409     end
410 end
411
412 $hostOS = determineOS unless $hostOS
413 $architecture = determineArchitecture unless $architecture
414 $isFTLPlatform = !($architecture == "x86" || $architecture == "arm" || $hostOS == "windows" || $hostOS == "linux" && $architecture == "arm64")
415
416 if !$testRunnerType
417     if $remote and $hostOS == "darwin"
418         $testRunnerType = :shell
419     else
420         $testRunnerType = :make
421     end
422 end
423
424 $numFailures = 0
425 $numPasses = 0
426
427 # We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster.
428 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--maxPerThreadStackUsage=1572864"]
429 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000", "--useEagerCodeBlockJettisonTiming=true"]
430 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--scribbleFreeCells=true"]
431 FTL_OPTIONS = ["--useFTLJIT=true"]
432
433 def shouldCollectContinuously?
434     $buildType == "release" or $forceCollectContinuously
435 end
436
437 COLLECT_CONTINUOUSLY_OPTIONS = shouldCollectContinuously? ? ["--collectContinuously=true", "--useGenerationalGC=false"] : []
438
439 $runlist = []
440
441 def frameworkFromJSCPath(jscPath)
442     parentDirectory = jscPath.dirname
443     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
444         parentDirectory.dirname
445     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
446         jscPath.dirname + "JavaScriptCore.framework"
447     else
448         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
449         nil
450     end
451 end
452
453 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
454     dir = Pathname.new(".")
455     $benchmarkDirectory.each_filename {
456         | pathComponent |
457         dir += ".."
458     }
459     dir + resourcePath
460 end
461
462 def pathToVM
463     pathToBundleResourceFromBenchmarkDirectory($jscPath)
464 end
465
466 def pathToHelpers
467     pathToBundleResourceFromBenchmarkDirectory(".helpers")
468 end
469
470 def prefixCommand(prefix)
471     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
472 end
473
474 def redirectAndPrefixCommand(prefix)
475     prefixCommand(prefix) + " 2>&1"
476 end
477
478 def pipeAndPrefixCommand(outputFilename, prefix)
479     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
480 end
481
482 # Output handler for tests that are expected to be silent.
483 def silentOutputHandler
484     Proc.new {
485         | name |
486         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
487     }
488 end
489
490 # Output handler for tests that are expected to produce meaningful output.
491 def noisyOutputHandler
492     Proc.new {
493         | name |
494         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
495     }
496 end
497
498 # Error handler for tests that fail exactly when they return non-zero exit status.
499 # This is useful when a test is expected to fail.
500 def simpleErrorHandler
501     Proc.new {
502         | outp, plan |
503         outp.puts "if test -e #{plan.failFile}"
504         outp.puts "then"
505         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
506         outp.puts "    " + plan.failCommand
507         outp.puts "else"
508         outp.puts "    " + plan.successCommand
509         outp.puts "fi"
510     }
511 end
512
513 # Error handler for tests that fail exactly when they return zero exit status.
514 def expectedFailErrorHandler
515     Proc.new {
516         | outp, plan |
517         outp.puts "if test -e #{plan.failFile}"
518         outp.puts "then"
519         outp.puts "    " + plan.successCommand
520         outp.puts "else"
521         outp.puts "    (echo ERROR: Unexpected exit code: 0) | " + redirectAndPrefixCommand(plan.name)
522         outp.puts "    " + plan.failCommand
523         outp.puts "fi"
524     }
525 end
526
527 # Error handler for tests that fail exactly when they return non-zero exit status and produce
528 # lots of spew. This will echo that spew when the test fails.
529 def noisyErrorHandler
530     Proc.new {
531         | outp, plan |
532         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
533     
534         outp.puts "if test -e #{plan.failFile}"
535         outp.puts "then"
536         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
537         outp.puts "    " + plan.failCommand
538         outp.puts "else"
539         outp.puts "    " + plan.successCommand
540         outp.puts "fi"
541     }
542 end
543
544 # Error handler for tests that diff their output with some expectation.
545 def diffErrorHandler(expectedFilename)
546     Proc.new {
547         | outp, plan |
548         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
549         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
550         
551         outp.puts "if test -e #{plan.failFile}"
552         outp.puts "then"
553         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
554         outp.puts "    " + plan.failCommand
555         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
556         outp.puts "then"
557         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
558         outp.puts "    if [ $? -eq 0 ]"
559         outp.puts "    then"
560         outp.puts "    " + plan.successCommand
561         outp.puts "    else"
562         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
563         outp.puts "        " + plan.failCommand
564         outp.puts "    fi"
565         outp.puts "else"
566         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
567         outp.puts "    " + plan.failCommand
568         outp.puts "fi"
569     }
570 end
571
572 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
573 # tests.
574 def mozillaErrorHandler
575     Proc.new {
576         | outp, plan |
577         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
578
579         outp.puts "if test -e #{plan.failFile}"
580         outp.puts "then"
581         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
582         outp.puts "    " + plan.failCommand
583         outp.puts "elif grep -i -q failed! #{outputFilename}"
584         outp.puts "then"
585         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
586         outp.puts "    " + plan.failCommand
587         outp.puts "else"
588         outp.puts "    " + plan.successCommand
589         outp.puts "fi"
590     }
591 end
592
593 # Error handler for tests that report error by saying "failed!", and are expected to
594 # fail. This is used by Mozilla tests.
595 def mozillaFailErrorHandler
596     Proc.new {
597         | outp, plan |
598         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
599
600         outp.puts "if test -e #{plan.failFile}"
601         outp.puts "then"
602         outp.puts "    " + plan.successCommand
603         outp.puts "elif grep -i -q failed! #{outputFilename}"
604         outp.puts "then"
605         outp.puts "    " + plan.successCommand
606         outp.puts "else"
607         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
608         outp.puts "    " + plan.failCommand
609         outp.puts "fi"
610     }
611 end
612
613 # Error handler for tests that report error by saying "failed!", and are expected to have
614 # an exit code of 3.
615 def mozillaExit3ErrorHandler
616     Proc.new {
617         | outp, plan |
618         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
619
620         outp.puts "if test -e #{plan.failFile}"
621         outp.puts "then"
622         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
623         outp.puts "    then"
624         outp.puts "        if grep -i -q failed! #{outputFilename}"
625         outp.puts "        then"
626         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
627         outp.puts "            " + plan.failCommand
628         outp.puts "        else"
629         outp.puts "            " + plan.successCommand
630         outp.puts "        fi"
631         outp.puts "    else"
632         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
633         outp.puts "        " + plan.failCommand
634         outp.puts "    fi"
635         outp.puts "else"
636         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
637         outp.puts "    " + plan.failCommand
638         outp.puts "fi"
639     }
640 end
641
642 # Error handler for tests that report success by saying "Passed" or error by saying "FAILED".
643 # This is used by Chakra tests.
644 def chakraPassFailErrorHandler
645     Proc.new {
646         | outp, plan |
647         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
648
649         outp.puts "if test -e #{plan.failFile}"
650         outp.puts "then"
651         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
652         outp.puts "    " + plan.failCommand
653         outp.puts "elif grep -i -q FAILED #{outputFilename}"
654         outp.puts "then"
655         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
656         outp.puts "    " + plan.failCommand
657         outp.puts "else"
658         outp.puts "    " + plan.successCommand
659         outp.puts "fi"
660     }
661 end
662
663 $runCommandOptions = {}
664
665 class Plan
666     attr_reader :directory, :arguments, :family, :name, :outputHandler, :errorHandler
667     attr_accessor :index
668     
669     def initialize(directory, arguments, family, name, outputHandler, errorHandler)
670         @directory = directory
671         @arguments = arguments
672         @family = family
673         @name = name
674         @outputHandler = outputHandler
675         @errorHandler = errorHandler
676         @isSlow = !!$runCommandOptions[:isSlow]
677     end
678     
679     def shellCommand
680         # It's important to remember that the test is actually run in a subshell, so if we change directory
681         # in the subshell when we return we will be in our original directory. This is nice because we don't
682         # have to bend over backwards to do things relative to the root.
683         script = "(cd ../#{Shellwords.shellescape(@directory.to_s)} && ("
684         $envVars.each { |var| script += "export " << var << "; " }
685         script += "\"$@\" " + escapeAll(@arguments) + "))"
686         return script
687     end
688     
689     def reproScriptCommand
690         # We have to find our way back to the .runner directory since that's where all of the relative
691         # paths assume they start out from.
692         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
693         script += "cd $CURRENT_DIR\n"
694         Pathname.new(@name).dirname.each_filename {
695             | pathComponent |
696             script += "cd ..\n"
697         }
698         script += "cd .runner\n"
699
700         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
701         script += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])}\n"
702         $envVars.each { |var| script += "export " << var << "\n" }
703         script += "#{shellCommand} || exit 1"
704         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
705     end
706     
707     def failCommand
708         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
709     end
710     
711     def successCommand
712         if $progressMeter or $verbosity >= 2
713             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
714         else
715             "rm -f #{failFile}"
716         end
717     end
718     
719     def failFile
720         "test_fail_#{@index}"
721     end
722     
723     def writeRunScript(filename)
724         File.open(filename, "w") {
725             | outp |
726             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
727             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
728             cmd += @outputHandler.call(@name)
729             if $verbosity >= 3
730                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
731             end
732             outp.puts cmd
733             @errorHandler.call(outp, self)
734         }
735     end
736 end
737
738 $uniqueFilenameCounter = 0
739 def uniqueFilename(extension)
740     payloadDir = $outputDir + "_payload"
741     Dir.mkdir payloadDir unless payloadDir.directory?
742     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
743     $uniqueFilenameCounter += 1
744     result
745 end
746
747 def baseOutputName(kind)
748     "#{$collectionName}/#{$benchmark}.#{kind}"
749 end
750
751 def addRunCommand(kind, command, outputHandler, errorHandler)
752     $didAddRunCommand = true
753     name = baseOutputName(kind)
754     if $filter and name !~ $filter
755         return
756     end
757     plan = Plan.new(
758         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
759         errorHandler)
760     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
761         $runlist.unshift plan
762     else
763         $runlist << plan
764     end
765 end
766
767 # Returns true if there were run commands found in the file ($benchmarkDirectory +
768 # $benchmark), in which case those run commands have already been executed. Otherwise
769 # returns false, in which case you're supposed to add your own run commands.
770 def parseRunCommands
771     oldDidAddRunCommand = $didAddRunCommand
772     $didAddRunCommand = false
773
774     Dir.chdir($outputDir) {
775         File.open($benchmarkDirectory + $benchmark) {
776             | inp |
777             inp.each_line {
778                 | line |
779                 begin
780                     doesMatch = line =~ /^\/\/@/
781                 rescue Exception => e
782                     # Apparently this happens in the case of some UTF8 stuff in some files, where
783                     # Ruby tries to be strict and throw exceptions.
784                     next
785                 end
786                 next unless doesMatch
787                 eval $~.post_match
788             }
789         }
790     }
791
792     result = $didAddRunCommand
793     $didAddRunCommand = result or oldDidAddRunCommand
794     result
795 end
796
797 def slow!
798     $runCommandOptions[:isSlow] = true
799 end
800
801 def runWithOutputHandler(kind, outputHandler, *options)
802     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], outputHandler, simpleErrorHandler)
803 end
804
805 def run(kind, *options)
806     runWithOutputHandler(kind, silentOutputHandler, *options)
807 end
808
809 def runNoFTL(*optionalTestSpecificOptions)
810     run("no-ftl", *optionalTestSpecificOptions)
811 end
812
813 def runWithRAMSize(size, *optionalTestSpecificOptions)
814     run("ram-size-#{size}", "--forceRAMSize=#{size}", *optionalTestSpecificOptions)
815 end
816
817 def runOneLargeHeap(*optionalTestSpecificOptions)
818     if $memoryLimited
819         $didAddRunCommand = true
820         puts "Skipping #{$collectionName}/#{$benchmark}"
821     else
822         run("default", *optionalTestSpecificOptions)
823     end
824 end
825
826 def runNoJIT(*optionalTestSpecificOptions)
827     run("no-jit", "--useJIT=false", *optionalTestSpecificOptions)
828 end
829
830 def runNoLLInt(*optionalTestSpecificOptions)
831     if $jitTests
832         run("no-llint", "--useLLInt=false", *optionalTestSpecificOptions)
833     end
834 end
835
836 def runNoCJITValidate(*optionalTestSpecificOptions)
837     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
838 end
839
840 def runNoCJITValidatePhases(*optionalTestSpecificOptions)
841     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", "--useSourceProviderCache=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
842 end
843
844 def runNoCJITCollectContinuously(*optionalTestSpecificOptions)
845     run("no-cjit-collect-continuously", *(NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
846 end
847
848 def runDefault(*optionalTestSpecificOptions)
849     run("default", *(FTL_OPTIONS + optionalTestSpecificOptions))
850 end
851
852 def runFTLNoCJIT(*optionalTestSpecificOptions)
853     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
854 end
855
856 def runFTLNoCJITValidate(*optionalTestSpecificOptions)
857     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
858 end
859
860 def runFTLNoCJITNoPutStackValidate(*optionalTestSpecificOptions)
861     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
862 end
863
864 def runFTLNoCJITNoInlineValidate(*optionalTestSpecificOptions)
865     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
866 end
867
868 def runFTLNoCJITOSRValidation(*optionalTestSpecificOptions)
869     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
870 end
871
872 def runDFGEager(*optionalTestSpecificOptions)
873     run("dfg-eager", *(EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
874 end
875
876 def runDFGEagerNoCJITValidate(*optionalTestSpecificOptions)
877     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
878 end
879
880 def runFTLEager(*optionalTestSpecificOptions)
881     run("ftl-eager", *(FTL_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
882 end
883
884 def runFTLEagerNoCJITValidate(*optionalTestSpecificOptions)
885     run("ftl-eager-no-cjit", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
886 end
887
888 def runFTLEagerNoCJITOSRValidation(*optionalTestSpecificOptions)
889     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS + optionalTestSpecificOptions))
890 end
891
892 def runNoCJITNoASO(*optionalTestSpecificOptions)
893     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
894 end
895
896 def runNoCJITNoAccessInlining(*optionalTestSpecificOptions)
897     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
898 end
899
900 def runFTLNoCJITNoAccessInlining(*optionalTestSpecificOptions)
901     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
902 end
903
904 def runFTLNoCJITSmallPool(*optionalTestSpecificOptions)
905     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS + optionalTestSpecificOptions))
906 end
907
908 def runNoCJIT(*optionalTestSpecificOptions)
909     run("no-cjit", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
910 end
911
912 def runDFGMaximalFlushPhase(*optionalTestSpecificOptions)
913     run("dfg-maximal-flush-validate-no-cjit", "--forceCodeBlockToJettisonDueToOldAge=true", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *(NO_CJIT_OPTIONS + optionalTestSpecificOptions))
914 end
915
916 def runShadowChicken(*optionalTestSpecificOptions)
917     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true", *optionalTestSpecificOptions)
918 end
919
920 def defaultRun
921     if $quickMode
922         defaultQuickRun
923     else
924         runDefault
925         if $jitTests
926             runNoLLInt
927             runNoCJITValidatePhases
928             runNoCJITCollectContinuously if shouldCollectContinuously?
929             runDFGEager
930             runDFGEagerNoCJITValidate
931             runDFGMaximalFlushPhase
932
933             return if !$isFTLPlatform
934
935             runNoFTL
936             runFTLNoCJITValidate
937             runFTLNoCJITNoPutStackValidate
938             runFTLNoCJITNoInlineValidate
939             runFTLEager
940             runFTLEagerNoCJITValidate
941             runFTLNoCJITSmallPool
942         end
943     end
944 end
945
946 def defaultNoNoLLIntRun
947     if $quickMode
948         defaultQuickRun
949     else
950         runDefault
951         if $jitTests
952             runNoCJITValidatePhases
953             runNoCJITCollectContinuously if shouldCollectContinuously?
954             runDFGEager
955             runDFGEagerNoCJITValidate
956             runDFGMaximalFlushPhase
957
958             return if !$isFTLPlatform
959
960             runNoFTL
961             runFTLNoCJITValidate
962             runFTLNoCJITNoPutStackValidate
963             runFTLNoCJITNoInlineValidate
964             runFTLEager
965             runFTLEagerNoCJITValidate
966             runFTLNoCJITSmallPool
967         end
968     end
969 end
970
971 def defaultQuickRun
972     runDefault
973     if $jitTests
974         runNoCJITValidate
975
976         return if $isFTLPlatform
977
978         runNoFTL
979         runFTLNoCJITValidate
980     end
981 end
982
983 def defaultSpotCheckNoMaximalFlush
984     defaultQuickRun
985     runNoCJITNoAccessInlining
986
987     return if !$isFTLPlatform
988
989     runFTLNoCJITOSRValidation
990     runFTLNoCJITNoAccessInlining
991 end
992
993 def defaultSpotCheck
994     defaultSpotCheckNoMaximalFlush
995     runDFGMaximalFlushPhase
996 end
997
998 # This is expected to not do eager runs because eager runs can have a lot of recompilations
999 # for reasons that don't arise in the real world. It's used for tests that assert convergence
1000 # by counting recompilations.
1001 def defaultNoEagerRun
1002     runDefault
1003     if $jitTests
1004         runNoLLInt
1005         runNoCJITValidatePhases
1006         runNoCJITCollectContinuously if shouldCollectContinuously?
1007
1008         return if !$isFTLPlatform
1009
1010         runNoFTL
1011         runFTLNoCJITValidate
1012         runFTLNoCJITNoInlineValidate
1013     end
1014 end
1015
1016 def defaultNoSamplingProfilerRun
1017     runDefault
1018     if $jitTests
1019         runNoLLInt
1020         runNoCJITValidatePhases
1021         runNoCJITCollectContinuously if shouldCollectContinuously?
1022         runDFGEager
1023         runDFGEagerNoCJITValidate
1024         runDFGMaximalFlushPhase
1025
1026         return if !$isFTLPlatform
1027
1028         runNoFTL
1029         runFTLNoCJITNoPutStackValidate
1030         runFTLNoCJITNoInlineValidate
1031         runFTLEager
1032         runFTLEagerNoCJITValidate
1033         runFTLNoCJITSmallPool
1034     end
1035 end
1036
1037 def runProfiler
1038     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
1039         skip
1040         return
1041     end
1042
1043     profilerOutput = uniqueFilename(".json")
1044     if $canRunDisplayProfilerOutput
1045         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
1046     else
1047         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
1048         run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
1049     end
1050 end
1051
1052 def runExceptionFuzz
1053     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
1054     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
1055 end
1056
1057 def runExecutableAllocationFuzz(name, *options)
1058     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
1059     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
1060 end
1061
1062 def runTypeProfiler
1063     if !$jitTests
1064         return
1065     end
1066
1067     return if !$isFTLPlatform
1068
1069     run("ftl-no-cjit-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1070     run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
1071 end
1072
1073 def runControlFlowProfiler
1074     if !$jitTests
1075         return
1076     end
1077
1078     return if !$isFTLPlatform
1079
1080     run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1081 end
1082
1083 def runTest262(mode, exception, includeFiles, flags)
1084     failsWithException = exception != "NoException"
1085     isStrict = false
1086     isModule = false
1087     isAsync = false
1088
1089     flags.each {
1090         | flag |
1091         case flag
1092         when :strict
1093             isStrict = true
1094         when :module
1095             isModule = true
1096         when :async
1097             isAsync = true
1098         else
1099             raise "Invalid flag for runTest262, #{flag}"
1100         end
1101     }
1102
1103     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
1104
1105     args = [pathToVM.to_s] + BASE_OPTIONS
1106     args << "--exception=" + exception if failsWithException
1107     args << "--test262-async" if isAsync
1108     args += includeFiles
1109
1110     case mode
1111     when :normal
1112         errorHandler = simpleErrorHandler
1113         outputHandler = silentOutputHandler
1114     when :fail
1115         errorHandler = expectedFailErrorHandler
1116         outputHandler = noisyOutputHandler
1117     else
1118         raise "Invalid mode: #{mode}"
1119     end
1120
1121     if isStrict
1122         kind = "default-strict"
1123         args << "--strict-file=#{$benchmark}"
1124     else
1125         kind = "default"
1126         if isModule
1127             args << "--module-file=#{$benchmark}"
1128         else
1129             args << $benchmark.to_s
1130         end
1131     end
1132
1133     addRunCommand(kind, args, outputHandler, errorHandler)
1134 end
1135
1136 def prepareTest262Fixture
1137     # This function is used to add the files used by Test262 modules tests.
1138     prepareExtraRelativeFiles([""], $collection)
1139 end
1140
1141 def runES6(mode)
1142     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
1143     case mode
1144     when :normal
1145         errorHandler = simpleErrorHandler
1146     when :fail
1147         errorHandler = expectedFailErrorHandler
1148     else
1149         raise "Invalid mode: #{mode}"
1150     end
1151     addRunCommand("default", args, noisyOutputHandler, errorHandler)
1152 end
1153
1154 def runModules
1155     run("default-modules", "-m")
1156
1157     if !$jitTests
1158         return
1159     end
1160
1161     run("no-llint-modules", "-m", "--useLLInt=false")
1162     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
1163     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
1164     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1165
1166     return if !$isFTLPlatform
1167
1168     run("default-ftl-modules", "-m", *FTL_OPTIONS)
1169     run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1170     run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1171     run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
1172     run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1173     run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1174 end
1175
1176 def runWebAssembly
1177     return if !$jitTests
1178     return if !$isFTLPlatform
1179     # FIXME: The current WebAssembly implementation includes Darwin specific things.
1180     # Once WebAssembly becomes ready to be ported, we will enable it on the other OSes (like Linux) and drop this workaround.
1181     # https://bugs.webkit.org/show_bug.cgi?id=164032
1182     return if $hostOS != "darwin"
1183     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1184     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1185     prepareExtraRelativeFiles(modules.map { |f| "../" + f }, $collection)
1186     run("default-wasm", "-m", "--useWebAssembly=1")
1187 end
1188
1189 def runWebAssemblySpecTest(mode)
1190     case mode
1191     when :skip
1192         return
1193     end
1194
1195     return if !$jitTests
1196     return if !$isFTLPlatform
1197     # FIXME: The current WebAssembly implementation includes Darwin specific things.
1198     # Once WebAssembly becomes ready to be ported, we will enable it on the other OSes (like Linux) and drop this workaround.
1199     # https://bugs.webkit.org/show_bug.cgi?id=164032
1200     return if $hostOS != "darwin"
1201     modules = Dir[WASMTESTS_PATH + "*.js"].map { |f| File.basename(f) }
1202     prepareExtraAbsoluteFiles(WASMTESTS_PATH, ["wasm.json"])
1203     prepareExtraRelativeFiles(modules.map { |f| "../../" + f }, $collection)
1204
1205     runWithOutputHandler("default-wasm", noisyOutputHandler, "-m", "--useWebAssembly=1")
1206 end
1207
1208 def runChakra(mode, exception, baselineFile, extraFiles)
1209     raise unless $benchmark.to_s =~ /\.js$/
1210     failsWithException = exception != "NoException"
1211     testName = $~.pre_match
1212
1213     prepareExtraAbsoluteFiles(CHAKRATESTS_PATH, ["jsc-lib.js"])
1214     prepareExtraRelativeFiles(extraFiles.map { |f| "../" + f }, $collection)
1215
1216     args = [pathToVM.to_s] + BASE_OPTIONS
1217     args += FTL_OPTIONS if $isFTLPlatform
1218     args += EAGER_OPTIONS
1219     args << "--exception=" + exception if failsWithException
1220     args << "--dumpException" if failsWithException
1221     args += ["jsc-lib.js"]
1222
1223     case mode
1224     when :baseline
1225         prepareExtraRelativeFiles([(Pathname("..") + baselineFile).to_s], $collection)
1226         errorHandler = diffErrorHandler(($benchmarkDirectory + baselineFile).to_s)
1227         outputHandler = noisyOutputHandler
1228     when :pass
1229         errorHandler = chakraPassFailErrorHandler
1230         outputHandler = noisyOutputHandler
1231     when :skip
1232         return
1233     else
1234         raise "Invalid mode: #{mode}"
1235     end
1236
1237     kind = "default"
1238     args << $benchmark.to_s
1239
1240     addRunCommand(kind, args, outputHandler, errorHandler)
1241 end
1242
1243 def runLayoutTest(kind, *options)
1244     raise unless $benchmark.to_s =~ /\.js$/
1245     testName = $~.pre_match
1246     if kind
1247         kind = "layout-" + kind
1248     else
1249         kind = "layout"
1250     end
1251
1252     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1253     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1254
1255     args = [pathToVM.to_s] + BASE_OPTIONS + options +
1256         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1257          $benchmark.to_s,
1258          (Pathname.new("resources") + "standalone-post.js").to_s]
1259     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1260 end
1261
1262 def runLayoutTestNoFTL
1263     runLayoutTest("no-ftl")
1264 end
1265
1266 def runLayoutTestNoLLInt
1267     runLayoutTest("no-llint", "--useLLInt=false")
1268 end
1269
1270 def runLayoutTestNoCJIT
1271     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1272 end
1273
1274 def runLayoutTestDFGEagerNoCJIT
1275     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1276 end
1277
1278 def runLayoutTestDefault
1279     runLayoutTest(nil, "--testTheFTL=true", *FTL_OPTIONS)
1280 end
1281
1282 def runLayoutTestFTLNoCJIT
1283     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1284 end
1285
1286 def runLayoutTestFTLEagerNoCJIT
1287     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1288 end
1289
1290 def noFTLRunLayoutTest
1291     if !$jitTests
1292         return
1293     end
1294
1295     runLayoutTestNoLLInt
1296     runLayoutTestNoCJIT
1297     runLayoutTestDFGEagerNoCJIT
1298 end
1299
1300 def defaultQuickRunLayoutTest
1301     runLayoutTestDefault
1302     if $jitTests
1303         if $isFTLPlatform
1304             runLayoutTestNoFTL
1305             runLayoutTestFTLNoCJIT
1306             runLayoutTestFTLEagerNoCJIT
1307         else
1308             noFTLRunLayoutTest
1309         end
1310     end
1311 end
1312
1313 def defaultRunLayoutTest
1314     if $quickMode
1315         defaultQuickRunLayoutTest
1316     else
1317         runLayoutTestDefault
1318         if $jitTests
1319             noFTLRunLayoutTest
1320
1321             return if !$isFTLPlatform
1322
1323             runLayoutTestNoFTL
1324             runLayoutTestFTLNoCJIT
1325             runLayoutTestFTLEagerNoCJIT
1326         end
1327     end
1328 end
1329
1330 def noEagerNoNoLLIntTestsRunLayoutTest
1331     runLayoutTestDefault
1332     if $jitTests
1333         runLayoutTestNoCJIT
1334
1335         return if !$isFTLPlatform
1336
1337         runLayoutTestNoFTL
1338         runLayoutTestFTLNoCJIT
1339     end
1340 end
1341
1342 def noNoLLIntRunLayoutTest
1343     runLayoutTestDefault
1344     if $jitTests
1345         runLayoutTestNoCJIT
1346         runLayoutTestDFGEagerNoCJIT
1347
1348         return if !$isFTLPlatform
1349
1350         runLayoutTestNoFTL
1351         runLayoutTestFTLNoCJIT
1352         runLayoutTestFTLEagerNoCJIT
1353     end
1354 end
1355
1356 def prepareExtraRelativeFiles(extraFiles, destination)
1357     Dir.chdir($outputDir) {
1358         extraFiles.each {
1359             | file |
1360             dest = destination + file
1361             FileUtils.mkdir_p(dest.dirname)
1362             FileUtils.cp $extraFilesBaseDir + file, dest
1363         }
1364     }
1365 end
1366
1367 def baseDirForCollection(collectionName)
1368     Pathname(".tests") + collectionName
1369 end
1370
1371 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1372     raise unless absoluteBase.absolute?
1373     Dir.chdir($outputDir) {
1374         collectionBaseDir = baseDirForCollection($collectionName)
1375         extraFiles.each {
1376             | file |
1377             destination = collectionBaseDir + file
1378             FileUtils.mkdir_p destination.dirname unless destination.directory?
1379             FileUtils.cp absoluteBase + file, destination
1380         }
1381     }
1382 end
1383
1384 def runMozillaTest(kind, mode, extraFiles, *options)
1385     if kind
1386         kind = "mozilla-" + kind
1387     else
1388         kind = "mozilla"
1389     end
1390     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1391     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1392     case mode
1393     when :normal
1394         errorHandler = mozillaErrorHandler
1395     when :negative
1396         errorHandler = mozillaExit3ErrorHandler
1397     when :fail
1398         errorHandler = mozillaFailErrorHandler
1399     when :skip
1400         return
1401     else
1402         raise "Invalid mode: #{mode}"
1403     end
1404     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1405 end
1406
1407 def runMozillaTestDefault(mode, *extraFiles)
1408     runMozillaTest(nil, mode, extraFiles, *FTL_OPTIONS)
1409 end
1410
1411 def runMozillaTestNoFTL(mode, *extraFiles)
1412     runMozillaTest("no-ftl", mode, extraFiles)
1413 end
1414
1415 def runMozillaTestLLInt(mode, *extraFiles)
1416     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1417 end
1418
1419 def runMozillaTestBaselineJIT(mode, *extraFiles)
1420     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1421 end
1422
1423 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1424     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1425 end
1426
1427 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1428     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1429 end
1430
1431 def defaultQuickRunMozillaTest(mode, *extraFiles)
1432     if $jitTests
1433         runMozillaTestDefault(mode, *extraFiles)
1434         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1435     else
1436         runMozillaTestNoFTL(mode, *extraFiles)
1437         if $jitTests
1438             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1439         end
1440     end
1441 end
1442
1443 def defaultRunMozillaTest(mode, *extraFiles)
1444     if $quickMode
1445         defaultQuickRunMozillaTest(mode, *extraFiles)
1446     else
1447         runMozillaTestNoFTL(mode, *extraFiles)
1448         if $jitTests
1449             runMozillaTestLLInt(mode, *extraFiles)
1450             runMozillaTestBaselineJIT(mode, *extraFiles)
1451             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1452             runMozillaTestDefault(mode, *extraFiles)
1453             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles) if $isFTLPlatform
1454         end
1455     end
1456 end
1457
1458 def runNoisyTest(kind, *options)
1459     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
1460 end
1461
1462 def runNoisyTestDefault
1463     runNoisyTest("default", *FTL_OPTIONS)
1464 end
1465
1466 def runNoisyTestNoFTL
1467     runNoisyTest("no-ftl")
1468 end
1469
1470 def runNoisyTestNoCJIT
1471     runNoisyTest("ftl-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1472 end
1473
1474 def runNoisyTestEagerNoCJIT
1475     runNoisyTest("ftl-eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS + COLLECT_CONTINUOUSLY_OPTIONS))
1476 end
1477
1478 def defaultRunNoisyTest
1479     runNoisyTestDefault
1480     if $jitTests and $isFTLPlatform
1481         runNoisyTestNoFTL
1482         runNoisyTestNoCJIT
1483         runNoisyTestEagerNoCJIT
1484     end
1485 end
1486
1487 def skip
1488     $didAddRunCommand = true
1489     puts "Skipping #{$collectionName}/#{$benchmark}"
1490 end
1491
1492 def allJSFiles(path)
1493     if path.file?
1494         [path]
1495     else
1496         result = []
1497         Dir.foreach(path) {
1498             | filename |
1499             next unless filename =~ /\.js$/
1500             next unless (path + filename).file?
1501             result << path + filename
1502         }
1503         result
1504     end
1505 end
1506
1507 def uniqueifyName(names, name)
1508     result = name.to_s
1509     toAdd = 1
1510     while names[result]
1511         result = "#{name}-#{toAdd}"
1512         toAdd += 1
1513     end
1514     names[result] = true
1515     result
1516 end
1517
1518 def simplifyCollectionName(collectionPath)
1519     outerDir = collectionPath.dirname
1520     name = collectionPath.basename
1521     lastName = name
1522     if collectionPath.directory?
1523         while lastName.to_s =~ /test/
1524             lastName = outerDir.basename
1525             name = lastName + name
1526             outerDir = outerDir.dirname
1527         end
1528     end
1529     uniqueifyName($collectionNames, name)
1530 end
1531
1532 def prepareCollection(name)
1533     FileUtils.mkdir_p $outputDir + name
1534
1535     absoluteCollection = $collection.realpath
1536
1537     Dir.chdir($outputDir) {
1538         bundleDir = baseDirForCollection(name)
1539
1540         # Create the proper directory structures.
1541         FileUtils.mkdir_p bundleDir
1542         if bundleDir.basename == $collection.basename
1543             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1544             $collection = bundleDir
1545         else
1546             FileUtils.cp_r absoluteCollection, bundleDir
1547             $collection = bundleDir + $collection.basename
1548         end
1549
1550         $extraFilesBaseDir = absoluteCollection
1551     }
1552 end
1553
1554 $collectionNames = {}
1555
1556 def handleCollectionFile(collection)
1557     collectionName = simplifyCollectionName(collection)
1558    
1559     paths = {}
1560     subCollections = []
1561     YAML::load(IO::read(collection)).each {
1562         | entry |
1563         if entry["collection"]
1564             subCollections << entry["collection"]
1565             next
1566         end
1567         
1568         if Pathname.new(entry["path"]).absolute?
1569             raise "Absolute path: " + entry["path"] + " in #{collection}"
1570         end
1571         
1572         if paths[entry["path"]]
1573             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1574         end
1575         
1576         subCollection = collection.dirname + entry["path"]
1577         
1578         if subCollection.file?
1579             subCollectionName = Pathname.new(entry["path"]).dirname
1580         else
1581             subCollectionName = entry["path"]
1582         end
1583         
1584         $collection = subCollection
1585         $collectionName = Pathname.new(collectionName)
1586         Pathname.new(subCollectionName).each_filename {
1587             | filename |
1588             next if filename =~ /^\./
1589             $collectionName += filename
1590         }
1591         $collectionName = $collectionName.to_s
1592         
1593         prepareCollection($collectionName)
1594       
1595         Dir.chdir($outputDir) {
1596             pathsToSearch = [$collection]
1597             if entry["tests"]
1598                 if entry["tests"].is_a? Array
1599                     pathsToSearch = entry["tests"].map {
1600                         | testName |
1601                         pathsToSearch[0] + testName
1602                     }
1603                 else
1604                     pathsToSearch[0] += entry["tests"]
1605                 end
1606             end
1607             pathsToSearch.each {
1608                 | pathToSearch |
1609                 allJSFiles(pathToSearch).each {
1610                     | path |
1611                     
1612                     $benchmark = path.basename
1613                     $benchmarkDirectory = path.dirname
1614                     
1615                     $runCommandOptions = {}
1616                     eval entry["cmd"]
1617                 }
1618             }
1619         }
1620     }
1621     
1622     subCollections.each {
1623         | subCollection |
1624         handleCollection(collection.dirname + subCollection)
1625     }
1626 end
1627
1628 def handleCollectionDirectory(collection)
1629     collectionName = simplifyCollectionName(collection)
1630     
1631     $collection = collection
1632     $collectionName = collectionName
1633     prepareCollection(collectionName)
1634    
1635     Dir.chdir($outputDir) {
1636         $benchmarkDirectory = $collection
1637         allJSFiles($collection).each {
1638             | path |
1639             
1640             $benchmark = path.basename
1641             
1642             $runCommandOptions = {}
1643             defaultRun unless parseRunCommands
1644         }
1645     }
1646 end
1647
1648 def handleCollection(collection)
1649     collection = Pathname.new(collection)
1650     
1651     if collection.file?
1652         handleCollectionFile(collection)
1653     else
1654         handleCollectionDirectory(collection)
1655     end
1656 end
1657
1658 def appendFailure(plan)
1659     File.open($outputDir + "failed", "a") {
1660         | outp |
1661         outp.puts plan.name
1662     }
1663     $numFailures += 1
1664 end
1665
1666 def appendPass(plan)
1667     File.open($outputDir + "passed", "a") {
1668         | outp |
1669         outp.puts plan.name
1670     }
1671     $numPasses += 1
1672 end
1673
1674 def appendResult(plan, didPass)
1675     File.open($outputDir + "results", "a") {
1676         | outp |
1677         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1678     }
1679 end
1680
1681 def prepareBundle
1682     raise if $bundle
1683
1684     if $doNotMessWithVMPath
1685         if !$remote and !$tarball
1686             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1687             $jscPath = Pathname.new($jscPath).realpath
1688         else
1689             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1690         end
1691     else
1692         originalJSCPath = $jscPath
1693         vmDir = $outputDir + ".vm"
1694         FileUtils.mkdir_p vmDir
1695         
1696         frameworkPath = frameworkFromJSCPath($jscPath)
1697         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1698         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1699         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1700
1701         if frameworkPath
1702             source = frameworkPath
1703             destination = Pathname.new(".vm")
1704         else
1705             source = originalJSCPath
1706             destination = $jscPath
1707
1708             Dir.chdir($outputDir) {
1709                 FileUtils.mkdir_p $jscPath.dirname
1710             }
1711         end
1712
1713         Dir.chdir($outputDir) {
1714             if $copyVM
1715                 FileUtils.cp_r source, destination
1716             else
1717                 begin 
1718                     FileUtils.ln_s source, destination
1719                 rescue Exception
1720                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1721                     FileUtils.cp_r source, destination
1722                 end
1723             end
1724
1725             if $remote and $hostOS == "linux"
1726                 begin
1727                     dependencies = `ldd #{source}`
1728                     dependencies.split(/\n/).each {
1729                         | dependency |
1730                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1731                     }
1732                 rescue
1733                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1734                 end
1735             end
1736         }
1737     end
1738     
1739     Dir.chdir($outputDir) {
1740         FileUtils.cp_r HELPERS_PATH, ".helpers"
1741     }
1742
1743     ARGV.each {
1744         | collection |
1745         handleCollection(collection)
1746     }
1747
1748     puts
1749 end
1750
1751 def cleanOldResults
1752     raise unless $bundle
1753
1754     eachResultFile($outputDir) {
1755         | path |
1756         FileUtils.rm_f path
1757     }
1758 end
1759
1760 def cleanEmptyResultFiles
1761     eachResultFile($outputDir) {
1762         | path |
1763         next unless path.basename.to_s =~ /\.out$/
1764         next unless FileTest.size(path) == 0
1765         FileUtils.rm_f path
1766     }
1767 end
1768
1769 def eachResultFile(startingDir, &block)
1770     dirsToClean = [startingDir]
1771     until dirsToClean.empty?
1772         nextDir = dirsToClean.pop
1773         Dir.foreach(nextDir) {
1774             | entry |
1775             next if entry =~ /^\./
1776             path = nextDir + entry
1777             if path.directory?
1778                 dirsToClean.push(path)
1779             else
1780                 block.call(path)
1781             end
1782         }
1783     end
1784 end
1785
1786 def prepareTestRunner
1787     raise if $bundle
1788
1789     $runlist.each_with_index {
1790         | plan, index |
1791         plan.index = index
1792     }
1793
1794     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1795     toDelete = []
1796     Dir.foreach($runnerDir) {
1797         | filename |
1798         if filename =~ /^test_/
1799             toDelete << filename
1800         end
1801     }
1802     
1803     toDelete.each {
1804         | filename |
1805         File.unlink($runnerDir + filename)
1806     }
1807
1808     $runlist.each {
1809         | plan |
1810         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1811     }
1812
1813     case $testRunnerType
1814     when :make
1815         prepareMakeTestRunner
1816     when :shell
1817         prepareShellTestRunner
1818     else
1819         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1820     end
1821 end
1822
1823 def prepareShellTestRunner
1824     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
1825 end
1826
1827 def prepareMakeTestRunner
1828     # The goals of our parallel test runner are scalability and simplicity. The
1829     # simplicity part is particularly important. We don't want to have to have
1830     # a full-time contributor just philosophising about parallel testing.
1831     #
1832     # As such, we just pass off all of the hard work to 'make'. This creates a
1833     # dummy directory ("$outputDir/.runner") in which we create a dummy
1834     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
1835     # That is, for each test we know we will run, there is a rule in the
1836     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
1837     # Makefile results in 'make' doing all of the hard work:
1838     #
1839     # - Load balancing just works. Most systems have a great load balancer in
1840     #   'make'. If your system doesn't then just install a real 'make'.
1841     #
1842     # - Interruptions just work. For example Ctrl-C handling in 'make' is
1843     #   exactly right. You don't have to worry about zombie processes.
1844     #
1845     # We then do some tricks to make failure detection work and to make this
1846     # totally sound. If a test fails, we don't want the whole 'make' job to
1847     # stop. We also don't have any facility for makefile-escaping of path names.
1848     # We do have such a thing for shell-escaping, though. We fix both problems
1849     # by having the actual work for each of the test rules be done in a shell
1850     # script on the side. There is one such script per test. The script responds
1851     # to failure by printing something on the console and then touching a
1852     # failure file for that test, but then still returns 0. This makes 'make'
1853     # continue past that failure and complete all the tests anyway.
1854     #
1855     # In the end, this script collects all of the failures by searching for
1856     # files in the .runner directory whose name matches /^test_fail_/, where
1857     # the thing after the 'fail_' is the test index. Those are the files that
1858     # would be created by the test scripts if they detect failure. We're
1859     # basically using the filesystem as a concurrent database of test failures.
1860     # Even if two tests fail at the same time, since they're touching different
1861     # files we won't miss any failures.
1862     runIndices = []
1863     $runlist.each {
1864         | plan |
1865         runIndices << plan.index
1866     }
1867     
1868     File.open($runnerDir + "Makefile", "w") {
1869         | outp |
1870         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1871         runIndices.each {
1872             | index |
1873             plan = $runlist[index]
1874             outp.puts "test_done_#{index}:"
1875             outp.puts "\tsh test_script_#{plan.index}"
1876         }
1877     }
1878 end
1879
1880 def cleanRunnerDirectory
1881     raise unless $bundle
1882     Dir.foreach($runnerDir) {
1883         | filename |
1884         next unless filename =~ /^test_fail/
1885         FileUtils.rm_f $runnerDir + filename
1886     }
1887 end
1888
1889 def sshRead(cmd)
1890     raise unless $remote
1891
1892     result = ""
1893     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1894       | inp |
1895       inp.each_line {
1896         | line |
1897         result += line
1898       }
1899     }
1900     raise "#{$?}" unless $?.success?
1901     result
1902 end
1903
1904 def runCommandOnTester(cmd)
1905     if $remote
1906         result = sshRead(cmd)
1907     else
1908         result = `#{cmd}`
1909     end
1910 end
1911
1912 def numberOfProcessors
1913     if $hostOS == "windows"
1914         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1915     else
1916         begin
1917             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1918         rescue
1919             numProcessors = 0
1920         end
1921
1922         if numProcessors == 0
1923             begin
1924                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1925             rescue
1926                 numProcessors == 0
1927             end
1928         end
1929     end
1930
1931     if numProcessors == 0
1932         numProcessors = 1
1933     end
1934     return numProcessors
1935 end
1936
1937 def runAndMonitorTestRunnerCommand(*cmd)
1938     numberOfTests = 0
1939     Dir.chdir($runnerDir) {
1940         # -1 for the runscript, and -2 for '..' and '.'
1941         numberOfTests = Dir.entries(".").count - 3
1942     }
1943     unless $progressMeter
1944         mysys(cmd.join(' '))
1945     else
1946        running = {}
1947        didRun = {}
1948        didFail = {}
1949        blankLine = true
1950        prevStringLength = 0
1951        IO.popen(cmd.join(' '), mode="r") {
1952            | inp |
1953            inp.each_line {
1954                | line |
1955                line = line.scrub.chomp
1956                if line =~ /^Running /
1957                    running[$~.post_match] = true
1958                elsif line =~ /^PASS: /
1959                    didRun[$~.post_match] = true
1960                elsif line =~ /^FAIL: /
1961                    didRun[$~.post_match] = true
1962                    didFail[$~.post_match] = true
1963                else
1964                    unless blankLine
1965                        print("\r" + " " * prevStringLength + "\r")
1966                    end
1967                    puts line
1968                    blankLine = true
1969                end
1970
1971                def lpad(str, chars)
1972                    str = str.to_s
1973                    if str.length > chars
1974                        str
1975                    else
1976                       "%#{chars}s"%(str)
1977                    end
1978                end
1979
1980                string  = ""
1981                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1982                unless didFail.empty?
1983                    string += " (failed #{didFail.size})"
1984                end
1985                string += " "
1986                (running.size - didRun.size).times {
1987                    string += "."
1988                }
1989                if string.length < prevStringLength
1990                    print string
1991                    print(" " * (prevStringLength - string.length))
1992                end
1993                print string
1994                prevStringLength = string.length
1995                blankLine = false
1996                $stdout.flush
1997            }
1998        }
1999        puts
2000        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
2001     end
2002 end
2003
2004 def runTestRunner
2005     case $testRunnerType
2006     when :shell
2007         testRunnerCommand = "sh runscript"
2008     when :make
2009         testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
2010     else
2011         raise "Unknown test runner type: #{$testRunnerType.to_s}"
2012     end
2013
2014     if $remote
2015         if !$remoteDirectory
2016             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
2017         end
2018         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
2019         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
2020         remoteScript = "\""
2021         remoteScript += "cd #{$remoteDirectory} && "
2022         remoteScript += "rm -rf #{$outputDir.basename} && "
2023         remoteScript += "tar xzf #{$tarFileName} && "
2024         remoteScript += "cd #{$outputDir.basename}/.runner && "
2025         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
2026         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
2027         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
2028         $envVars.each { |var| remoteScript += "export " << var << "\n" }
2029         remoteScript += "#{testRunnerCommand}\""
2030         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
2031     else
2032         Dir.chdir($runnerDir) {
2033             runAndMonitorTestRunnerCommand(testRunnerCommand)
2034         }
2035     end
2036 end
2037
2038 def detectFailures
2039     raise if $bundle
2040
2041     failures = []
2042     
2043     if $remote
2044         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
2045         output.split(/\n/).each {
2046             | line |
2047             next unless line =~ /test_fail_/
2048             failures << $~.post_match.to_i
2049         }
2050     else
2051         Dir.foreach($runnerDir) {
2052             | filename |
2053             next unless filename =~ /test_fail_/
2054             failures << $~.post_match.to_i
2055         }
2056     end
2057
2058     failureSet = {}
2059
2060     failures.each {
2061         | failure | 
2062         appendFailure($runlist[failure])
2063         failureSet[failure] = true
2064     }
2065
2066     familyMap = {}
2067     $runlist.each_with_index {
2068         | plan, index |
2069         unless familyMap[plan.family]
2070             familyMap[plan.family] = []
2071         end
2072         if failureSet[index]
2073             appendResult(plan, false)
2074             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
2075             next
2076         else
2077             appendResult(plan, true)
2078             familyMap[plan.family] << {:result => "PASS", :plan => plan};
2079         end
2080         appendPass(plan)
2081     }
2082
2083     File.open($outputDir + "resultsByFamily", "w") {
2084         | outp |
2085         first = true
2086         familyMap.keys.sort.each {
2087             | familyName |
2088             if first
2089                 first = false
2090             else
2091                 outp.puts
2092             end
2093             
2094             outp.print "#{familyName}:"
2095
2096             numPassed = 0
2097             familyMap[familyName].each {
2098                 | entry |
2099                 if entry[:result] == "PASS"
2100                     numPassed += 1
2101                 end
2102             }
2103
2104             if numPassed == familyMap[familyName].size
2105                 outp.puts " PASSED"
2106             elsif numPassed == 0
2107                 outp.puts " FAILED"
2108             else
2109                 outp.puts
2110                 familyMap[familyName].each {
2111                     | entry |
2112                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
2113                 }
2114             end
2115         }
2116     }
2117 end
2118
2119 def compressBundle
2120     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
2121     $stderr.puts ">> #{cmd}" if $verbosity >= 2
2122     raise unless system(cmd)
2123 end
2124
2125 def clean(file)
2126     FileUtils.rm_rf file unless $bundle
2127 end
2128
2129 clean($outputDir + "failed")
2130 clean($outputDir + "passed")
2131 clean($outputDir + "results")
2132 clean($outputDir + "resultsByFamily")
2133 clean($outputDir + ".vm")
2134 clean($outputDir + ".helpers")
2135 clean($outputDir + ".runner")
2136 clean($outputDir + ".tests")
2137 clean($outputDir + "_payload")
2138
2139 Dir.mkdir($outputDir) unless $outputDir.directory?
2140
2141 $outputDir = $outputDir.realpath
2142 $runnerDir = $outputDir + ".runner"
2143
2144 if !$numChildProcesses
2145     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
2146         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
2147     else
2148         $numChildProcesses = numberOfProcessors
2149     end
2150 end
2151
2152 if ENV["JSCTEST_timeout"]
2153     # In the worst case, the processors just interfere with each other.
2154     # Increase the timeout proportionally to the number of processors.
2155     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
2156 end
2157
2158 def runBundle
2159     raise unless $bundle
2160
2161     cleanRunnerDirectory
2162     cleanOldResults
2163     runTestRunner
2164     cleanEmptyResultFiles
2165 end
2166
2167 def runNormal
2168     raise if $bundle or $tarball
2169
2170     prepareBundle
2171     prepareTestRunner
2172     runTestRunner
2173     cleanEmptyResultFiles
2174     detectFailures
2175 end
2176
2177 def runTarball
2178     raise unless $tarball
2179
2180     prepareBundle 
2181     prepareTestRunner
2182     compressBundle
2183 end
2184
2185 def runRemote
2186     raise unless $remote
2187
2188     prepareBundle
2189     prepareTestRunner
2190     compressBundle
2191     runTestRunner
2192     detectFailures
2193 end
2194
2195 puts
2196 if $bundle
2197     runBundle
2198 elsif $remote
2199     runRemote
2200 elsif $tarball
2201     runTarball
2202 else
2203     runNormal
2204 end