TypeProfiler and TypeProfilerLog don't play nicely with the concurrent JIT
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2015 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
47 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
48 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
49 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
50 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
51 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
52
53 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
54
55 begin
56     require 'shellwords'
57 rescue Exception => e
58     $stderr.puts "Warning: did not find shellwords, not running any tests."
59     exit 0
60 end
61
62 $canRunDisplayProfilerOutput = false
63
64 begin
65     require 'rubygems'
66     require 'json'
67     require 'highline'
68     $canRunDisplayProfilerOutput = true
69 rescue Exception => e
70     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
71     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
72     $stderr.puts "Error: #{e.inspect}"
73 end
74
75 def printCommandArray(*cmd)
76     begin
77         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
78     rescue
79         commandArray = cmd.join(' ')
80     end
81     $stderr.puts ">> #{commandArray}"
82 end
83
84 def mysys(*cmd)
85     printCommandArray(*cmd) if $verbosity >= 1
86     raise "Command failed: #{$?.inspect}" unless system(*cmd)
87 end
88
89 def escapeAll(array)
90     array.map {
91         | v |
92         raise "Detected a non-string in #{inspect}" unless v.is_a? String
93         Shellwords.shellescape(v)
94     }.join(' ')
95 end
96
97
98 $jscPath = nil
99 $doNotMessWithVMPath = false
100 $enableFTL = false
101 $jitTests = true
102 $memoryLimited = false
103 $outputDir = Pathname.new("results")
104 $verbosity = 0
105 $bundle = nil
106 $tarball = false
107 $tarFileName = "payload.tar.gz"
108 $copyVM = false
109 $testRunnerType = nil
110 $remoteUser = nil
111 $remoteHost = nil
112 $remotePort = nil
113 $remoteDirectory = nil
114 $architecture = nil
115 $hostOS = nil
116 $filter = nil
117 $envVars = []
118 $quickMode = false
119
120
121 def usage
122     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
123     puts
124     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
125     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
126     puts "                            --jsc specifies an already present JavaScriptCore to test."
127     puts "--ftl-jit                   Indicate that we have the FTL JIT."
128     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
129     puts "                            Skip tests tagged with //@large-heap"
130     puts "--no-jit                    Do not run JIT specific tests."
131     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
132     puts "--verbose            (-v)   Print more things while running."
133     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
134     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
135     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
136     puts "                            e.g. x86, x86_64, arm."
137     puts "--os                        Specify os instead of determining from JavaScriptCore build."
138     puts "                            e.g. darwin, linux & windows."
139     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
140     puts "                            In general the shell runner is slower than the make runner."
141     puts "--make-runner               Uses the faster make-based runner."
142     puts "--remote                    Specify a remote host on which to run tests from command line argument."
143     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
144     puts "--child-processes    (-c)   Specify the number of child processes."
145     puts "--filter                    Only run tests whose name matches the given regular expression."
146     puts "--help               (-h)   Print this message."
147     puts "--env-vars                  Add a list of environment variables to set before running jsc."
148     puts "                            Each environment variable should be separated by a space."
149     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
150     puts "                            it will override the default value."
151     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
152     exit 1
153 end
154
155 jscArg = nil
156 ifJSCArgIsntProvidedAreWeReleaseBuild = true
157
158 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
159                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
160                ['--no-copy', GetoptLong::NO_ARGUMENT],
161                ['--ftl-jit', GetoptLong::NO_ARGUMENT],
162                ['--memory-limited', GetoptLong::NO_ARGUMENT],
163                ['--no-jit', GetoptLong::NO_ARGUMENT],
164                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
165                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
166                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
167                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
168                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
169                ['--os', GetoptLong::REQUIRED_ARGUMENT],
170                ['--shell-runner', GetoptLong::NO_ARGUMENT],
171                ['--make-runner', GetoptLong::NO_ARGUMENT],
172                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
173                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
174                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
175                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
176                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
177                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
178                ['--debug', GetoptLong::NO_ARGUMENT],
179                ['--release', GetoptLong::NO_ARGUMENT],
180                ['--quick', '-q', GetoptLong::NO_ARGUMENT]).each {
181     | opt, arg |
182     case opt
183     when '--help'
184         usage
185     when '--jsc'
186         jscArg = arg
187     when '--no-copy'
188         $doNotMessWithVMPath = true
189     when '--output-dir'
190         $outputDir = Pathname.new(arg)
191     when '--ftl-jit'
192         $enableFTL = true
193     when '--memory-limited'
194         $memoryLimited = true
195     when '--no-jit'
196         $jitTests = false
197     when '--verbose'
198         $verbosity += 1
199     when '--run-bundle'
200         $bundle = Pathname.new(arg)
201     when '--tarball'
202         $tarball = true
203         $copyVM = true
204         $tarFileName = arg unless arg == ''
205     when '--force-vm-copy'
206         $copyVM = true
207     when '--shell-runner'
208         $testRunnerType = :shell
209     when '--make-runner'
210         $testRunnerType = :make
211     when '--remote'
212         $copyVM = true
213         $tarball = true
214         $remote = true
215         uri = URI("ssh://" + arg)
216         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
217     when '--remote-config-file'
218         $remoteConfigFile = arg
219     when '--child-processes'
220         $numChildProcesses = arg.to_i
221     when '--filter'
222         $filter = Regexp.new(arg)
223     when '--arch'
224         $architecture = arg
225     when '--os'
226         $hostOS = arg
227     when '--env-vars'
228         $envVars = arg.gsub(/\s+/, ' ').split(' ')
229     when '--quick'
230         $quickMode = true
231     when '--debug'
232         ifJSCArgIsntProvidedAreWeReleaseBuild = false
233     when '--release'
234         ifJSCArgIsntProvidedAreWeReleaseBuild = true
235     end
236 }
237
238 if $remoteConfigFile
239     file = File.read($remoteConfigFile)
240     config = JSON.parse(file)
241
242     if !$remote and config['remote']
243         $copyVM = true
244         $tarball = true
245         $remote = true
246         uri = URI("ssh://" + config['remote'])
247         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
248     end
249
250     if config['remoteDirectory']
251         $remoteDirectory = config['remoteDirectory']
252     end
253 end
254
255 unless jscArg
256     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
257     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
258     command += ifJSCArgIsntProvidedAreWeReleaseBuild ? " --release" : " --debug"
259     command += " --executablePath"
260
261     output = `#{command}`.split("\n")
262     if !output.length
263         $stderr.puts "Error: must specify --jsc <path>"
264         exit 1
265     end
266
267     output = output[0]
268     jscArg = Pathname.new(output).join("jsc")
269     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
270     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
271     if !File.file?(jscArg)
272         $stderr.puts "Error: must specify --jsc <path>"
273         exit 1
274     end
275
276     puts "Using the following jsc path: #{jscArg}"
277 end
278
279 if $enableFTL and !$jitTests
280     $stderr.puts "Error: can only specify one of --no-jit and --ftl-jit"
281     exit 1
282 end
283
284 if $doNotMessWithVMPath
285     $jscPath = Pathname.new(jscArg)
286 else
287     $jscPath = Pathname.new(jscArg).realpath
288 end
289
290 $progressMeter = ($verbosity == 0 and $stdout.tty?)
291
292 if $bundle
293     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
294     $outputDir = $bundle
295 end
296
297 # Try to determine architecture. Return nil on failure.
298 def machOArchitectureCode
299     begin 
300         otoolLines = `otool -aSfh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
301         otoolLines.each_with_index {
302             | value, index |
303             if value =~ /magic/ and value =~ /cputype/
304                 return otoolLines[index + 1].split[1].to_i
305             end
306         }
307     rescue
308         $stderr.puts "Warning: unable to execute otool."
309     end
310     $stderr.puts "Warning: unable to determine architecture."
311     nil
312 end
313
314 def determineArchitectureFromMachOBinary
315     code = machOArchitectureCode
316     return nil unless code
317     is64BitFlag = 0x01000000
318     case code
319     when 7
320         "x86"
321     when 7 | is64BitFlag
322         "x86-64"
323     when 12
324         "arm"
325     when 12 | is64BitFlag
326         "arm64"
327     else
328         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
329         nil
330     end
331 end
332
333 def determineArchitectureFromELFBinary
334     f = File.open($jscPath.to_s)
335     data = f.read(19)
336
337     if !(data[0,4] == "\x7F\x45\x4C\x46")
338         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
339         return nil
340     end
341
342     code = data[18].ord
343     case code
344     when 3
345         "x86"
346     when 62
347         "x86-64"
348     when 40
349         "arm"
350     when 183
351         "arm64"
352     else
353         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
354         nil
355     end
356 end
357
358 def determineArchitectureFromPEBinary
359     f = File.open($jscPath.to_s)
360     data = f.read(1024)
361
362     if !(data[0, 2] == "MZ")
363         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
364         return nil
365     end
366
367     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
368
369     if !(data[peHeaderAddr, 4] == "PE\0\0")
370         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
371         return nil
372     end
373
374     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
375
376     case machine
377     when 0x014c
378         "x86"
379     when 0x8664
380         "x86-64"
381     else
382         $stderr.puts "Warning: unsupported machine type: #{machine}"
383         nil
384     end
385 end
386
387 def determineArchitecture
388     case $hostOS
389     when "darwin"
390         determineArchitectureFromMachOBinary
391     when "linux"
392         determineArchitectureFromELFBinary
393     when "windows"
394         determineArchitectureFromPEBinary
395     else
396         $stderr.puts "Warning: unable to determine architecture on this platform."
397         nil
398     end
399 end
400
401 def determineOS
402     case RbConfig::CONFIG["host_os"]
403     when /darwin/i
404         "darwin"
405     when /linux/i
406         "linux"
407     when /mswin|mingw|cygwin/
408         "windows"
409     else
410         $stderr.puts "Warning: unable to determine host operating system"
411         nil
412     end
413 end
414
415 $hostOS = determineOS unless $hostOS
416 $architecture = determineArchitecture unless $architecture
417
418 if !$testRunnerType
419     if $remote and $hostOS == "darwin"
420         $testRunnerType = :shell
421     else
422         $testRunnerType = :make
423     end
424 end
425
426 $numFailures = 0
427 $numPasses = 0
428
429 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true"]
430 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000"]
431 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100"]
432 FTL_OPTIONS = ["--useFTLJIT=true"]
433
434 $runlist = []
435
436 def frameworkFromJSCPath(jscPath)
437     parentDirectory = jscPath.dirname
438     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
439         parentDirectory.dirname
440     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
441         jscPath.dirname + "JavaScriptCore.framework"
442     else
443         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
444         nil
445     end
446 end
447
448 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
449     dir = Pathname.new(".")
450     $benchmarkDirectory.each_filename {
451         | pathComponent |
452         dir += ".."
453     }
454     dir + resourcePath
455 end
456
457 def pathToVM
458     pathToBundleResourceFromBenchmarkDirectory($jscPath)
459 end
460
461 def pathToHelpers
462     pathToBundleResourceFromBenchmarkDirectory(".helpers")
463 end
464
465 def prefixCommand(prefix)
466     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
467 end
468
469 def redirectAndPrefixCommand(prefix)
470     prefixCommand(prefix) + " 2>&1"
471 end
472
473 def pipeAndPrefixCommand(outputFilename, prefix)
474     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
475 end
476
477 # Output handler for tests that are expected to be silent.
478 def silentOutputHandler
479     Proc.new {
480         | name |
481         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
482     }
483 end
484
485 # Output handler for tests that are expected to produce meaningful output.
486 def noisyOutputHandler
487     Proc.new {
488         | name |
489         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
490     }
491 end
492
493 # Error handler for tests that fail exactly when they return non-zero exit status.
494 # This is useful when a test is expected to fail.
495 def simpleErrorHandler
496     Proc.new {
497         | outp, plan |
498         outp.puts "if test -e #{plan.failFile}"
499         outp.puts "then"
500         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
501         outp.puts "    " + plan.failCommand
502         outp.puts "else"
503         outp.puts "    " + plan.successCommand
504         outp.puts "fi"
505     }
506 end
507
508 # Error handler for tests that fail exactly when they return zero exit status.
509 def expectedFailErrorHandler
510     Proc.new {
511         | outp, plan |
512         outp.puts "if test -e #{plan.failFile}"
513         outp.puts "then"
514         outp.puts "    " + plan.successCommand
515         outp.puts "else"
516         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
517         outp.puts "    " + plan.failCommand
518         outp.puts "fi"
519     }
520 end
521
522 # Error handler for tests that fail exactly when they return non-zero exit status and produce
523 # lots of spew. This will echo that spew when the test fails.
524 def noisyErrorHandler
525     Proc.new {
526         | outp, plan |
527         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
528     
529         outp.puts "if test -e #{plan.failFile}"
530         outp.puts "then"
531         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
532         outp.puts "    " + plan.failCommand
533         outp.puts "else"
534         outp.puts "    " + plan.successCommand
535         outp.puts "fi"
536     }
537 end
538
539 # Error handler for tests that diff their output with some expectation.
540 def diffErrorHandler(expectedFilename)
541     Proc.new {
542         | outp, plan |
543         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
544         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
545         
546         outp.puts "if test -e #{plan.failFile}"
547         outp.puts "then"
548         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
549         outp.puts "    " + plan.failCommand
550         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
551         outp.puts "then"
552         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
553         outp.puts "    if [ $? -eq 0 ]"
554         outp.puts "    then"
555         outp.puts "    " + plan.successCommand
556         outp.puts "    else"
557         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
558         outp.puts "        " + plan.failCommand
559         outp.puts "    fi"
560         outp.puts "else"
561         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
562         outp.puts "    " + plan.failCommand
563         outp.puts "fi"
564     }
565 end
566
567 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
568 # tests.
569 def mozillaErrorHandler
570     Proc.new {
571         | outp, plan |
572         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
573
574         outp.puts "if test -e #{plan.failFile}"
575         outp.puts "then"
576         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
577         outp.puts "    " + plan.failCommand
578         outp.puts "elif grep -i -q failed! #{outputFilename}"
579         outp.puts "then"
580         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
581         outp.puts "    " + plan.failCommand
582         outp.puts "else"
583         outp.puts "    " + plan.successCommand
584         outp.puts "fi"
585     }
586 end
587
588 # Error handler for tests that report error by saying "failed!", and are expected to
589 # fail. This is used by Mozilla tests.
590 def mozillaFailErrorHandler
591     Proc.new {
592         | outp, plan |
593         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
594
595         outp.puts "if test -e #{plan.failFile}"
596         outp.puts "then"
597         outp.puts "    " + plan.successCommand
598         outp.puts "elif grep -i -q failed! #{outputFilename}"
599         outp.puts "then"
600         outp.puts "    " + plan.successCommand
601         outp.puts "else"
602         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
603         outp.puts "    " + plan.failCommand
604         outp.puts "fi"
605     }
606 end
607
608 # Error handler for tests that report error by saying "failed!", and are expected to have
609 # an exit code of 3.
610 def mozillaExit3ErrorHandler
611     Proc.new {
612         | outp, plan |
613         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
614
615         outp.puts "if test -e #{plan.failFile}"
616         outp.puts "then"
617         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
618         outp.puts "    then"
619         outp.puts "        if grep -i -q failed! #{outputFilename}"
620         outp.puts "        then"
621         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
622         outp.puts "            " + plan.failCommand
623         outp.puts "        else"
624         outp.puts "            " + plan.successCommand
625         outp.puts "        fi"
626         outp.puts "    else"
627         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
628         outp.puts "        " + plan.failCommand
629         outp.puts "    fi"
630         outp.puts "else"
631         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
632         outp.puts "    " + plan.failCommand
633         outp.puts "fi"
634     }
635 end
636
637 $runCommandOptions = {}
638
639 class Plan
640     attr_reader :directory, :arguments, :family, :name, :outputHandler, :errorHandler
641     attr_accessor :index
642     
643     def initialize(directory, arguments, family, name, outputHandler, errorHandler)
644         @directory = directory
645         @arguments = arguments
646         @family = family
647         @name = name
648         @outputHandler = outputHandler
649         @errorHandler = errorHandler
650         @isSlow = !!$runCommandOptions[:isSlow]
651     end
652     
653     def shellCommand
654         # It's important to remember that the test is actually run in a subshell, so if we change directory
655         # in the subshell when we return we will be in our original directory. This is nice because we don't
656         # have to bend over backwards to do things relative to the root.
657         script = "(cd ../#{Shellwords.shellescape(@directory.to_s)} && ("
658         $envVars.each { |var| script += "export " << var << "; " }
659         script += "\"$@\" " + escapeAll(@arguments) + "))"
660         return script
661     end
662     
663     def reproScriptCommand
664         # We have to find our way back to the .runner directory since that's where all of the relative
665         # paths assume they start out from.
666         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
667         script += "cd $CURRENT_DIR\n"
668         Pathname.new(@name).dirname.each_filename {
669             | pathComponent |
670             script += "cd ..\n"
671         }
672         script += "cd .runner\n"
673
674         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
675         script += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])}\n"
676         $envVars.each { |var| script += "export " << var << "\n" }
677         script += "#{shellCommand} || exit 1"
678         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
679     end
680     
681     def failCommand
682         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
683     end
684     
685     def successCommand
686         if $progressMeter or $verbosity >= 2
687             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
688         else
689             "rm -f #{failFile}"
690         end
691     end
692     
693     def failFile
694         "test_fail_#{@index}"
695     end
696     
697     def writeRunScript(filename)
698         File.open(filename, "w") {
699             | outp |
700             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
701             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
702             cmd += @outputHandler.call(@name)
703             if $verbosity >= 3
704                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
705             end
706             outp.puts cmd
707             @errorHandler.call(outp, self)
708         }
709     end
710 end
711
712 $uniqueFilenameCounter = 0
713 def uniqueFilename(extension)
714     payloadDir = $outputDir + "_payload"
715     Dir.mkdir payloadDir unless payloadDir.directory?
716     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
717     $uniqueFilenameCounter += 1
718     result
719 end
720
721 def baseOutputName(kind)
722     "#{$collectionName}/#{$benchmark}.#{kind}"
723 end
724
725 def addRunCommand(kind, command, outputHandler, errorHandler)
726     $didAddRunCommand = true
727     name = baseOutputName(kind)
728     if $filter and name !~ $filter
729         return
730     end
731     plan = Plan.new(
732         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
733         errorHandler)
734     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
735         $runlist.unshift plan
736     else
737         $runlist << plan
738     end
739 end
740
741 # Returns true if there were run commands found in the file ($benchmarkDirectory +
742 # $benchmark), in which case those run commands have already been executed. Otherwise
743 # returns false, in which case you're supposed to add your own run commands.
744 def parseRunCommands
745     oldDidAddRunCommand = $didAddRunCommand
746     $didAddRunCommand = false
747
748     Dir.chdir($outputDir) {
749         File.open($benchmarkDirectory + $benchmark) {
750             | inp |
751             inp.each_line {
752                 | line |
753                 begin
754                     doesMatch = line =~ /^\/\/@/
755                 rescue Exception => e
756                     # Apparently this happens in the case of some UTF8 stuff in some files, where
757                     # Ruby tries to be strict and throw exceptions.
758                     next
759                 end
760                 next unless doesMatch
761                 eval $~.post_match
762             }
763         }
764     }
765
766     result = $didAddRunCommand
767     $didAddRunCommand = result or oldDidAddRunCommand
768     result
769 end
770
771 def slow!
772     $runCommandOptions[:isSlow] = true
773 end
774
775 def run(kind, *options)
776     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
777 end
778
779 def runDefault
780     run("default")
781 end
782
783 def runWithRAMSize(size)
784     run("ram-size-#{size}", "--forceRAMSize=#{size}")
785 end
786
787 def runOneLargeHeap
788     if $memoryLimited
789         $didAddRunCommand = true
790         puts "Skipping #{$collectionName}/#{$benchmark}"
791     else
792         run("default")
793     end
794 end
795
796 def runNoJIT
797     run("no-jit", "--useJIT=false")
798 end
799
800 def runNoLLInt
801     if $jitTests
802         run("no-llint", "--useLLInt=false")
803     end
804 end
805
806 def runNoCJITValidate
807     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *NO_CJIT_OPTIONS)
808 end
809
810 def runNoCJITValidatePhases
811     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
812 end
813
814 def runDefaultFTL
815     run("default-ftl", *FTL_OPTIONS) if $enableFTL
816 end
817
818 def runFTLNoCJIT
819     run("ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
820 end
821
822 def runFTLNoCJITValidate
823     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
824 end
825
826 def runFTLNoCJITNoPutStackValidate
827     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
828 end
829
830 def runFTLNoCJITNoInlineValidate
831     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
832 end
833
834 def runFTLNoCJITOSRValidation
835     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
836 end
837
838 def runDFGEager
839     run("dfg-eager", *EAGER_OPTIONS)
840 end
841
842 def runDFGEagerNoCJITValidate
843     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
844 end
845
846 def runFTLEager
847     run("ftl-eager", *(FTL_OPTIONS + EAGER_OPTIONS)) if $enableFTL
848 end
849
850 def runFTLEagerNoCJITValidate
851     run("ftl-eager-no-cjit", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
852 end
853
854 def runFTLEagerNoCJITOSRValidation
855     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
856 end
857
858 def runAlwaysTriggerCopyPhase
859     run("always-trigger-copy-phase", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
860 end
861
862 def runNoCJITNoASO
863     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *NO_CJIT_OPTIONS)
864 end
865
866 def runNoCJITNoAccessInlining
867     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *NO_CJIT_OPTIONS)
868 end
869
870 def runFTLNoCJITNoAccessInlining
871     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
872 end
873
874 def runFTLNoCJITSmallPool
875     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
876 end
877
878 def runMiscNoCJITTest(*options)
879     run("misc-no-cjit", *(NO_CJIT_OPTIONS + options))
880 end
881
882 def runMiscFTLNoCJITTest(*options)
883     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + options))
884 end
885
886 def runDFGMaximalFlushPhase
887     run("dfg-maximal-flush-validate-no-cjit", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *NO_CJIT_OPTIONS)
888 end
889
890 def runShadowChicken
891     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true")
892 end
893
894 def defaultRun
895     if $quickMode
896         defaultQuickRun
897     else
898         runDefault
899         runAlwaysTriggerCopyPhase
900         if $jitTests
901             runNoLLInt
902             runNoCJITValidatePhases
903             runDFGEager
904             runDFGEagerNoCJITValidate
905             runDefaultFTL
906             runFTLNoCJITValidate
907             runFTLNoCJITNoPutStackValidate
908             runFTLNoCJITNoInlineValidate
909             runFTLEager
910             runFTLEagerNoCJITValidate
911             runFTLNoCJITSmallPool
912             runDFGMaximalFlushPhase
913         end
914     end
915 end
916
917 def defaultNoNoLLIntRun
918     if $quickMode
919         defaultQuickRun
920     else
921         runDefault
922         runAlwaysTriggerCopyPhase
923         if $jitTests
924             runNoCJITValidatePhases
925             runDFGEager
926             runDFGEagerNoCJITValidate
927             runDefaultFTL
928             runFTLNoCJITValidate
929             runFTLNoCJITNoPutStackValidate
930             runFTLNoCJITNoInlineValidate
931             runFTLEager
932             runFTLEagerNoCJITValidate
933             runFTLNoCJITSmallPool
934             runDFGMaximalFlushPhase
935         end
936     end
937 end
938
939 def defaultQuickRun
940     if $enableFTL and $jitTests
941         runDefaultFTL
942         runFTLNoCJITValidate
943     else
944         runDefault
945         if $jitTests
946             runNoCJITValidate
947         end
948     end
949 end
950
951 def defaultSpotCheckNoMaximalFlush
952     defaultQuickRun
953     runFTLNoCJITOSRValidation
954     runNoCJITNoAccessInlining
955     runFTLNoCJITNoAccessInlining
956 end
957
958 def defaultSpotCheck
959     defaultSpotCheckNoMaximalFlush
960     runDFGMaximalFlushPhase
961 end
962
963 # This is expected to not do eager runs because eager runs can have a lot of recompilations
964 # for reasons that don't arise in the real world. It's used for tests that assert convergence
965 # by counting recompilations.
966 def defaultNoEagerRun
967     runDefault
968     runAlwaysTriggerCopyPhase
969     if $jitTests
970         runNoLLInt
971         runNoCJITValidatePhases
972         runDefaultFTL
973         runFTLNoCJITValidate
974         runFTLNoCJITNoInlineValidate
975     end
976 end
977
978 def defaultNoSamplingProfilerRun
979     runDefault
980     runAlwaysTriggerCopyPhase
981     if $jitTests
982         runNoLLInt
983         runNoCJITValidatePhases
984         runDFGEager
985         runDFGEagerNoCJITValidate
986         runDefaultFTL
987         runFTLNoCJITNoPutStackValidate
988         runFTLNoCJITNoInlineValidate
989         runFTLEager
990         runFTLEagerNoCJITValidate
991         runFTLNoCJITSmallPool
992         runDFGMaximalFlushPhase
993     end
994 end
995
996 def runProfiler
997     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
998         skip
999         return
1000     end
1001
1002     profilerOutput = uniqueFilename(".json")
1003     if $canRunDisplayProfilerOutput
1004         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "--useConcurrentJIT=false", "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
1005     else
1006         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
1007         run("profiler-simple", "--useConcurrentJIT=false", "-p", profilerOutput.to_s)
1008     end
1009 end
1010
1011 def runExceptionFuzz
1012     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
1013     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
1014 end
1015
1016 def runExecutableAllocationFuzz(name, *options)
1017     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
1018     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
1019 end
1020
1021 def runTypeProfiler
1022     if !$jitTests
1023         return
1024     end
1025
1026     if $enableFTL
1027         run("ftl-no-cjit-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1028         run("ftl-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS))
1029     else
1030         run("no-cjit-type-profiler", "--useTypeProfiler=true", *NO_CJIT_OPTIONS)
1031         run("type-profiler", "--useTypeProfiler=true")
1032     end
1033 end
1034
1035 def runControlFlowProfiler
1036     if !$jitTests
1037         return
1038     end
1039
1040     if $enableFTL
1041         run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1042     else
1043         run("no-cjit-type-profiler", "--useControlFlowProfiler=true", *NO_CJIT_OPTIONS)
1044     end
1045 end
1046
1047 def runTest262(mode, exception, includeFiles, flags)
1048     failsWithException = exception != "NoException"
1049     isStrict = false
1050     isModule = false
1051     isAsync = false
1052
1053     flags.each {
1054         | flag |
1055         case flag
1056         when :strict
1057             isStrict = true
1058         when :module
1059             isModule = true
1060         when :async
1061             isAsync = true
1062         else
1063             raise "Invalid flag for runTest262, #{flag}"
1064         end
1065     }
1066
1067     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
1068
1069     args = [pathToVM.to_s] + BASE_OPTIONS
1070     args << "-m" if isModule
1071     args << "--exception=" + exception if failsWithException
1072     args << "--test262-async" if isAsync
1073     args += includeFiles
1074
1075     case mode
1076     when :normal
1077         errorHandler = simpleErrorHandler
1078         outputHandler = silentOutputHandler
1079     when :fail
1080         errorHandler = expectedFailErrorHandler
1081         outputHandler = noisyOutputHandler
1082     else
1083         raise "Invalid mode: #{mode}"
1084     end
1085
1086     if isStrict
1087         kind = "default-strict"
1088         args << "--strict-file=#{$benchmark}"
1089     else
1090         kind = "default"
1091         args << $benchmark.to_s
1092     end
1093
1094     addRunCommand(kind, args, outputHandler, errorHandler)
1095 end
1096
1097 def prepareTest262Fixture
1098     # This function is used to add the files used by Test262 modules tests.
1099     prepareExtraRelativeFiles([""], $collection)
1100 end
1101
1102 def runES6(mode)
1103     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
1104     case mode
1105     when :normal
1106         errorHandler = simpleErrorHandler
1107     when :fail
1108         errorHandler = expectedFailErrorHandler
1109     else
1110         raise "Invalid mode: #{mode}"
1111     end
1112     addRunCommand("default", args, noisyOutputHandler, errorHandler)
1113 end
1114
1115 def runModules
1116     run("default-modules", "-m")
1117     run("always-trigger-copy-phase-modules", "-m", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
1118
1119     if !$jitTests
1120         return
1121     end
1122
1123     run("no-llint-modules", "-m", "--useLLInt=false")
1124     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
1125     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
1126     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1127     if $enableFTL
1128         run("default-ftl-modules", "-m", *FTL_OPTIONS)
1129         run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1130         run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1131         run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
1132         run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1133         run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1134     end
1135 end
1136
1137 def runLayoutTest(kind, *options)
1138     raise unless $benchmark.to_s =~ /\.js$/
1139     testName = $~.pre_match
1140     if kind
1141         kind = "layout-" + kind
1142     else
1143         kind = "layout"
1144     end
1145
1146     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1147     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1148
1149     args = [pathToVM.to_s] + BASE_OPTIONS + options +
1150         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1151          $benchmark.to_s,
1152          (Pathname.new("resources") + "standalone-post.js").to_s]
1153     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1154 end
1155
1156 def runLayoutTestDefault
1157     runLayoutTest(nil)
1158 end
1159
1160 def runLayoutTestNoLLInt
1161     runLayoutTest("no-llint", "--useLLInt=false")
1162 end
1163
1164 def runLayoutTestNoCJIT
1165     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1166 end
1167
1168 def runLayoutTestDFGEagerNoCJIT
1169     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1170 end
1171
1172 def runLayoutTestDefaultFTL
1173     runLayoutTest("ftl", "--testTheFTL=true", *FTL_OPTIONS) if $enableFTL
1174 end
1175
1176 def runLayoutTestFTLNoCJIT
1177     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
1178 end
1179
1180 def runLayoutTestFTLEagerNoCJIT
1181     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
1182 end
1183
1184 def noFTLRunLayoutTest
1185     if !$jitTests
1186         return
1187     end
1188
1189     runLayoutTestNoLLInt
1190     runLayoutTestNoCJIT
1191     runLayoutTestDFGEagerNoCJIT
1192 end
1193
1194 def defaultQuickRunLayoutTest
1195     runLayoutTestDefault
1196     if $jitTests
1197         runLayoutTestFTLNoCJIT
1198         runLayoutTestFTLEagerNoCJIT
1199     end
1200 end
1201
1202 def defaultRunLayoutTest
1203     if $quickMode
1204         defaultQuickRunLayoutTest
1205     else
1206         runLayoutTestDefault
1207         if $jitTests
1208             noFTLRunLayoutTest
1209             runLayoutTestDefaultFTL
1210             runLayoutTestFTLNoCJIT
1211             runLayoutTestFTLEagerNoCJIT
1212         end
1213     end
1214 end
1215
1216 def noEagerNoNoLLIntTestsRunLayoutTest
1217     runLayoutTestDefault
1218     if $jitTests
1219         runLayoutTestNoCJIT
1220         runLayoutTestDefaultFTL
1221         runLayoutTestFTLNoCJIT
1222     end
1223 end
1224
1225 def noNoLLIntRunLayoutTest
1226     runLayoutTestDefault
1227     if $jitTests
1228         runLayoutTestNoCJIT
1229         runLayoutTestDFGEagerNoCJIT
1230         runLayoutTestDefaultFTL
1231         runLayoutTestFTLNoCJIT
1232         runLayoutTestFTLEagerNoCJIT
1233     end
1234 end
1235
1236 def prepareExtraRelativeFiles(extraFiles, destination)
1237     Dir.chdir($outputDir) {
1238         extraFiles.each {
1239             | file |
1240             dest = destination + file
1241             FileUtils.mkdir_p(dest.dirname)
1242             FileUtils.cp $extraFilesBaseDir + file, dest
1243         }
1244     }
1245 end
1246
1247 def baseDirForCollection(collectionName)
1248     Pathname(".tests") + collectionName
1249 end
1250
1251 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1252     raise unless absoluteBase.absolute?
1253     Dir.chdir($outputDir) {
1254         collectionBaseDir = baseDirForCollection($collectionName)
1255         extraFiles.each {
1256             | file |
1257             destination = collectionBaseDir + file
1258             FileUtils.mkdir_p destination.dirname unless destination.directory?
1259             FileUtils.cp absoluteBase + file, destination
1260         }
1261     }
1262 end
1263
1264 def runMozillaTest(kind, mode, extraFiles, *options)
1265     if kind
1266         kind = "mozilla-" + kind
1267     else
1268         kind = "mozilla"
1269     end
1270     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1271     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1272     case mode
1273     when :normal
1274         errorHandler = mozillaErrorHandler
1275     when :negative
1276         errorHandler = mozillaExit3ErrorHandler
1277     when :fail
1278         errorHandler = mozillaFailErrorHandler
1279     when :skip
1280         return
1281     else
1282         raise "Invalid mode: #{mode}"
1283     end
1284     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1285 end
1286
1287 def runMozillaTestDefault(mode, *extraFiles)
1288     runMozillaTest(nil, mode, extraFiles)
1289 end
1290
1291 def runMozillaTestDefaultFTL(mode, *extraFiles)
1292     runMozillaTest("ftl", mode, extraFiles, *FTL_OPTIONS) if $enableFTL
1293 end
1294
1295 def runMozillaTestLLInt(mode, *extraFiles)
1296     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1297 end
1298
1299 def runMozillaTestBaselineJIT(mode, *extraFiles)
1300     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1301 end
1302
1303 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1304     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1305 end
1306
1307 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1308     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
1309 end
1310
1311 def defaultQuickRunMozillaTest(mode, *extraFiles)
1312     if $enableFTL and $jitTests
1313         runMozillaTestDefaultFTL(mode, *extraFiles)
1314         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1315     else
1316         runMozillaTestDefault(mode, *extraFiles)
1317         if $jitTests
1318             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1319         end
1320     end
1321 end
1322
1323 def defaultRunMozillaTest(mode, *extraFiles)
1324     if $quickMode
1325         defaultQuickRunMozillaTest(mode, *extraFiles)
1326     else
1327         runMozillaTestDefault(mode, *extraFiles)
1328         if $jitTests
1329             runMozillaTestLLInt(mode, *extraFiles)
1330             runMozillaTestBaselineJIT(mode, *extraFiles)
1331             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1332             runMozillaTestDefaultFTL(mode, *extraFiles)
1333             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1334         end
1335     end
1336 end
1337
1338 def runNoisyTest(kind, *options)
1339     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
1340 end
1341
1342 def runNoisyTestDefault
1343     runNoisyTest("default")
1344 end
1345
1346 def runNoisyTestDefaultFTL
1347     runNoisyTest("ftl", *FTL_OPTIONS) if $enableFTL
1348 end
1349
1350 def runNoisyTestNoCJIT
1351     runNoisyTest($enableFTL ? "ftl-no-cjit" : "no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(($enableFTL ? FTL_OPTIONS : []) + NO_CJIT_OPTIONS))
1352 end
1353
1354 def runNoisyTestEagerNoCJIT
1355     runNoisyTest($enableFTL ? "ftl-eager-no-cjit" : "eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(($enableFTL ? FTL_OPTIONS : []) + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1356 end
1357
1358 def defaultRunNoisyTest
1359     runNoisyTestDefault
1360     if $jitTests
1361         runNoisyTestDefaultFTL
1362         runNoisyTestNoCJIT
1363         runNoisyTestEagerNoCJIT
1364     end
1365 end
1366
1367 def skip
1368     $didAddRunCommand = true
1369     puts "Skipping #{$collectionName}/#{$benchmark}"
1370 end
1371
1372 def largeHeap
1373     if $memoryLimited
1374         $didAddRunCommand = true
1375         puts "Skipping #{$collectionName}/#{$benchmark}"
1376     end
1377 end
1378
1379 def allJSFiles(path)
1380     if path.file?
1381         [path]
1382     else
1383         result = []
1384         Dir.foreach(path) {
1385             | filename |
1386             next unless filename =~ /\.js$/
1387             next unless (path + filename).file?
1388             result << path + filename
1389         }
1390         result
1391     end
1392 end
1393
1394 def uniqueifyName(names, name)
1395     result = name.to_s
1396     toAdd = 1
1397     while names[result]
1398         result = "#{name}-#{toAdd}"
1399         toAdd += 1
1400     end
1401     names[result] = true
1402     result
1403 end
1404
1405 def simplifyCollectionName(collectionPath)
1406     outerDir = collectionPath.dirname
1407     name = collectionPath.basename
1408     lastName = name
1409     if collectionPath.directory?
1410         while lastName.to_s =~ /test/
1411             lastName = outerDir.basename
1412             name = lastName + name
1413             outerDir = outerDir.dirname
1414         end
1415     end
1416     uniqueifyName($collectionNames, name)
1417 end
1418
1419 def prepareCollection(name)
1420     FileUtils.mkdir_p $outputDir + name
1421
1422     absoluteCollection = $collection.realpath
1423
1424     Dir.chdir($outputDir) {
1425         bundleDir = baseDirForCollection(name)
1426
1427         # Create the proper directory structures.
1428         FileUtils.mkdir_p bundleDir
1429         if bundleDir.basename == $collection.basename
1430             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1431             $collection = bundleDir
1432         else
1433             FileUtils.cp_r absoluteCollection, bundleDir
1434             $collection = bundleDir + $collection.basename
1435         end
1436
1437         $extraFilesBaseDir = absoluteCollection
1438     }
1439 end
1440
1441 $collectionNames = {}
1442
1443 def handleCollectionFile(collection)
1444     collectionName = simplifyCollectionName(collection)
1445    
1446     paths = {}
1447     subCollections = []
1448     YAML::load(IO::read(collection)).each {
1449         | entry |
1450         if entry["collection"]
1451             subCollections << entry["collection"]
1452             next
1453         end
1454         
1455         if Pathname.new(entry["path"]).absolute?
1456             raise "Absolute path: " + entry["path"] + " in #{collection}"
1457         end
1458         
1459         if paths[entry["path"]]
1460             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1461         end
1462         
1463         subCollection = collection.dirname + entry["path"]
1464         
1465         if subCollection.file?
1466             subCollectionName = Pathname.new(entry["path"]).dirname
1467         else
1468             subCollectionName = entry["path"]
1469         end
1470         
1471         $collection = subCollection
1472         $collectionName = Pathname.new(collectionName)
1473         Pathname.new(subCollectionName).each_filename {
1474             | filename |
1475             next if filename =~ /^\./
1476             $collectionName += filename
1477         }
1478         $collectionName = $collectionName.to_s
1479         
1480         prepareCollection($collectionName)
1481       
1482         Dir.chdir($outputDir) {
1483             pathsToSearch = [$collection]
1484             if entry["tests"]
1485                 if entry["tests"].is_a? Array
1486                     pathsToSearch = entry["tests"].map {
1487                         | testName |
1488                         pathsToSearch[0] + testName
1489                     }
1490                 else
1491                     pathsToSearch[0] += entry["tests"]
1492                 end
1493             end
1494             pathsToSearch.each {
1495                 | pathToSearch |
1496                 allJSFiles(pathToSearch).each {
1497                     | path |
1498                     
1499                     $benchmark = path.basename
1500                     $benchmarkDirectory = path.dirname
1501                     
1502                     $runCommandOptions = {}
1503                     eval entry["cmd"]
1504                 }
1505             }
1506         }
1507     }
1508     
1509     subCollections.each {
1510         | subCollection |
1511         handleCollection(collection.dirname + subCollection)
1512     }
1513 end
1514
1515 def handleCollectionDirectory(collection)
1516     collectionName = simplifyCollectionName(collection)
1517     
1518     $collection = collection
1519     $collectionName = collectionName
1520     prepareCollection(collectionName)
1521    
1522     Dir.chdir($outputDir) {
1523         $benchmarkDirectory = $collection
1524         allJSFiles($collection).each {
1525             | path |
1526             
1527             $benchmark = path.basename
1528             
1529             $runCommandOptions = {}
1530             defaultRun unless parseRunCommands
1531         }
1532     }
1533 end
1534
1535 def handleCollection(collection)
1536     collection = Pathname.new(collection)
1537     
1538     if collection.file?
1539         handleCollectionFile(collection)
1540     else
1541         handleCollectionDirectory(collection)
1542     end
1543 end
1544
1545 def appendFailure(plan)
1546     File.open($outputDir + "failed", "a") {
1547         | outp |
1548         outp.puts plan.name
1549     }
1550     $numFailures += 1
1551 end
1552
1553 def appendPass(plan)
1554     File.open($outputDir + "passed", "a") {
1555         | outp |
1556         outp.puts plan.name
1557     }
1558     $numPasses += 1
1559 end
1560
1561 def appendResult(plan, didPass)
1562     File.open($outputDir + "results", "a") {
1563         | outp |
1564         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1565     }
1566 end
1567
1568 def prepareBundle
1569     raise if $bundle
1570
1571     if $doNotMessWithVMPath
1572         if !$remote and !$tarball
1573             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1574             $jscPath = Pathname.new($jscPath).realpath
1575         else
1576             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1577         end
1578     else
1579         originalJSCPath = $jscPath
1580         vmDir = $outputDir + ".vm"
1581         FileUtils.mkdir_p vmDir
1582         
1583         frameworkPath = frameworkFromJSCPath($jscPath)
1584         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1585         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1586         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1587
1588         if frameworkPath
1589             source = frameworkPath
1590             destination = Pathname.new(".vm")
1591         else
1592             source = originalJSCPath
1593             destination = $jscPath
1594
1595             Dir.chdir($outputDir) {
1596                 FileUtils.mkdir_p $jscPath.dirname
1597             }
1598         end
1599
1600         Dir.chdir($outputDir) {
1601             if $copyVM
1602                 FileUtils.cp_r source, destination
1603             else
1604                 begin 
1605                     FileUtils.ln_s source, destination
1606                 rescue Exception
1607                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1608                     FileUtils.cp_r source, destination
1609                 end
1610             end
1611
1612             if $remote and $hostOS == "linux"
1613                 begin
1614                     dependencies = `ldd #{source}`
1615                     dependencies.split(/\n/).each {
1616                         | dependency |
1617                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1618                     }
1619                 rescue
1620                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1621                 end
1622             end
1623         }
1624     end
1625     
1626     Dir.chdir($outputDir) {
1627         FileUtils.cp_r HELPERS_PATH, ".helpers"
1628     }
1629
1630     ARGV.each {
1631         | collection |
1632         handleCollection(collection)
1633     }
1634
1635     puts
1636 end
1637
1638 def cleanOldResults
1639     raise unless $bundle
1640
1641     eachResultFile($outputDir) {
1642         | path |
1643         FileUtils.rm_f path
1644     }
1645 end
1646
1647 def cleanEmptyResultFiles
1648     eachResultFile($outputDir) {
1649         | path |
1650         next unless path.basename.to_s =~ /\.out$/
1651         next unless FileTest.size(path) == 0
1652         FileUtils.rm_f path
1653     }
1654 end
1655
1656 def eachResultFile(startingDir, &block)
1657     dirsToClean = [startingDir]
1658     until dirsToClean.empty?
1659         nextDir = dirsToClean.pop
1660         Dir.foreach(nextDir) {
1661             | entry |
1662             next if entry =~ /^\./
1663             path = nextDir + entry
1664             if path.directory?
1665                 dirsToClean.push(path)
1666             else
1667                 block.call(path)
1668             end
1669         }
1670     end
1671 end
1672
1673 def prepareTestRunner
1674     raise if $bundle
1675
1676     $runlist.each_with_index {
1677         | plan, index |
1678         plan.index = index
1679     }
1680
1681     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1682     toDelete = []
1683     Dir.foreach($runnerDir) {
1684         | filename |
1685         if filename =~ /^test_/
1686             toDelete << filename
1687         end
1688     }
1689     
1690     toDelete.each {
1691         | filename |
1692         File.unlink($runnerDir + filename)
1693     }
1694
1695     $runlist.each {
1696         | plan |
1697         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1698     }
1699
1700     case $testRunnerType
1701     when :make
1702         prepareMakeTestRunner
1703     when :shell
1704         prepareShellTestRunner
1705     else
1706         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1707     end
1708 end
1709
1710 def prepareShellTestRunner
1711     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
1712 end
1713
1714 def prepareMakeTestRunner
1715     # The goals of our parallel test runner are scalability and simplicity. The
1716     # simplicity part is particularly important. We don't want to have to have
1717     # a full-time contributor just philosophising about parallel testing.
1718     #
1719     # As such, we just pass off all of the hard work to 'make'. This creates a
1720     # dummy directory ("$outputDir/.runner") in which we create a dummy
1721     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
1722     # That is, for each test we know we will run, there is a rule in the
1723     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
1724     # Makefile results in 'make' doing all of the hard work:
1725     #
1726     # - Load balancing just works. Most systems have a great load balancer in
1727     #   'make'. If your system doesn't then just install a real 'make'.
1728     #
1729     # - Interruptions just work. For example Ctrl-C handling in 'make' is
1730     #   exactly right. You don't have to worry about zombie processes.
1731     #
1732     # We then do some tricks to make failure detection work and to make this
1733     # totally sound. If a test fails, we don't want the whole 'make' job to
1734     # stop. We also don't have any facility for makefile-escaping of path names.
1735     # We do have such a thing for shell-escaping, though. We fix both problems
1736     # by having the actual work for each of the test rules be done in a shell
1737     # script on the side. There is one such script per test. The script responds
1738     # to failure by printing something on the console and then touching a
1739     # failure file for that test, but then still returns 0. This makes 'make'
1740     # continue past that failure and complete all the tests anyway.
1741     #
1742     # In the end, this script collects all of the failures by searching for
1743     # files in the .runner directory whose name matches /^test_fail_/, where
1744     # the thing after the 'fail_' is the test index. Those are the files that
1745     # would be created by the test scripts if they detect failure. We're
1746     # basically using the filesystem as a concurrent database of test failures.
1747     # Even if two tests fail at the same time, since they're touching different
1748     # files we won't miss any failures.
1749     runIndices = []
1750     $runlist.each {
1751         | plan |
1752         runIndices << plan.index
1753     }
1754     
1755     File.open($runnerDir + "Makefile", "w") {
1756         | outp |
1757         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1758         runIndices.each {
1759             | index |
1760             plan = $runlist[index]
1761             outp.puts "test_done_#{index}:"
1762             outp.puts "\tsh test_script_#{plan.index}"
1763         }
1764     }
1765 end
1766
1767 def cleanRunnerDirectory
1768     raise unless $bundle
1769     Dir.foreach($runnerDir) {
1770         | filename |
1771         next unless filename =~ /^test_fail/
1772         FileUtils.rm_f $runnerDir + filename
1773     }
1774 end
1775
1776 def sshRead(cmd)
1777     raise unless $remote
1778
1779     result = ""
1780     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1781       | inp |
1782       inp.each_line {
1783         | line |
1784         result += line
1785       }
1786     }
1787     raise "#{$?}" unless $?.success?
1788     result
1789 end
1790
1791 def runCommandOnTester(cmd)
1792     if $remote
1793         result = sshRead(cmd)
1794     else
1795         result = `#{cmd}`
1796     end
1797 end
1798
1799 def numberOfProcessors
1800     if $hostOS == "windows"
1801         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1802     else
1803         begin
1804             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1805         rescue
1806             numProcessors = 0
1807         end
1808
1809         if numProcessors == 0
1810             begin
1811                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1812             rescue
1813                 numProcessors == 0
1814             end
1815         end
1816     end
1817
1818     if numProcessors == 0
1819         numProcessors = 1
1820     end
1821     return numProcessors
1822 end
1823
1824 def runAndMonitorTestRunnerCommand(*cmd)
1825     numberOfTests = 0
1826     Dir.chdir($runnerDir) {
1827         # -1 for the runscript, and -2 for '..' and '.'
1828         numberOfTests = Dir.entries(".").count - 3
1829     }
1830     unless $progressMeter
1831         mysys(cmd.join(' '))
1832     else
1833        running = {}
1834        didRun = {}
1835        didFail = {}
1836        blankLine = true
1837        prevStringLength = 0
1838        IO.popen(cmd.join(' '), mode="r") {
1839            | inp |
1840            inp.each_line {
1841                | line |
1842                line = line.scrub.chomp
1843                if line =~ /^Running /
1844                    running[$~.post_match] = true
1845                elsif line =~ /^PASS: /
1846                    didRun[$~.post_match] = true
1847                elsif line =~ /^FAIL: /
1848                    didRun[$~.post_match] = true
1849                    didFail[$~.post_match] = true
1850                else
1851                    unless blankLine
1852                        print("\r" + " " * prevStringLength + "\r")
1853                    end
1854                    puts line
1855                    blankLine = true
1856                end
1857
1858                def lpad(str, chars)
1859                    str = str.to_s
1860                    if str.length > chars
1861                        str
1862                    else
1863                       "%#{chars}s"%(str)
1864                    end
1865                end
1866
1867                string  = ""
1868                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1869                unless didFail.empty?
1870                    string += " (failed #{didFail.size})"
1871                end
1872                string += " "
1873                (running.size - didRun.size).times {
1874                    string += "."
1875                }
1876                if string.length < prevStringLength
1877                    print string
1878                    print(" " * (prevStringLength - string.length))
1879                end
1880                print string
1881                prevStringLength = string.length
1882                blankLine = false
1883                $stdout.flush
1884            }
1885        }
1886        puts
1887        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1888     end
1889 end
1890
1891 def runTestRunner
1892     case $testRunnerType
1893     when :shell
1894         testRunnerCommand = "sh runscript"
1895     when :make
1896         testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
1897     else
1898         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1899     end
1900
1901     if $remote
1902         if !$remoteDirectory
1903             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1904         end
1905         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1906         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1907         remoteScript = "\""
1908         remoteScript += "cd #{$remoteDirectory} && "
1909         remoteScript += "rm -rf #{$outputDir.basename} && "
1910         remoteScript += "tar xzf #{$tarFileName} && "
1911         remoteScript += "cd #{$outputDir.basename}/.runner && "
1912         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1913         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1914         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
1915         $envVars.each { |var| remoteScript += "export " << var << "\n" }
1916         remoteScript += "#{testRunnerCommand}\""
1917         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1918     else
1919         Dir.chdir($runnerDir) {
1920             runAndMonitorTestRunnerCommand(testRunnerCommand)
1921         }
1922     end
1923 end
1924
1925 def detectFailures
1926     raise if $bundle
1927
1928     failures = []
1929     
1930     if $remote
1931         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1932         output.split(/\n/).each {
1933             | line |
1934             next unless line =~ /test_fail_/
1935             failures << $~.post_match.to_i
1936         }
1937     else
1938         Dir.foreach($runnerDir) {
1939             | filename |
1940             next unless filename =~ /test_fail_/
1941             failures << $~.post_match.to_i
1942         }
1943     end
1944
1945     failureSet = {}
1946
1947     failures.each {
1948         | failure | 
1949         appendFailure($runlist[failure])
1950         failureSet[failure] = true
1951     }
1952
1953     familyMap = {}
1954     $runlist.each_with_index {
1955         | plan, index |
1956         unless familyMap[plan.family]
1957             familyMap[plan.family] = []
1958         end
1959         if failureSet[index]
1960             appendResult(plan, false)
1961             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
1962             next
1963         else
1964             appendResult(plan, true)
1965             familyMap[plan.family] << {:result => "PASS", :plan => plan};
1966         end
1967         appendPass(plan)
1968     }
1969
1970     File.open($outputDir + "resultsByFamily", "w") {
1971         | outp |
1972         first = true
1973         familyMap.keys.sort.each {
1974             | familyName |
1975             if first
1976                 first = false
1977             else
1978                 outp.puts
1979             end
1980             
1981             outp.print "#{familyName}:"
1982
1983             numPassed = 0
1984             familyMap[familyName].each {
1985                 | entry |
1986                 if entry[:result] == "PASS"
1987                     numPassed += 1
1988                 end
1989             }
1990
1991             if numPassed == familyMap[familyName].size
1992                 outp.puts " PASSED"
1993             elsif numPassed == 0
1994                 outp.puts " FAILED"
1995             else
1996                 outp.puts
1997                 familyMap[familyName].each {
1998                     | entry |
1999                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
2000                 }
2001             end
2002         }
2003     }
2004 end
2005
2006 def compressBundle
2007     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
2008     $stderr.puts ">> #{cmd}" if $verbosity >= 2
2009     raise unless system(cmd)
2010 end
2011
2012 def clean(file)
2013     FileUtils.rm_rf file unless $bundle
2014 end
2015
2016 clean($outputDir + "failed")
2017 clean($outputDir + "passed")
2018 clean($outputDir + "results")
2019 clean($outputDir + "resultsByFamily")
2020 clean($outputDir + ".vm")
2021 clean($outputDir + ".helpers")
2022 clean($outputDir + ".runner")
2023 clean($outputDir + ".tests")
2024 clean($outputDir + "_payload")
2025
2026 Dir.mkdir($outputDir) unless $outputDir.directory?
2027
2028 $outputDir = $outputDir.realpath
2029 $runnerDir = $outputDir + ".runner"
2030
2031 if !$numChildProcesses
2032     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
2033         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
2034     else
2035         $numChildProcesses = numberOfProcessors
2036     end
2037 end
2038
2039 if $enableFTL and ENV["JSCTEST_timeout"] or !ifJSCArgIsntProvidedAreWeReleaseBuild
2040     # Currently, using the FTL is a performance regression particularly in real
2041     # (i.e. non-loopy) benchmarks. Account for this in the timeout.
2042     # Increase the timeout for debug builds too. (--debug command line option)
2043     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i * 2).to_s
2044 end
2045
2046 if ENV["JSCTEST_timeout"]
2047     # In the worst case, the processors just interfere with each other.
2048     # Increase the timeout proportionally to the number of processors.
2049     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
2050 end
2051
2052 def runBundle
2053     raise unless $bundle
2054
2055     cleanRunnerDirectory
2056     cleanOldResults
2057     runTestRunner
2058     cleanEmptyResultFiles
2059 end
2060
2061 def runNormal
2062     raise if $bundle or $tarball
2063
2064     prepareBundle
2065     prepareTestRunner
2066     runTestRunner
2067     cleanEmptyResultFiles
2068     detectFailures
2069 end
2070
2071 def runTarball
2072     raise unless $tarball
2073
2074     prepareBundle 
2075     prepareTestRunner
2076     compressBundle
2077 end
2078
2079 def runRemote
2080     raise unless $remote
2081
2082     prepareBundle
2083     prepareTestRunner
2084     compressBundle
2085     runTestRunner
2086     detectFailures
2087 end
2088
2089 puts
2090 if $bundle
2091     runBundle
2092 elsif $remote
2093     runRemote
2094 elsif $tarball
2095     runTarball
2096 else
2097     runNormal
2098 end