[JSC] Change some parameters based on a random search
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2015 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
47 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
48 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
49 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
50 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
51 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
52
53 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
54
55 begin
56     require 'shellwords'
57 rescue Exception => e
58     $stderr.puts "Warning: did not find shellwords, not running any tests."
59     exit 0
60 end
61
62 $canRunDisplayProfilerOutput = false
63
64 begin
65     require 'rubygems'
66     require 'json'
67     require 'highline'
68     $canRunDisplayProfilerOutput = true
69 rescue Exception => e
70     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
71     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
72     $stderr.puts "Error: #{e.inspect}"
73 end
74
75 def printCommandArray(*cmd)
76     begin
77         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
78     rescue
79         commandArray = cmd.join(' ')
80     end
81     $stderr.puts ">> #{commandArray}"
82 end
83
84 def mysys(*cmd)
85     printCommandArray(*cmd) if $verbosity >= 1
86     raise "Command failed: #{$?.inspect}" unless system(*cmd)
87 end
88
89 def escapeAll(array)
90     array.map {
91         | v |
92         raise "Detected a non-string in #{inspect}" unless v.is_a? String
93         Shellwords.shellescape(v)
94     }.join(' ')
95 end
96
97
98 $jscPath = nil
99 $doNotMessWithVMPath = false
100 $enableFTL = false
101 $jitTests = true
102 $memoryLimited = false
103 $outputDir = Pathname.new("results")
104 $verbosity = 0
105 $bundle = nil
106 $tarball = false
107 $tarFileName = "payload.tar.gz"
108 $copyVM = false
109 $testRunnerType = nil
110 $remoteUser = nil
111 $remoteHost = nil
112 $remotePort = nil
113 $remoteDirectory = nil
114 $architecture = nil
115 $hostOS = nil
116 $filter = nil
117 $envVars = []
118 $quickMode = false
119
120
121 def usage
122     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
123     puts
124     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
125     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
126     puts "                            --jsc specifies an already present JavaScriptCore to test."
127     puts "--ftl-jit                   Indicate that we have the FTL JIT."
128     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
129     puts "                            Skip tests tagged with //@large-heap"
130     puts "--no-jit                    Do not run JIT specific tests."
131     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
132     puts "--verbose            (-v)   Print more things while running."
133     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
134     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
135     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
136     puts "                            e.g. x86, x86_64, arm."
137     puts "--os                        Specify os instead of determining from JavaScriptCore build."
138     puts "                            e.g. darwin, linux & windows."
139     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
140     puts "                            In general the shell runner is slower than the make runner."
141     puts "--make-runner               Uses the faster make-based runner."
142     puts "--remote                    Specify a remote host on which to run tests from command line argument."
143     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
144     puts "--child-processes    (-c)   Specify the number of child processes."
145     puts "--filter                    Only run tests whose name matches the given regular expression."
146     puts "--help               (-h)   Print this message."
147     puts "--env-vars                  Add a list of environment variables to set before running jsc."
148     puts "                            Each environment variable should be separated by a space."
149     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
150     puts "                            it will override the default value."
151     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
152     exit 1
153 end
154
155 jscArg = nil
156 ifJSCArgIsntProvidedAreWeReleaseBuild = true
157
158 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
159                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
160                ['--no-copy', GetoptLong::NO_ARGUMENT],
161                ['--ftl-jit', GetoptLong::NO_ARGUMENT],
162                ['--memory-limited', GetoptLong::NO_ARGUMENT],
163                ['--no-jit', GetoptLong::NO_ARGUMENT],
164                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
165                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
166                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
167                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
168                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
169                ['--os', GetoptLong::REQUIRED_ARGUMENT],
170                ['--shell-runner', GetoptLong::NO_ARGUMENT],
171                ['--make-runner', GetoptLong::NO_ARGUMENT],
172                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
173                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
174                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
175                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
176                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
177                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
178                ['--debug', GetoptLong::NO_ARGUMENT],
179                ['--release', GetoptLong::NO_ARGUMENT],
180                ['--quick', '-q', GetoptLong::NO_ARGUMENT]).each {
181     | opt, arg |
182     case opt
183     when '--help'
184         usage
185     when '--jsc'
186         jscArg = arg
187     when '--no-copy'
188         $doNotMessWithVMPath = true
189     when '--output-dir'
190         $outputDir = Pathname.new(arg)
191     when '--ftl-jit'
192         $enableFTL = true
193     when '--memory-limited'
194         $memoryLimited = true
195     when '--no-jit'
196         $jitTests = false
197     when '--verbose'
198         $verbosity += 1
199     when '--run-bundle'
200         $bundle = Pathname.new(arg)
201     when '--tarball'
202         $tarball = true
203         $copyVM = true
204         $tarFileName = arg unless arg == ''
205     when '--force-vm-copy'
206         $copyVM = true
207     when '--shell-runner'
208         $testRunnerType = :shell
209     when '--make-runner'
210         $testRunnerType = :make
211     when '--remote'
212         $copyVM = true
213         $tarball = true
214         $remote = true
215         uri = URI("ssh://" + arg)
216         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
217     when '--remote-config-file'
218         $remoteConfigFile = arg
219     when '--child-processes'
220         $numChildProcesses = arg.to_i
221     when '--filter'
222         $filter = Regexp.new(arg)
223     when '--arch'
224         $architecture = arg
225     when '--os'
226         $hostOS = arg
227     when '--env-vars'
228         $envVars = arg.gsub(/\s+/, ' ').split(' ')
229     when '--quick'
230         $quickMode = true
231     when '--debug'
232         ifJSCArgIsntProvidedAreWeReleaseBuild = false
233     when '--release'
234         ifJSCArgIsntProvidedAreWeReleaseBuild = true
235     end
236 }
237
238 if $remoteConfigFile
239     file = File.read($remoteConfigFile)
240     config = JSON.parse(file)
241
242     if !$remote and config['remote']
243         $copyVM = true
244         $tarball = true
245         $remote = true
246         uri = URI("ssh://" + config['remote'])
247         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
248     end
249
250     if config['remoteDirectory']
251         $remoteDirectory = config['remoteDirectory']
252     end
253 end
254
255 unless jscArg
256     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
257     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
258     command += ifJSCArgIsntProvidedAreWeReleaseBuild ? " --release" : " --debug"
259     command += " --executablePath"
260
261     output = `#{command}`.split("\n")
262     if !output.length
263         $stderr.puts "Error: must specify --jsc <path>"
264         exit 1
265     end
266
267     output = output[0]
268     jscArg = Pathname.new(output).join("jsc")
269     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
270     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
271     if !File.file?(jscArg)
272         $stderr.puts "Error: must specify --jsc <path>"
273         exit 1
274     end
275
276     puts "Using the following jsc path: #{jscArg}"
277 end
278
279 if $enableFTL and !$jitTests
280     $stderr.puts "Error: can only specify one of --no-jit and --ftl-jit"
281     exit 1
282 end
283
284 if $doNotMessWithVMPath
285     $jscPath = Pathname.new(jscArg)
286 else
287     $jscPath = Pathname.new(jscArg).realpath
288 end
289
290 $progressMeter = ($verbosity == 0 and $stdout.tty?)
291
292 if $bundle
293     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
294     $outputDir = $bundle
295 end
296
297 # Try to determine architecture. Return nil on failure.
298 def machOArchitectureCode
299     begin 
300         otoolLines = `otool -aSfh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
301         otoolLines.each_with_index {
302             | value, index |
303             if value =~ /magic/ and value =~ /cputype/
304                 return otoolLines[index + 1].split[1].to_i
305             end
306         }
307     rescue
308         $stderr.puts "Warning: unable to execute otool."
309     end
310     $stderr.puts "Warning: unable to determine architecture."
311     nil
312 end
313
314 def determineArchitectureFromMachOBinary
315     code = machOArchitectureCode
316     return nil unless code
317     is64BitFlag = 0x01000000
318     case code
319     when 7
320         "x86"
321     when 7 | is64BitFlag
322         "x86-64"
323     when 12
324         "arm"
325     when 12 | is64BitFlag
326         "arm64"
327     else
328         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
329         nil
330     end
331 end
332
333 def determineArchitectureFromELFBinary
334     f = File.open($jscPath.to_s)
335     data = f.read(19)
336
337     if !(data[0,4] == "\x7F\x45\x4C\x46")
338         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
339         return nil
340     end
341
342     code = data[18].ord
343     case code
344     when 3
345         "x86"
346     when 62
347         "x86-64"
348     when 40
349         "arm"
350     when 183
351         "arm64"
352     else
353         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
354         nil
355     end
356 end
357
358 def determineArchitectureFromPEBinary
359     f = File.open($jscPath.to_s)
360     data = f.read(1024)
361
362     if !(data[0, 2] == "MZ")
363         $stderr.puts "Warning: Missing PE magic in file #{Shellwords.shellescape($jscPath.to_s)}"
364         return nil
365     end
366
367     peHeaderAddr = data[0x3c, 4].unpack('V').first # 32-bit unsigned int little endian
368
369     if !(data[peHeaderAddr, 4] == "PE\0\0")
370         $stderr.puts "Warning: Incorrect PE header in file #{Shellwords.shellescape($jscPath.to_s)}"
371         return nil
372     end
373
374     machine = data[peHeaderAddr + 4, 2].unpack('v').first # 16-bit unsigned short, little endian
375
376     case machine
377     when 0x014c
378         "x86"
379     when 0x8664
380         "x86-64"
381     else
382         $stderr.puts "Warning: unsupported machine type: #{machine}"
383         nil
384     end
385 end
386
387 def determineArchitecture
388     case $hostOS
389     when "darwin"
390         determineArchitectureFromMachOBinary
391     when "linux"
392         determineArchitectureFromELFBinary
393     when "windows"
394         determineArchitectureFromPEBinary
395     else
396         $stderr.puts "Warning: unable to determine architecture on this platform."
397         nil
398     end
399 end
400
401 def determineOS
402     case RbConfig::CONFIG["host_os"]
403     when /darwin/i
404         "darwin"
405     when /linux/i
406         "linux"
407     when /mswin|mingw|cygwin/
408         "windows"
409     else
410         $stderr.puts "Warning: unable to determine host operating system"
411         nil
412     end
413 end
414
415 $hostOS = determineOS unless $hostOS
416 $architecture = determineArchitecture unless $architecture
417
418 if !$testRunnerType
419     if $remote and $hostOS == "darwin"
420         $testRunnerType = :shell
421     else
422         $testRunnerType = :make
423     end
424 end
425
426 $numFailures = 0
427 $numPasses = 0
428
429 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true"]
430 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000"]
431 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100", "--thresholdForJITSoon=100", "--thresholdForOptimizeAfterWarmUp=1000", "--thresholdForOptimizeAfterLongWarmUp=1000", "--thresholdForOptimizeSoon=1000", "--executionCounterIncrementForLoop=1", "--executionCounterIncrementForEntry=15", "--thresholdForFTLOptimizeAfterWarmUp=100000", "--thresholdForFTLOptimizeSoon=1000", "--ftlTierUpCounterIncrementForLoop=1", "--ftlTierUpCounterIncrementForReturn=15", "--evalThresholdMultiplier=10", "--optimizationThresholdScalingFactorA=0.061504", "--optimizationThresholdScalingFactorB=1.02406", "--optimizationThresholdScalingFactorC=0.0", "--optimizationThresholdScalingFactorD=0.825914"]
432 FTL_OPTIONS = ["--useFTLJIT=true"]
433
434 $runlist = []
435
436 def frameworkFromJSCPath(jscPath)
437     parentDirectory = jscPath.dirname
438     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
439         parentDirectory.dirname
440     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
441         jscPath.dirname + "JavaScriptCore.framework"
442     else
443         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
444         nil
445     end
446 end
447
448 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
449     dir = Pathname.new(".")
450     $benchmarkDirectory.each_filename {
451         | pathComponent |
452         dir += ".."
453     }
454     dir + resourcePath
455 end
456
457 def pathToVM
458     pathToBundleResourceFromBenchmarkDirectory($jscPath)
459 end
460
461 def pathToHelpers
462     pathToBundleResourceFromBenchmarkDirectory(".helpers")
463 end
464
465 def prefixCommand(prefix)
466     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
467 end
468
469 def redirectAndPrefixCommand(prefix)
470     prefixCommand(prefix) + " 2>&1"
471 end
472
473 def pipeAndPrefixCommand(outputFilename, prefix)
474     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
475 end
476
477 # Output handler for tests that are expected to be silent.
478 def silentOutputHandler
479     Proc.new {
480         | name |
481         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
482     }
483 end
484
485 # Output handler for tests that are expected to produce meaningful output.
486 def noisyOutputHandler
487     Proc.new {
488         | name |
489         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
490     }
491 end
492
493 # Error handler for tests that fail exactly when they return non-zero exit status.
494 # This is useful when a test is expected to fail.
495 def simpleErrorHandler
496     Proc.new {
497         | outp, plan |
498         outp.puts "if test -e #{plan.failFile}"
499         outp.puts "then"
500         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
501         outp.puts "    " + plan.failCommand
502         outp.puts "else"
503         outp.puts "    " + plan.successCommand
504         outp.puts "fi"
505     }
506 end
507
508 # Error handler for tests that fail exactly when they return zero exit status.
509 def expectedFailErrorHandler
510     Proc.new {
511         | outp, plan |
512         outp.puts "if test -e #{plan.failFile}"
513         outp.puts "then"
514         outp.puts "    " + plan.successCommand
515         outp.puts "else"
516         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
517         outp.puts "    " + plan.failCommand
518         outp.puts "fi"
519     }
520 end
521
522 # Error handler for tests that fail exactly when they return non-zero exit status and produce
523 # lots of spew. This will echo that spew when the test fails.
524 def noisyErrorHandler
525     Proc.new {
526         | outp, plan |
527         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
528     
529         outp.puts "if test -e #{plan.failFile}"
530         outp.puts "then"
531         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
532         outp.puts "    " + plan.failCommand
533         outp.puts "else"
534         outp.puts "    " + plan.successCommand
535         outp.puts "fi"
536     }
537 end
538
539 # Error handler for tests that diff their output with some expectation.
540 def diffErrorHandler(expectedFilename)
541     Proc.new {
542         | outp, plan |
543         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
544         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
545         
546         outp.puts "if test -e #{plan.failFile}"
547         outp.puts "then"
548         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
549         outp.puts "    " + plan.failCommand
550         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
551         outp.puts "then"
552         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
553         outp.puts "    if [ $? -eq 0 ]"
554         outp.puts "    then"
555         outp.puts "    " + plan.successCommand
556         outp.puts "    else"
557         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
558         outp.puts "        " + plan.failCommand
559         outp.puts "    fi"
560         outp.puts "else"
561         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
562         outp.puts "    " + plan.failCommand
563         outp.puts "fi"
564     }
565 end
566
567 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
568 # tests.
569 def mozillaErrorHandler
570     Proc.new {
571         | outp, plan |
572         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
573
574         outp.puts "if test -e #{plan.failFile}"
575         outp.puts "then"
576         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
577         outp.puts "    " + plan.failCommand
578         outp.puts "elif grep -i -q failed! #{outputFilename}"
579         outp.puts "then"
580         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
581         outp.puts "    " + plan.failCommand
582         outp.puts "else"
583         outp.puts "    " + plan.successCommand
584         outp.puts "fi"
585     }
586 end
587
588 # Error handler for tests that report error by saying "failed!", and are expected to
589 # fail. This is used by Mozilla tests.
590 def mozillaFailErrorHandler
591     Proc.new {
592         | outp, plan |
593         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
594
595         outp.puts "if test -e #{plan.failFile}"
596         outp.puts "then"
597         outp.puts "    " + plan.successCommand
598         outp.puts "elif grep -i -q failed! #{outputFilename}"
599         outp.puts "then"
600         outp.puts "    " + plan.successCommand
601         outp.puts "else"
602         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
603         outp.puts "    " + plan.failCommand
604         outp.puts "fi"
605     }
606 end
607
608 # Error handler for tests that report error by saying "failed!", and are expected to have
609 # an exit code of 3.
610 def mozillaExit3ErrorHandler
611     Proc.new {
612         | outp, plan |
613         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
614
615         outp.puts "if test -e #{plan.failFile}"
616         outp.puts "then"
617         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
618         outp.puts "    then"
619         outp.puts "        if grep -i -q failed! #{outputFilename}"
620         outp.puts "        then"
621         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
622         outp.puts "            " + plan.failCommand
623         outp.puts "        else"
624         outp.puts "            " + plan.successCommand
625         outp.puts "        fi"
626         outp.puts "    else"
627         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
628         outp.puts "        " + plan.failCommand
629         outp.puts "    fi"
630         outp.puts "else"
631         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
632         outp.puts "    " + plan.failCommand
633         outp.puts "fi"
634     }
635 end
636
637 $runCommandOptions = {}
638
639 class Plan
640     attr_reader :directory, :arguments, :family, :name, :outputHandler, :errorHandler
641     attr_accessor :index
642     
643     def initialize(directory, arguments, family, name, outputHandler, errorHandler)
644         @directory = directory
645         @arguments = arguments
646         @family = family
647         @name = name
648         @outputHandler = outputHandler
649         @errorHandler = errorHandler
650         @isSlow = !!$runCommandOptions[:isSlow]
651     end
652     
653     def shellCommand
654         # It's important to remember that the test is actually run in a subshell, so if we change directory
655         # in the subshell when we return we will be in our original directory. This is nice because we don't
656         # have to bend over backwards to do things relative to the root.
657         script = "(cd ../#{Shellwords.shellescape(@directory.to_s)} && ("
658         $envVars.each { |var| script += "export " << var << "; " }
659         script += "\"$@\" " + escapeAll(@arguments) + "))"
660         return script
661     end
662     
663     def reproScriptCommand
664         # We have to find our way back to the .runner directory since that's where all of the relative
665         # paths assume they start out from.
666         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
667         script += "cd $CURRENT_DIR\n"
668         Pathname.new(@name).dirname.each_filename {
669             | pathComponent |
670             script += "cd ..\n"
671         }
672         script += "cd .runner\n"
673
674         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
675         script += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])}\n"
676         $envVars.each { |var| script += "export " << var << "\n" }
677         script += "#{shellCommand} || exit 1"
678         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
679     end
680     
681     def failCommand
682         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
683     end
684     
685     def successCommand
686         if $progressMeter or $verbosity >= 2
687             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
688         else
689             "rm -f #{failFile}"
690         end
691     end
692     
693     def failFile
694         "test_fail_#{@index}"
695     end
696     
697     def writeRunScript(filename)
698         File.open(filename, "w") {
699             | outp |
700             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
701             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
702             cmd += @outputHandler.call(@name)
703             if $verbosity >= 3
704                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
705             end
706             outp.puts cmd
707             @errorHandler.call(outp, self)
708         }
709     end
710 end
711
712 $uniqueFilenameCounter = 0
713 def uniqueFilename(extension)
714     payloadDir = $outputDir + "_payload"
715     Dir.mkdir payloadDir unless payloadDir.directory?
716     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
717     $uniqueFilenameCounter += 1
718     result
719 end
720
721 def baseOutputName(kind)
722     "#{$collectionName}/#{$benchmark}.#{kind}"
723 end
724
725 def addRunCommand(kind, command, outputHandler, errorHandler)
726     $didAddRunCommand = true
727     name = baseOutputName(kind)
728     if $filter and name !~ $filter
729         return
730     end
731     plan = Plan.new(
732         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
733         errorHandler)
734     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
735         $runlist.unshift plan
736     else
737         $runlist << plan
738     end
739 end
740
741 # Returns true if there were run commands found in the file ($benchmarkDirectory +
742 # $benchmark), in which case those run commands have already been executed. Otherwise
743 # returns false, in which case you're supposed to add your own run commands.
744 def parseRunCommands
745     oldDidAddRunCommand = $didAddRunCommand
746     $didAddRunCommand = false
747
748     Dir.chdir($outputDir) {
749         File.open($benchmarkDirectory + $benchmark) {
750             | inp |
751             inp.each_line {
752                 | line |
753                 begin
754                     doesMatch = line =~ /^\/\/@/
755                 rescue Exception => e
756                     # Apparently this happens in the case of some UTF8 stuff in some files, where
757                     # Ruby tries to be strict and throw exceptions.
758                     next
759                 end
760                 next unless doesMatch
761                 eval $~.post_match
762             }
763         }
764     }
765
766     result = $didAddRunCommand
767     $didAddRunCommand = result or oldDidAddRunCommand
768     result
769 end
770
771 def slow!
772     $runCommandOptions[:isSlow] = true
773 end
774
775 def run(kind, *options)
776     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
777 end
778
779 def runDefault
780     run("default")
781 end
782
783 def runWithRAMSize(size)
784     run("ram-size-#{size}", "--forceRAMSize=#{size}")
785 end
786
787 def runOneLargeHeap
788     if $memoryLimited
789         $didAddRunCommand = true
790         puts "Skipping #{$collectionName}/#{$benchmark}"
791     else
792         run("default")
793     end
794 end
795
796 def runNoJIT
797     run("no-jit", "--useJIT=false")
798 end
799
800 def runNoLLInt
801     if $jitTests
802         run("no-llint", "--useLLInt=false")
803     end
804 end
805
806 def runNoCJITValidate
807     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *NO_CJIT_OPTIONS)
808 end
809
810 def runNoCJITValidatePhases
811     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
812 end
813
814 def runDefaultFTL
815     run("default-ftl", *FTL_OPTIONS) if $enableFTL
816 end
817
818 def runFTLNoCJIT
819     run("ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
820 end
821
822 def runFTLNoCJITValidate
823     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
824 end
825
826 def runFTLNoCJITNoPutStackValidate
827     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
828 end
829
830 def runFTLNoCJITNoInlineValidate
831     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
832 end
833
834 def runFTLNoCJITOSRValidation
835     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
836 end
837
838 def runDFGEager
839     run("dfg-eager", *EAGER_OPTIONS)
840 end
841
842 def runDFGEagerNoCJITValidate
843     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
844 end
845
846 def runFTLEager
847     run("ftl-eager", *(FTL_OPTIONS + EAGER_OPTIONS)) if $enableFTL
848 end
849
850 def runFTLEagerNoCJITValidate
851     run("ftl-eager-no-cjit", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
852 end
853
854 def runFTLEagerNoCJITOSRValidation
855     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
856 end
857
858 def runAlwaysTriggerCopyPhase
859     run("always-trigger-copy-phase", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
860 end
861
862 def runNoCJITNoASO
863     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *NO_CJIT_OPTIONS)
864 end
865
866 def runNoCJITNoAccessInlining
867     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *NO_CJIT_OPTIONS)
868 end
869
870 def runFTLNoCJITNoAccessInlining
871     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
872 end
873
874 def runFTLNoCJITSmallPool
875     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
876 end
877
878 def runMiscNoCJITTest(*options)
879     run("misc-no-cjit", *(NO_CJIT_OPTIONS + options))
880 end
881
882 def runMiscFTLNoCJITTest(*options)
883     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + options))
884 end
885
886 def runDFGMaximalFlushPhase
887     run("dfg-maximal-flush-validate-no-cjit", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *NO_CJIT_OPTIONS)
888 end
889
890 def runShadowChicken
891     run("shadow-chicken", "--useDFGJIT=false", "--alwaysUseShadowChicken=true")
892 end
893
894 def defaultRun
895     if $quickMode
896         defaultQuickRun
897     else
898         runDefault
899         runAlwaysTriggerCopyPhase
900         if $jitTests
901             runNoLLInt
902             runNoCJITValidatePhases
903             runDFGEager
904             runDFGEagerNoCJITValidate
905             runDefaultFTL
906             runFTLNoCJITValidate
907             runFTLNoCJITNoPutStackValidate
908             runFTLNoCJITNoInlineValidate
909             runFTLEager
910             runFTLEagerNoCJITValidate
911             runFTLNoCJITSmallPool
912             runDFGMaximalFlushPhase
913         end
914     end
915 end
916
917 def defaultNoNoLLIntRun
918     if $quickMode
919         defaultQuickRun
920     else
921         runDefault
922         runAlwaysTriggerCopyPhase
923         if $jitTests
924             runNoCJITValidatePhases
925             runDFGEager
926             runDFGEagerNoCJITValidate
927             runDefaultFTL
928             runFTLNoCJITValidate
929             runFTLNoCJITNoPutStackValidate
930             runFTLNoCJITNoInlineValidate
931             runFTLEager
932             runFTLEagerNoCJITValidate
933             runFTLNoCJITSmallPool
934             runDFGMaximalFlushPhase
935         end
936     end
937 end
938
939 def defaultQuickRun
940     if $enableFTL and $jitTests
941         runDefaultFTL
942         runFTLNoCJITValidate
943     else
944         runDefault
945         if $jitTests
946             runNoCJITValidate
947         end
948     end
949 end
950
951 def defaultSpotCheckNoMaximalFlush
952     defaultQuickRun
953     runFTLNoCJITOSRValidation
954     runNoCJITNoAccessInlining
955     runFTLNoCJITNoAccessInlining
956 end
957
958 def defaultSpotCheck
959     defaultSpotCheckNoMaximalFlush
960     runDFGMaximalFlushPhase
961 end
962
963 # This is expected to not do eager runs because eager runs can have a lot of recompilations
964 # for reasons that don't arise in the real world. It's used for tests that assert convergence
965 # by counting recompilations.
966 def defaultNoEagerRun
967     runDefault
968     runAlwaysTriggerCopyPhase
969     if $jitTests
970         runNoLLInt
971         runNoCJITValidatePhases
972         runDefaultFTL
973         runFTLNoCJITValidate
974         runFTLNoCJITNoInlineValidate
975     end
976 end
977
978 def defaultNoSamplingProfilerRun
979     runDefault
980     runAlwaysTriggerCopyPhase
981     if $jitTests
982         runNoLLInt
983         runNoCJITValidatePhases
984         runDFGEager
985         runDFGEagerNoCJITValidate
986         runDefaultFTL
987         runFTLNoCJITNoPutStackValidate
988         runFTLNoCJITNoInlineValidate
989         runFTLEager
990         runFTLEagerNoCJITValidate
991         runFTLNoCJITSmallPool
992         runDFGMaximalFlushPhase
993     end
994 end
995
996 def runProfiler
997     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
998         skip
999         return
1000     end
1001
1002     profilerOutput = uniqueFilename(".json")
1003     if $canRunDisplayProfilerOutput
1004         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
1005     else
1006         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
1007         run("profiler-simple", "-p", profilerOutput.to_s)
1008     end
1009 end
1010
1011 def runExceptionFuzz
1012     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
1013     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
1014 end
1015
1016 def runExecutableAllocationFuzz(name, *options)
1017     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
1018     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
1019 end
1020
1021 def runTypeProfiler
1022     if !$jitTests
1023         return
1024     end
1025
1026     if $enableFTL
1027         run("ftl-no-cjit-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1028     else
1029         run("no-cjit-type-profiler", "--useTypeProfiler=true", *NO_CJIT_OPTIONS)
1030     end
1031 end
1032
1033 def runControlFlowProfiler
1034     if !$jitTests
1035         return
1036     end
1037
1038     if $enableFTL
1039         run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1040     else
1041         run("no-cjit-type-profiler", "--useControlFlowProfiler=true", *NO_CJIT_OPTIONS)
1042     end
1043 end
1044
1045 def runTest262(mode, exception, includeFiles, flags)
1046     failsWithException = exception != "NoException"
1047     isStrict = false
1048     isModule = false
1049     isAsync = false
1050
1051     flags.each {
1052         | flag |
1053         case flag
1054         when :strict
1055             isStrict = true
1056         when :module
1057             isModule = true
1058         when :async
1059             isAsync = true
1060         else
1061             raise "Invalid flag for runTest262, #{flag}"
1062         end
1063     }
1064
1065     prepareExtraRelativeFiles(includeFiles.map { |f| "../" + f }, $collection)
1066
1067     args = [pathToVM.to_s] + BASE_OPTIONS
1068     args << "-m" if isModule
1069     args << "--exception=" + exception if failsWithException
1070     args << "--test262-async" if isAsync
1071     args += includeFiles
1072
1073     case mode
1074     when :normal
1075         errorHandler = simpleErrorHandler
1076         outputHandler = silentOutputHandler
1077     when :fail
1078         errorHandler = expectedFailErrorHandler
1079         outputHandler = noisyOutputHandler
1080     else
1081         raise "Invalid mode: #{mode}"
1082     end
1083
1084     if isStrict
1085         kind = "default-strict"
1086         args << "--strict-file=#{$benchmark}"
1087     else
1088         kind = "default"
1089         args << $benchmark.to_s
1090     end
1091
1092     addRunCommand(kind, args, outputHandler, errorHandler)
1093 end
1094
1095 def prepareTest262Fixture
1096     # This function is used to add the files used by Test262 modules tests.
1097     prepareExtraRelativeFiles([""], $collection)
1098 end
1099
1100 def runES6(mode)
1101     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
1102     case mode
1103     when :normal
1104         errorHandler = simpleErrorHandler
1105     when :fail
1106         errorHandler = expectedFailErrorHandler
1107     else
1108         raise "Invalid mode: #{mode}"
1109     end
1110     addRunCommand("default", args, noisyOutputHandler, errorHandler)
1111 end
1112
1113 def runModules
1114     run("default-modules", "-m")
1115     run("always-trigger-copy-phase-modules", "-m", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
1116
1117     if !$jitTests
1118         return
1119     end
1120
1121     run("no-llint-modules", "-m", "--useLLInt=false")
1122     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
1123     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
1124     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1125     if $enableFTL
1126         run("default-ftl-modules", "-m", *FTL_OPTIONS)
1127         run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1128         run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1129         run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
1130         run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1131         run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1132     end
1133 end
1134
1135 def runLayoutTest(kind, *options)
1136     raise unless $benchmark.to_s =~ /\.js$/
1137     testName = $~.pre_match
1138     if kind
1139         kind = "layout-" + kind
1140     else
1141         kind = "layout"
1142     end
1143
1144     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1145     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1146
1147     args = [pathToVM.to_s] + BASE_OPTIONS + options +
1148         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1149          $benchmark.to_s,
1150          (Pathname.new("resources") + "standalone-post.js").to_s]
1151     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1152 end
1153
1154 def runLayoutTestDefault
1155     runLayoutTest(nil)
1156 end
1157
1158 def runLayoutTestNoLLInt
1159     runLayoutTest("no-llint", "--useLLInt=false")
1160 end
1161
1162 def runLayoutTestNoCJIT
1163     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1164 end
1165
1166 def runLayoutTestDFGEagerNoCJIT
1167     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1168 end
1169
1170 def runLayoutTestDefaultFTL
1171     runLayoutTest("ftl", "--testTheFTL=true", *FTL_OPTIONS) if $enableFTL
1172 end
1173
1174 def runLayoutTestFTLNoCJIT
1175     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
1176 end
1177
1178 def runLayoutTestFTLEagerNoCJIT
1179     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
1180 end
1181
1182 def noFTLRunLayoutTest
1183     if !$jitTests
1184         return
1185     end
1186
1187     runLayoutTestNoLLInt
1188     runLayoutTestNoCJIT
1189     runLayoutTestDFGEagerNoCJIT
1190 end
1191
1192 def defaultQuickRunLayoutTest
1193     runLayoutTestDefault
1194     if $jitTests
1195         runLayoutTestFTLNoCJIT
1196         runLayoutTestFTLEagerNoCJIT
1197     end
1198 end
1199
1200 def defaultRunLayoutTest
1201     if $quickMode
1202         defaultQuickRunLayoutTest
1203     else
1204         runLayoutTestDefault
1205         if $jitTests
1206             noFTLRunLayoutTest
1207             runLayoutTestDefaultFTL
1208             runLayoutTestFTLNoCJIT
1209             runLayoutTestFTLEagerNoCJIT
1210         end
1211     end
1212 end
1213
1214 def noEagerNoNoLLIntTestsRunLayoutTest
1215     runLayoutTestDefault
1216     if $jitTests
1217         runLayoutTestNoCJIT
1218         runLayoutTestDefaultFTL
1219         runLayoutTestFTLNoCJIT
1220     end
1221 end
1222
1223 def noNoLLIntRunLayoutTest
1224     runLayoutTestDefault
1225     if $jitTests
1226         runLayoutTestNoCJIT
1227         runLayoutTestDFGEagerNoCJIT
1228         runLayoutTestDefaultFTL
1229         runLayoutTestFTLNoCJIT
1230         runLayoutTestFTLEagerNoCJIT
1231     end
1232 end
1233
1234 def prepareExtraRelativeFiles(extraFiles, destination)
1235     Dir.chdir($outputDir) {
1236         extraFiles.each {
1237             | file |
1238             dest = destination + file
1239             FileUtils.mkdir_p(dest.dirname)
1240             FileUtils.cp $extraFilesBaseDir + file, dest
1241         }
1242     }
1243 end
1244
1245 def baseDirForCollection(collectionName)
1246     Pathname(".tests") + collectionName
1247 end
1248
1249 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1250     raise unless absoluteBase.absolute?
1251     Dir.chdir($outputDir) {
1252         collectionBaseDir = baseDirForCollection($collectionName)
1253         extraFiles.each {
1254             | file |
1255             destination = collectionBaseDir + file
1256             FileUtils.mkdir_p destination.dirname unless destination.directory?
1257             FileUtils.cp absoluteBase + file, destination
1258         }
1259     }
1260 end
1261
1262 def runMozillaTest(kind, mode, extraFiles, *options)
1263     if kind
1264         kind = "mozilla-" + kind
1265     else
1266         kind = "mozilla"
1267     end
1268     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1269     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1270     case mode
1271     when :normal
1272         errorHandler = mozillaErrorHandler
1273     when :negative
1274         errorHandler = mozillaExit3ErrorHandler
1275     when :fail
1276         errorHandler = mozillaFailErrorHandler
1277     when :skip
1278         return
1279     else
1280         raise "Invalid mode: #{mode}"
1281     end
1282     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1283 end
1284
1285 def runMozillaTestDefault(mode, *extraFiles)
1286     runMozillaTest(nil, mode, extraFiles)
1287 end
1288
1289 def runMozillaTestDefaultFTL(mode, *extraFiles)
1290     runMozillaTest("ftl", mode, extraFiles, *FTL_OPTIONS) if $enableFTL
1291 end
1292
1293 def runMozillaTestLLInt(mode, *extraFiles)
1294     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1295 end
1296
1297 def runMozillaTestBaselineJIT(mode, *extraFiles)
1298     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1299 end
1300
1301 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1302     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1303 end
1304
1305 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1306     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
1307 end
1308
1309 def defaultQuickRunMozillaTest(mode, *extraFiles)
1310     if $enableFTL and $jitTests
1311         runMozillaTestDefaultFTL(mode, *extraFiles)
1312         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1313     else
1314         runMozillaTestDefault(mode, *extraFiles)
1315         if $jitTests
1316             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1317         end
1318     end
1319 end
1320
1321 def defaultRunMozillaTest(mode, *extraFiles)
1322     if $quickMode
1323         defaultQuickRunMozillaTest(mode, *extraFiles)
1324     else
1325         runMozillaTestDefault(mode, *extraFiles)
1326         if $jitTests
1327             runMozillaTestLLInt(mode, *extraFiles)
1328             runMozillaTestBaselineJIT(mode, *extraFiles)
1329             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1330             runMozillaTestDefaultFTL(mode, *extraFiles)
1331             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1332         end
1333     end
1334 end
1335
1336 def runNoisyTest(kind, *options)
1337     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
1338 end
1339
1340 def runNoisyTestDefault
1341     runNoisyTest("default")
1342 end
1343
1344 def runNoisyTestDefaultFTL
1345     runNoisyTest("ftl", *FTL_OPTIONS) if $enableFTL
1346 end
1347
1348 def runNoisyTestNoCJIT
1349     runNoisyTest($enableFTL ? "ftl-no-cjit" : "no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(($enableFTL ? FTL_OPTIONS : []) + NO_CJIT_OPTIONS))
1350 end
1351
1352 def runNoisyTestEagerNoCJIT
1353     runNoisyTest($enableFTL ? "ftl-eager-no-cjit" : "eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(($enableFTL ? FTL_OPTIONS : []) + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1354 end
1355
1356 def defaultRunNoisyTest
1357     runNoisyTestDefault
1358     if $jitTests
1359         runNoisyTestDefaultFTL
1360         runNoisyTestNoCJIT
1361         runNoisyTestEagerNoCJIT
1362     end
1363 end
1364
1365 def skip
1366     $didAddRunCommand = true
1367     puts "Skipping #{$collectionName}/#{$benchmark}"
1368 end
1369
1370 def largeHeap
1371     if $memoryLimited
1372         $didAddRunCommand = true
1373         puts "Skipping #{$collectionName}/#{$benchmark}"
1374     end
1375 end
1376
1377 def allJSFiles(path)
1378     if path.file?
1379         [path]
1380     else
1381         result = []
1382         Dir.foreach(path) {
1383             | filename |
1384             next unless filename =~ /\.js$/
1385             next unless (path + filename).file?
1386             result << path + filename
1387         }
1388         result
1389     end
1390 end
1391
1392 def uniqueifyName(names, name)
1393     result = name.to_s
1394     toAdd = 1
1395     while names[result]
1396         result = "#{name}-#{toAdd}"
1397         toAdd += 1
1398     end
1399     names[result] = true
1400     result
1401 end
1402
1403 def simplifyCollectionName(collectionPath)
1404     outerDir = collectionPath.dirname
1405     name = collectionPath.basename
1406     lastName = name
1407     if collectionPath.directory?
1408         while lastName.to_s =~ /test/
1409             lastName = outerDir.basename
1410             name = lastName + name
1411             outerDir = outerDir.dirname
1412         end
1413     end
1414     uniqueifyName($collectionNames, name)
1415 end
1416
1417 def prepareCollection(name)
1418     FileUtils.mkdir_p $outputDir + name
1419
1420     absoluteCollection = $collection.realpath
1421
1422     Dir.chdir($outputDir) {
1423         bundleDir = baseDirForCollection(name)
1424
1425         # Create the proper directory structures.
1426         FileUtils.mkdir_p bundleDir
1427         if bundleDir.basename == $collection.basename
1428             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1429             $collection = bundleDir
1430         else
1431             FileUtils.cp_r absoluteCollection, bundleDir
1432             $collection = bundleDir + $collection.basename
1433         end
1434
1435         $extraFilesBaseDir = absoluteCollection
1436     }
1437 end
1438
1439 $collectionNames = {}
1440
1441 def handleCollectionFile(collection)
1442     collectionName = simplifyCollectionName(collection)
1443    
1444     paths = {}
1445     subCollections = []
1446     YAML::load(IO::read(collection)).each {
1447         | entry |
1448         if entry["collection"]
1449             subCollections << entry["collection"]
1450             next
1451         end
1452         
1453         if Pathname.new(entry["path"]).absolute?
1454             raise "Absolute path: " + entry["path"] + " in #{collection}"
1455         end
1456         
1457         if paths[entry["path"]]
1458             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1459         end
1460         
1461         subCollection = collection.dirname + entry["path"]
1462         
1463         if subCollection.file?
1464             subCollectionName = Pathname.new(entry["path"]).dirname
1465         else
1466             subCollectionName = entry["path"]
1467         end
1468         
1469         $collection = subCollection
1470         $collectionName = Pathname.new(collectionName)
1471         Pathname.new(subCollectionName).each_filename {
1472             | filename |
1473             next if filename =~ /^\./
1474             $collectionName += filename
1475         }
1476         $collectionName = $collectionName.to_s
1477         
1478         prepareCollection($collectionName)
1479       
1480         Dir.chdir($outputDir) {
1481             pathsToSearch = [$collection]
1482             if entry["tests"]
1483                 if entry["tests"].is_a? Array
1484                     pathsToSearch = entry["tests"].map {
1485                         | testName |
1486                         pathsToSearch[0] + testName
1487                     }
1488                 else
1489                     pathsToSearch[0] += entry["tests"]
1490                 end
1491             end
1492             pathsToSearch.each {
1493                 | pathToSearch |
1494                 allJSFiles(pathToSearch).each {
1495                     | path |
1496                     
1497                     $benchmark = path.basename
1498                     $benchmarkDirectory = path.dirname
1499                     
1500                     $runCommandOptions = {}
1501                     eval entry["cmd"]
1502                 }
1503             }
1504         }
1505     }
1506     
1507     subCollections.each {
1508         | subCollection |
1509         handleCollection(collection.dirname + subCollection)
1510     }
1511 end
1512
1513 def handleCollectionDirectory(collection)
1514     collectionName = simplifyCollectionName(collection)
1515     
1516     $collection = collection
1517     $collectionName = collectionName
1518     prepareCollection(collectionName)
1519    
1520     Dir.chdir($outputDir) {
1521         $benchmarkDirectory = $collection
1522         allJSFiles($collection).each {
1523             | path |
1524             
1525             $benchmark = path.basename
1526             
1527             $runCommandOptions = {}
1528             defaultRun unless parseRunCommands
1529         }
1530     }
1531 end
1532
1533 def handleCollection(collection)
1534     collection = Pathname.new(collection)
1535     
1536     if collection.file?
1537         handleCollectionFile(collection)
1538     else
1539         handleCollectionDirectory(collection)
1540     end
1541 end
1542
1543 def appendFailure(plan)
1544     File.open($outputDir + "failed", "a") {
1545         | outp |
1546         outp.puts plan.name
1547     }
1548     $numFailures += 1
1549 end
1550
1551 def appendPass(plan)
1552     File.open($outputDir + "passed", "a") {
1553         | outp |
1554         outp.puts plan.name
1555     }
1556     $numPasses += 1
1557 end
1558
1559 def appendResult(plan, didPass)
1560     File.open($outputDir + "results", "a") {
1561         | outp |
1562         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1563     }
1564 end
1565
1566 def prepareBundle
1567     raise if $bundle
1568
1569     if $doNotMessWithVMPath
1570         if !$remote and !$tarball
1571             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1572             $jscPath = Pathname.new($jscPath).realpath
1573         else
1574             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1575         end
1576     else
1577         originalJSCPath = $jscPath
1578         vmDir = $outputDir + ".vm"
1579         FileUtils.mkdir_p vmDir
1580         
1581         frameworkPath = frameworkFromJSCPath($jscPath)
1582         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1583         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1584         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1585
1586         if frameworkPath
1587             source = frameworkPath
1588             destination = Pathname.new(".vm")
1589         else
1590             source = originalJSCPath
1591             destination = $jscPath
1592
1593             Dir.chdir($outputDir) {
1594                 FileUtils.mkdir_p $jscPath.dirname
1595             }
1596         end
1597
1598         Dir.chdir($outputDir) {
1599             if $copyVM
1600                 FileUtils.cp_r source, destination
1601             else
1602                 begin 
1603                     FileUtils.ln_s source, destination
1604                 rescue Exception
1605                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1606                     FileUtils.cp_r source, destination
1607                 end
1608             end
1609
1610             if $remote and $hostOS == "linux"
1611                 begin
1612                     dependencies = `ldd #{source}`
1613                     dependencies.split(/\n/).each {
1614                         | dependency |
1615                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1616                     }
1617                 rescue
1618                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1619                 end
1620             end
1621         }
1622     end
1623     
1624     Dir.chdir($outputDir) {
1625         FileUtils.cp_r HELPERS_PATH, ".helpers"
1626     }
1627
1628     ARGV.each {
1629         | collection |
1630         handleCollection(collection)
1631     }
1632
1633     puts
1634 end
1635
1636 def cleanOldResults
1637     raise unless $bundle
1638
1639     eachResultFile($outputDir) {
1640         | path |
1641         FileUtils.rm_f path
1642     }
1643 end
1644
1645 def cleanEmptyResultFiles
1646     eachResultFile($outputDir) {
1647         | path |
1648         next unless path.basename.to_s =~ /\.out$/
1649         next unless FileTest.size(path) == 0
1650         FileUtils.rm_f path
1651     }
1652 end
1653
1654 def eachResultFile(startingDir, &block)
1655     dirsToClean = [startingDir]
1656     until dirsToClean.empty?
1657         nextDir = dirsToClean.pop
1658         Dir.foreach(nextDir) {
1659             | entry |
1660             next if entry =~ /^\./
1661             path = nextDir + entry
1662             if path.directory?
1663                 dirsToClean.push(path)
1664             else
1665                 block.call(path)
1666             end
1667         }
1668     end
1669 end
1670
1671 def prepareTestRunner
1672     raise if $bundle
1673
1674     $runlist.each_with_index {
1675         | plan, index |
1676         plan.index = index
1677     }
1678
1679     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1680     toDelete = []
1681     Dir.foreach($runnerDir) {
1682         | filename |
1683         if filename =~ /^test_/
1684             toDelete << filename
1685         end
1686     }
1687     
1688     toDelete.each {
1689         | filename |
1690         File.unlink($runnerDir + filename)
1691     }
1692
1693     $runlist.each {
1694         | plan |
1695         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1696     }
1697
1698     case $testRunnerType
1699     when :make
1700         prepareMakeTestRunner
1701     when :shell
1702         prepareShellTestRunner
1703     else
1704         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1705     end
1706 end
1707
1708 def prepareShellTestRunner
1709     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
1710 end
1711
1712 def prepareMakeTestRunner
1713     # The goals of our parallel test runner are scalability and simplicity. The
1714     # simplicity part is particularly important. We don't want to have to have
1715     # a full-time contributor just philosophising about parallel testing.
1716     #
1717     # As such, we just pass off all of the hard work to 'make'. This creates a
1718     # dummy directory ("$outputDir/.runner") in which we create a dummy
1719     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
1720     # That is, for each test we know we will run, there is a rule in the
1721     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
1722     # Makefile results in 'make' doing all of the hard work:
1723     #
1724     # - Load balancing just works. Most systems have a great load balancer in
1725     #   'make'. If your system doesn't then just install a real 'make'.
1726     #
1727     # - Interruptions just work. For example Ctrl-C handling in 'make' is
1728     #   exactly right. You don't have to worry about zombie processes.
1729     #
1730     # We then do some tricks to make failure detection work and to make this
1731     # totally sound. If a test fails, we don't want the whole 'make' job to
1732     # stop. We also don't have any facility for makefile-escaping of path names.
1733     # We do have such a thing for shell-escaping, though. We fix both problems
1734     # by having the actual work for each of the test rules be done in a shell
1735     # script on the side. There is one such script per test. The script responds
1736     # to failure by printing something on the console and then touching a
1737     # failure file for that test, but then still returns 0. This makes 'make'
1738     # continue past that failure and complete all the tests anyway.
1739     #
1740     # In the end, this script collects all of the failures by searching for
1741     # files in the .runner directory whose name matches /^test_fail_/, where
1742     # the thing after the 'fail_' is the test index. Those are the files that
1743     # would be created by the test scripts if they detect failure. We're
1744     # basically using the filesystem as a concurrent database of test failures.
1745     # Even if two tests fail at the same time, since they're touching different
1746     # files we won't miss any failures.
1747     runIndices = []
1748     $runlist.each {
1749         | plan |
1750         runIndices << plan.index
1751     }
1752     
1753     File.open($runnerDir + "Makefile", "w") {
1754         | outp |
1755         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1756         runIndices.each {
1757             | index |
1758             plan = $runlist[index]
1759             outp.puts "test_done_#{index}:"
1760             outp.puts "\tsh test_script_#{plan.index}"
1761         }
1762     }
1763 end
1764
1765 def cleanRunnerDirectory
1766     raise unless $bundle
1767     Dir.foreach($runnerDir) {
1768         | filename |
1769         next unless filename =~ /^test_fail/
1770         FileUtils.rm_f $runnerDir + filename
1771     }
1772 end
1773
1774 def sshRead(cmd)
1775     raise unless $remote
1776
1777     result = ""
1778     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1779       | inp |
1780       inp.each_line {
1781         | line |
1782         result += line
1783       }
1784     }
1785     raise "#{$?}" unless $?.success?
1786     result
1787 end
1788
1789 def runCommandOnTester(cmd)
1790     if $remote
1791         result = sshRead(cmd)
1792     else
1793         result = `#{cmd}`
1794     end
1795 end
1796
1797 def numberOfProcessors
1798     if $hostOS == "windows"
1799         numProcessors = runCommandOnTester("cmd /c echo %NUMBER_OF_PROCESSORS%").to_i
1800     else
1801         begin
1802             numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1803         rescue
1804             numProcessors = 0
1805         end
1806
1807         if numProcessors == 0
1808             begin
1809                 numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1810             rescue
1811                 numProcessors == 0
1812             end
1813         end
1814     end
1815
1816     if numProcessors == 0
1817         numProcessors = 1
1818     end
1819     return numProcessors
1820 end
1821
1822 def runAndMonitorTestRunnerCommand(*cmd)
1823     numberOfTests = 0
1824     Dir.chdir($runnerDir) {
1825         # -1 for the runscript, and -2 for '..' and '.'
1826         numberOfTests = Dir.entries(".").count - 3
1827     }
1828     unless $progressMeter
1829         mysys(cmd.join(' '))
1830     else
1831        running = {}
1832        didRun = {}
1833        didFail = {}
1834        blankLine = true
1835        prevStringLength = 0
1836        IO.popen(cmd.join(' '), mode="r") {
1837            | inp |
1838            inp.each_line {
1839                | line |
1840                line = line.scrub.chomp
1841                if line =~ /^Running /
1842                    running[$~.post_match] = true
1843                elsif line =~ /^PASS: /
1844                    didRun[$~.post_match] = true
1845                elsif line =~ /^FAIL: /
1846                    didRun[$~.post_match] = true
1847                    didFail[$~.post_match] = true
1848                else
1849                    unless blankLine
1850                        print("\r" + " " * prevStringLength + "\r")
1851                    end
1852                    puts line
1853                    blankLine = true
1854                end
1855
1856                def lpad(str, chars)
1857                    str = str.to_s
1858                    if str.length > chars
1859                        str
1860                    else
1861                       "%#{chars}s"%(str)
1862                    end
1863                end
1864
1865                string  = ""
1866                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1867                unless didFail.empty?
1868                    string += " (failed #{didFail.size})"
1869                end
1870                string += " "
1871                (running.size - didRun.size).times {
1872                    string += "."
1873                }
1874                if string.length < prevStringLength
1875                    print string
1876                    print(" " * (prevStringLength - string.length))
1877                end
1878                print string
1879                prevStringLength = string.length
1880                blankLine = false
1881                $stdout.flush
1882            }
1883        }
1884        puts
1885        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1886     end
1887 end
1888
1889 def runTestRunner
1890     case $testRunnerType
1891     when :shell
1892         testRunnerCommand = "sh runscript"
1893     when :make
1894         testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
1895     else
1896         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1897     end
1898
1899     if $remote
1900         if !$remoteDirectory
1901             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1902         end
1903         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1904         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1905         remoteScript = "\""
1906         remoteScript += "cd #{$remoteDirectory} && "
1907         remoteScript += "rm -rf #{$outputDir.basename} && "
1908         remoteScript += "tar xzf #{$tarFileName} && "
1909         remoteScript += "cd #{$outputDir.basename}/.runner && "
1910         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1911         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1912         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
1913         $envVars.each { |var| remoteScript += "export " << var << "\n" }
1914         remoteScript += "#{testRunnerCommand}\""
1915         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1916     else
1917         Dir.chdir($runnerDir) {
1918             runAndMonitorTestRunnerCommand(testRunnerCommand)
1919         }
1920     end
1921 end
1922
1923 def detectFailures
1924     raise if $bundle
1925
1926     failures = []
1927     
1928     if $remote
1929         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1930         output.split(/\n/).each {
1931             | line |
1932             next unless line =~ /test_fail_/
1933             failures << $~.post_match.to_i
1934         }
1935     else
1936         Dir.foreach($runnerDir) {
1937             | filename |
1938             next unless filename =~ /test_fail_/
1939             failures << $~.post_match.to_i
1940         }
1941     end
1942
1943     failureSet = {}
1944
1945     failures.each {
1946         | failure | 
1947         appendFailure($runlist[failure])
1948         failureSet[failure] = true
1949     }
1950
1951     familyMap = {}
1952     $runlist.each_with_index {
1953         | plan, index |
1954         unless familyMap[plan.family]
1955             familyMap[plan.family] = []
1956         end
1957         if failureSet[index]
1958             appendResult(plan, false)
1959             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
1960             next
1961         else
1962             appendResult(plan, true)
1963             familyMap[plan.family] << {:result => "PASS", :plan => plan};
1964         end
1965         appendPass(plan)
1966     }
1967
1968     File.open($outputDir + "resultsByFamily", "w") {
1969         | outp |
1970         first = true
1971         familyMap.keys.sort.each {
1972             | familyName |
1973             if first
1974                 first = false
1975             else
1976                 outp.puts
1977             end
1978             
1979             outp.print "#{familyName}:"
1980
1981             numPassed = 0
1982             familyMap[familyName].each {
1983                 | entry |
1984                 if entry[:result] == "PASS"
1985                     numPassed += 1
1986                 end
1987             }
1988
1989             if numPassed == familyMap[familyName].size
1990                 outp.puts " PASSED"
1991             elsif numPassed == 0
1992                 outp.puts " FAILED"
1993             else
1994                 outp.puts
1995                 familyMap[familyName].each {
1996                     | entry |
1997                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
1998                 }
1999             end
2000         }
2001     }
2002 end
2003
2004 def compressBundle
2005     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
2006     $stderr.puts ">> #{cmd}" if $verbosity >= 2
2007     raise unless system(cmd)
2008 end
2009
2010 def clean(file)
2011     FileUtils.rm_rf file unless $bundle
2012 end
2013
2014 clean($outputDir + "failed")
2015 clean($outputDir + "passed")
2016 clean($outputDir + "results")
2017 clean($outputDir + "resultsByFamily")
2018 clean($outputDir + ".vm")
2019 clean($outputDir + ".helpers")
2020 clean($outputDir + ".runner")
2021 clean($outputDir + ".tests")
2022 clean($outputDir + "_payload")
2023
2024 Dir.mkdir($outputDir) unless $outputDir.directory?
2025
2026 $outputDir = $outputDir.realpath
2027 $runnerDir = $outputDir + ".runner"
2028
2029 if !$numChildProcesses
2030     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
2031         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
2032     else
2033         $numChildProcesses = numberOfProcessors
2034     end
2035 end
2036
2037 if $enableFTL and ENV["JSCTEST_timeout"] or !ifJSCArgIsntProvidedAreWeReleaseBuild
2038     # Currently, using the FTL is a performance regression particularly in real
2039     # (i.e. non-loopy) benchmarks. Account for this in the timeout.
2040     # Increase the timeout for debug builds too. (--debug command line option)
2041     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i * 2).to_s
2042 end
2043
2044 if ENV["JSCTEST_timeout"]
2045     # In the worst case, the processors just interfere with each other.
2046     # Increase the timeout proportionally to the number of processors.
2047     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
2048 end
2049
2050 def runBundle
2051     raise unless $bundle
2052
2053     cleanRunnerDirectory
2054     cleanOldResults
2055     runTestRunner
2056     cleanEmptyResultFiles
2057 end
2058
2059 def runNormal
2060     raise if $bundle or $tarball
2061
2062     prepareBundle
2063     prepareTestRunner
2064     runTestRunner
2065     cleanEmptyResultFiles
2066     detectFailures
2067 end
2068
2069 def runTarball
2070     raise unless $tarball
2071
2072     prepareBundle 
2073     prepareTestRunner
2074     compressBundle
2075 end
2076
2077 def runRemote
2078     raise unless $remote
2079
2080     prepareBundle
2081     prepareTestRunner
2082     compressBundle
2083     runTestRunner
2084     detectFailures
2085 end
2086
2087 puts
2088 if $bundle
2089     runBundle
2090 elsif $remote
2091     runRemote
2092 elsif $tarball
2093     runTarball
2094 else
2095     runNormal
2096 end