REGRESSION(r169265): sh: line 0: cd: ../.vm
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013, 2014 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'uri'
30 require 'yaml'
31
32 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
33 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
34 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
35 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
36 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
37 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
38
39 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
40
41 IMPORTANT_ENVS = ["JSC_timeout"]
42
43 begin
44     require 'shellwords'
45 rescue Exception => e
46     $stderr.puts "Warning: did not find shellwords, not running any tests."
47     exit 0
48 end
49
50 $canRunDisplayProfilerOutput = false
51
52 begin
53     require 'rubygems'
54     require 'json'
55     require 'highline'
56     $canRunDisplayProfilerOutput = true
57 rescue Exception => e
58     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
59     $stderr.puts "Error: #{e.inspect}"
60 end
61
62 def printCommandArray(*cmd)
63     begin
64         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
65     rescue
66         commandArray = cmd.join(' ')
67     end
68     $stderr.puts ">> #{commandArray}"
69 end
70
71 def mysys(*cmd)
72     printCommandArray(*cmd) if $verbosity >= 1
73     raise "Command failed: #{$?.inspect}" unless system(*cmd)
74 end
75
76 begin
77     $numProcessors = `sysctl -n hw.activecpu 2>/dev/null`.to_i
78 rescue
79     $numProcessors = 0
80 end
81
82 if $numProcessors == 0
83     $numProcessors = `nproc --all 2>/dev/null`.to_i
84 end
85 if $numProcessors == 0
86     $numProcessors = 1
87 end
88
89 if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
90     $numProcessors = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
91 end
92
93 $jscPath = nil
94 $copy = true
95 $enableFTL = false
96 $outputDir = Pathname.new("results")
97 $verbosity = 0
98 $bundle = nil
99 $tarball = false
100 $copyVM = false
101 $testRunnerType = :make
102 $remoteUser = nil
103 $remoteHost = nil
104 $remotePort = nil
105 $remoteDirectory = nil
106
107 def usage
108     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
109     puts
110     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
111     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
112     puts "                            --jsc specifies an already present JavaScriptCore to test."
113     puts "--ftl-jit                   Indicate that we have the FTL JIT."
114     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
115     puts "--verbose            (-v)   Print more things while running."
116     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
117     puts "--tarball                   Creates a tarball of the final bundle."
118     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
119     puts "                            In general the shell runner is slower than the make runner."
120     puts "--remote                    Specify a remote host on which to run tests."
121     puts "--child-processes    (-c)   Specify the number of child processes."
122     puts "--help               (-h)   Print this message."
123     exit 1
124 end
125
126 jscArg = ""
127
128 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
129                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
130                ['--no-copy', GetoptLong::NO_ARGUMENT],
131                ['--ftl-jit', GetoptLong::NO_ARGUMENT],
132                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
133                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
134                ['--tarball', GetoptLong::NO_ARGUMENT],
135                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
136                ['--shell-runner', GetoptLong::NO_ARGUMENT],
137                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
138                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
139                ['--verbose', '-v', GetoptLong::NO_ARGUMENT]).each {
140     | opt, arg |
141     case opt
142     when '--help'
143         usage
144     when '--jsc'
145         jscArg = arg
146     when '--no-copy'
147         $copy = false
148     when '--output-dir'
149         $outputDir = Pathname.new(arg)
150     when '--ftl-jit'
151         $enableFTL = true
152     when '--verbose'
153         $verbosity += 1
154     when '--run-bundle'
155         $bundle = Pathname.new(arg)
156     when '--tarball'
157         $tarball = true
158         $copyVM = true
159     when '--force-vm-copy'
160         $copyVM = true
161     when '--shell-runner'
162         $testRunnerType = :shell
163     when '--remote'
164         $copyVM = true
165         $testRunnerType = :shell
166         $tarball = true
167         $remote = true
168         uri = URI("ftp://" + arg)
169         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
170     when '--child-processes'
171         $numProcessors = arg.to_i
172     end
173 }
174
175 if jscArg
176     if $copy
177         $jscPath = Pathname.new(jscArg).realpath
178     else
179         $jscPath = Pathname.new(jscArg)
180     end
181 end
182
183 $progressMeter = ($verbosity == 0 and $stdout.tty?)
184
185 if $bundle
186     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
187     $outputDir = $bundle
188 end
189
190 unless $jscPath
191     $stderr.puts "Error: must specify -jsc <path>"
192     exit 1
193 end
194
195 # Try to determine architecture. Return nil on failure.
196 def machOArchitectureCode
197     begin 
198         otoolLines = `otool -aSfh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
199         otoolLines.each_with_index {
200             | value, index |
201             if value =~ /magic/ and value =~ /cputype/
202                 return otoolLines[index + 1].split[1].to_i
203             end
204         }
205     rescue
206         $stderr.puts "Warning: unable to execute otool."
207     end
208     $stderr.puts "Warning: unable to determine architecture."
209     nil
210 end
211
212 def determineArchitecture
213     code = machOArchitectureCode
214     return nil unless code
215     is64BitFlag = 0x01000000
216     case code
217     when 7
218         "x86"
219     when 7 | is64BitFlag
220         "x86-64"
221     when 12
222         "arm"
223     when 12 | is64BitFlag
224         "arm64"
225     else
226         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
227         nil
228     end
229 end
230
231 $architecture = determineArchitecture
232
233 $numFailures = 0
234
235 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20"]
236 NO_CJIT_OPTIONS = ["--enableConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100"]
237 NO_FTL_OPTIONS = ["--useFTLJIT=false"]
238 FTL_OPTIONS = ["--useFTLJIT=true", "--enableExperimentalFTLCoverage=true"]
239
240 $runlist = []
241
242 def frameworkFromJSCPath(jscPath)
243     parentDirectory = jscPath.dirname
244     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
245         parentDirectory.dirname
246     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
247         jscPath.dirname + "JavaScriptCore.framework"
248     else
249         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
250         nil
251     end
252 end
253
254 def prepareFramework(jscPath)
255     frameworkPath = frameworkFromJSCPath(jscPath)
256     destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
257     $jscPath = destinationFrameworkPath + "Resources" + "jsc"
258     $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
259
260     if frameworkPath
261         source = frameworkPath
262         destination = Pathname.new(".vm")
263     else
264         source = jscPath
265         destination = $jscPath
266
267         Dir.chdir($outputDir) {
268             FileUtils.mkdir_p $jscPath.dirname
269         }
270     end
271
272     Dir.chdir($outputDir) {
273         if $copyVM
274             FileUtils.cp_r source, destination
275         else
276             begin 
277                 FileUtils.ln_s source, destination
278             rescue
279                 FileUtils.cp_r source, destination
280             end
281         end
282     }
283 end
284
285 def copyVMToBundle
286     raise if $bundle
287
288     vmDir = $outputDir + ".vm"
289     FileUtils.mkdir_p vmDir
290    
291     prepareFramework($jscPath) 
292 end
293
294 def pathToVM
295     dir = Pathname.new(".")
296     $benchmarkDirectory.each_filename {
297         | pathComponent |
298         dir += ".."
299     }
300     dir + $jscPath
301 end
302
303 def prefixCommand(prefix)
304     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
305 end
306
307 def redirectAndPrefixCommand(prefix)
308     prefixCommand(prefix) + " 2>&1"
309 end
310
311 def pipeAndPrefixCommand(outputFilename, prefix)
312     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
313 end
314
315 # Output handler for tests that are expected to be silent.
316 def silentOutputHandler
317     Proc.new {
318         | name |
319         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
320     }
321 end
322
323 # Output handler for tests that are expected to produce meaningful output.
324 def noisyOutputHandler
325     Proc.new {
326         | name |
327         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
328     }
329 end
330
331 # Error handler for tests that fail exactly when they return non-zero exit status.
332 def simpleErrorHandler
333     Proc.new {
334         | outp, plan |
335         outp.puts "if test -e #{plan.failFile}"
336         outp.puts "then"
337         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
338         outp.puts "    " + plan.failCommand
339         outp.puts "else"
340         outp.puts "    " + plan.successCommand
341         outp.puts "fi"
342     }
343 end
344
345 # Error handler for tests that diff their output with some expectation.
346 def diffErrorHandler(expectedFilename)
347     Proc.new {
348         | outp, plan |
349         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
350         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
351         
352         outp.puts "if test -e #{plan.failFile}"
353         outp.puts "then"
354         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
355         outp.puts "    " + plan.failCommand
356         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
357         outp.puts "then"
358         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
359         outp.puts "    if [ $? -eq 0 ]"
360         outp.puts "    then"
361         outp.puts "    " + plan.successCommand
362         outp.puts "    else"
363         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
364         outp.puts "        " + plan.failCommand
365         outp.puts "    fi"
366         outp.puts "else"
367         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
368         outp.puts "    " + plan.failCommand
369         outp.puts "fi"
370     }
371 end
372
373 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
374 # tests.
375 def mozillaErrorHandler
376     Proc.new {
377         | outp, plan |
378         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
379
380         outp.puts "if test -e #{plan.failFile}"
381         outp.puts "then"
382         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
383         outp.puts "    " + plan.failCommand
384         outp.puts "elif grep -i -q failed! #{outputFilename}"
385         outp.puts "then"
386         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
387         outp.puts "    " + plan.failCommand
388         outp.puts "else"
389         outp.puts "    " + plan.successCommand
390         outp.puts "fi"
391     }
392 end
393
394 # Error handler for tests that report error by saying "failed!", and are expected to
395 # fail. This is used by Mozilla tests.
396 def mozillaFailErrorHandler
397     Proc.new {
398         | outp, plan |
399         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
400
401         outp.puts "if test -e #{plan.failFile}"
402         outp.puts "then"
403         outp.puts "    " + plan.successCommand
404         outp.puts "elif grep -i -q failed! #{outputFilename}"
405         outp.puts "then"
406         outp.puts "    " + plan.successCommand
407         outp.puts "else"
408         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
409         outp.puts "    " + plan.failCommand
410         outp.puts "fi"
411     }
412 end
413
414 # Error handler for tests that report error by saying "failed!", and are expected to have
415 # an exit code of 3.
416 def mozillaExit3ErrorHandler
417     Proc.new {
418         | outp, plan |
419         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
420
421         outp.puts "if test -e #{plan.failFile}"
422         outp.puts "then"
423         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
424         outp.puts "    then"
425         outp.puts "        if grep -i -q failed! #{outputFilename}"
426         outp.puts "        then"
427         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
428         outp.puts "            " + plan.failCommand
429         outp.puts "        else"
430         outp.puts "            " + plan.successCommand
431         outp.puts "        fi"
432         outp.puts "    else"
433         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
434         outp.puts "        " + plan.failCommand
435         outp.puts "    fi"
436         outp.puts "else"
437         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
438         outp.puts "    " + plan.failCommand
439         outp.puts "fi"
440     }
441 end
442
443 $runCommandOptions = {}
444
445 class Plan
446     attr_reader :directory, :arguments, :name, :outputHandler, :errorHandler
447     attr_accessor :index
448     
449     def initialize(directory, arguments, name, outputHandler, errorHandler)
450         @directory = directory
451         @arguments = arguments
452         @name = name
453         @outputHandler = outputHandler
454         @errorHandler = errorHandler
455         @isSlow = !!$runCommandOptions[:isSlow]
456     end
457     
458     def shellCommand
459         # It's important to remember that the test is actually run in a subshell, so if we change directory
460         # in the subshell when we return we will be in our original directory. This is nice because we don't
461         # have to bend over backwards to do things relative to the root.
462         "(cd ../#{Shellwords.shellescape(@directory.to_s)} && \"$@\" " + @arguments.map{
463             | v |
464             raise "Detected a non-string in #{inspect}" unless v.is_a? String
465             Shellwords.shellescape(v)
466         }.join(' ') + ")"
467     end
468     
469     def reproScriptCommand
470         # We have to find our way back to the .runner directory since that's where all of the relative
471         # paths assume they start out from.
472         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
473         script += "cd $CURRENT_DIR\n"
474         Pathname.new(@name).dirname.each_filename {
475             | pathComponent |
476             script += "cd ..\n"
477         }
478         script += "cd .runner\n"
479
480         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
481         IMPORTANT_ENVS.each {
482             | key |
483             if ENV[key]
484                 script += "export #{key}=#{Shellwords.shellescape(ENV[key])}\n"
485             end
486         }
487         script += "#{shellCommand} || exit 1"
488         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
489     end
490     
491     def failCommand
492         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
493     end
494     
495     def successCommand
496         if $progressMeter or $verbosity >= 2
497             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
498         else
499             "rm -f #{failFile}"
500         end
501     end
502     
503     def failFile
504         "test_fail_#{@index}"
505     end
506     
507     def writeRunScript(filename)
508         File.open(filename, "w") {
509             | outp |
510             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
511             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
512             cmd += @outputHandler.call(@name)
513             if $verbosity >= 3
514                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
515             end
516             outp.puts cmd
517             @errorHandler.call(outp, self)
518         }
519     end
520 end
521
522 $uniqueFilenameCounter = 0
523 def uniqueFilename(extension)
524     payloadDir = $outputDir + "_payload"
525     Dir.mkdir payloadDir unless payloadDir.directory?
526     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
527     $uniqueFilenameCounter += 1
528     result
529 end
530
531 def baseOutputName(kind)
532     "#{$collectionName}/#{$benchmark}.#{kind}"
533 end
534
535 def addRunCommand(kind, command, outputHandler, errorHandler)
536     $didAddRunCommand = true
537     plan = Plan.new($benchmarkDirectory, command, baseOutputName(kind), outputHandler, errorHandler)
538     if $numProcessors > 1 and $runCommandOptions[:isSlow]
539         $runlist.unshift plan
540     else
541         $runlist << plan
542     end
543 end
544
545 # Returns true if there were run commands found in the file ($benchmarkDirectory +
546 # $benchmark), in which case those run commands have already been executed. Otherwise
547 # returns false, in which case you're supposed to add your own run commands.
548 def parseRunCommands
549     oldDidAddRunCommand = $didAddRunCommand
550     $didAddRunCommand = false
551
552     Dir.chdir($outputDir) {
553         File.open($benchmarkDirectory + $benchmark) {
554             | inp |
555             inp.each_line {
556                 | line |
557                 begin
558                     doesMatch = line =~ /^\/\/@/
559                 rescue Exception => e
560                     # Apparently this happens in the case of some UTF8 stuff in some files, where
561                     # Ruby tries to be strict and throw exceptions.
562                     next
563                 end
564                 next unless doesMatch
565                 eval $~.post_match
566             }
567         }
568     }
569
570     result = $didAddRunCommand
571     $didAddRunCommand = result or oldDidAddRunCommand
572     result
573 end
574
575 def slow!
576     $runCommandOptions[:isSlow] = true
577 end
578
579 def run(kind, *options)
580     addRunCommand(kind, [pathToVM.to_s] + NO_FTL_OPTIONS + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
581 end
582
583 def runDefault
584     run("default")
585 end
586
587 def runNoLLInt
588     run("no-llint", "--useLLInt=false")
589 end
590
591 def runNoCJITValidate
592     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *NO_CJIT_OPTIONS)
593 end
594
595 def runNoCJITValidatePhases
596     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
597 end
598
599 def runDefaultFTL
600     run("default-ftl", *FTL_OPTIONS) if $enableFTL
601 end
602
603 def runFTLNoCJIT
604     run("ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
605 end
606
607 def runFTLNoCJITValidate
608     run("ftl-no-cjit-validate", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
609 end
610
611 def runFTLNoCJITNoInlineValidate
612     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--maximumInliningDepthForMustInline=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
613 end
614
615 def runFTLNoCJITOSRValidation
616     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
617 end
618
619 def runDFGEager
620     run("dfg-eager", *EAGER_OPTIONS)
621 end
622
623 def runDFGEagerNoCJITValidate
624     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(EAGER_OPTIONS + NO_CJIT_OPTIONS))
625 end
626
627 def runFTLEager
628     run("ftl-eager", *(FTL_OPTIONS + EAGER_OPTIONS)) if $enableFTL
629 end
630
631 def runFTLEagerNoCJITValidate
632     run("ftl-eager-no-cjit", "--validateGraph=true", *(FTL_OPTIONS + EAGER_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
633 end
634
635 def runFTLEagerNoCJITOSRValidation
636     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + EAGER_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
637 end
638
639 def runAlwaysTriggerCopyPhase
640     run("always-trigger-copy-phase", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
641 end
642
643 def runNoCJITNoASO
644     run("no-cjit-no-aso", "--enableArchitectureSpecificOptimizations=false", *NO_CJIT_OPTIONS)
645 end
646
647 def runFTLNoCJITNoSimpleOpt
648     run("ftl-no-cjit-no-simple-opt", "--llvmSimpleOpt=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
649 end
650
651 def runNoCJITNoAccessInlining
652     run("no-cjit-no-access-inlining", "--enableAccessInlining=false", *NO_CJIT_OPTIONS)
653 end
654
655 def runFTLNoCJITNoAccessInlining
656     run("ftl-no-cjit-no-access-inlining", "--enableAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
657 end
658
659 def defaultRun
660     runDefault
661     runNoLLInt
662     runAlwaysTriggerCopyPhase
663     runNoCJITValidatePhases
664     runDFGEager
665     runDFGEagerNoCJITValidate
666     runDefaultFTL
667     runFTLNoCJITValidate
668     runFTLNoCJITNoInlineValidate
669     runFTLEager
670     runFTLEagerNoCJITValidate
671 end
672
673 def defaultQuickRun
674     if $enableFTL
675         runDefaultFTL
676         runFTLNoCJITValidate
677     else
678         runDefault
679         runNoCJITValidate
680     end
681 end
682
683 def defaultSpotCheck
684     defaultQuickRun
685     runFTLNoCJITNoSimpleOpt
686     runFTLNoCJITOSRValidation
687     runNoCJITNoAccessInlining
688     runFTLNoCJITNoAccessInlining
689 end
690
691 # This is expected to not do eager runs because eager runs can have a lot of recompilations
692 # for reasons that don't arise in the real world. It's used for tests that assert convergence
693 # by counting recompilations.
694 def defaultNoEagerRun
695     runDefault
696     runNoLLInt
697     runAlwaysTriggerCopyPhase
698     runNoCJITValidatePhases
699     runDefaultFTL
700     runFTLNoCJITValidate
701     runFTLNoCJITNoInlineValidate
702 end
703
704 def runProfiler
705     profilerOutput = uniqueFilename(".json")
706     if $canRunDisplayProfilerOutput
707         addRunCommand("profiler", ["ruby", (HELPERS_PATH + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
708     else
709         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
710         run("profiler-simple", "-p", profilerOutput.to_s)
711     end
712 end
713
714 def runLayoutTest(kind, *options)
715     raise unless $benchmark.to_s =~ /\.js$/
716     testName = $~.pre_match
717     if kind
718         kind = "layout-" + kind
719     else
720         kind = "layout"
721     end
722
723     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
724     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
725
726     args = [pathToVM.to_s] + NO_FTL_OPTIONS + options +
727         [(Pathname.new("resources") + "standalone-pre.js").to_s,
728          $benchmark.to_s,
729          (Pathname.new("resources") + "standalone-post.js").to_s]
730     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
731 end
732
733 def runLayoutTestDefault
734     runLayoutTest(nil)
735 end
736
737 def runLayoutTestNoLLInt
738     runLayoutTest("no-llint", "--useLLInt=false")
739 end
740
741 def runLayoutTestNoCJIT
742     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
743 end
744
745 def runLayoutTestDFGEagerNoCJIT
746     runLayoutTest("dfg-eager-no-cjit", *(EAGER_OPTIONS + NO_CJIT_OPTIONS))
747 end
748
749 def runLayoutTestDefaultFTL
750     runLayoutTest("ftl", "--testTheFTL=true", *FTL_OPTIONS) if $enableFTL
751 end
752
753 def runLayoutTestFTLNoCJIT
754     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
755 end
756
757 def runLayoutTestFTLEagerNoCJIT
758     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + EAGER_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
759 end
760
761 def noFTLRunLayoutTest
762     runLayoutTestDefault
763     runLayoutTestNoLLInt
764     runLayoutTestNoCJIT
765     runLayoutTestDFGEagerNoCJIT
766 end
767
768 def defaultRunLayoutTest
769     noFTLRunLayoutTest
770     runLayoutTestDefaultFTL
771     runLayoutTestFTLNoCJIT
772     runLayoutTestFTLEagerNoCJIT
773 end
774
775 def prepareExtraRelativeFiles(extraFiles, destination)
776     Dir.chdir($outputDir) {
777         extraFiles.each {
778             | file |
779             FileUtils.cp $extraFilesBaseDir + file, destination + file
780         }
781     }
782 end
783
784 def baseDirForCollection(collectionName)
785     Pathname(".tests") + collectionName
786 end
787
788 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
789     raise unless absoluteBase.absolute?
790     Dir.chdir($outputDir) {
791         collectionBaseDir = baseDirForCollection($collectionName)
792         extraFiles.each {
793             | file |
794             destination = collectionBaseDir + file
795             FileUtils.mkdir_p destination.dirname unless destination.directory?
796             FileUtils.cp absoluteBase + file, destination
797         }
798     }
799 end
800
801 def runMozillaTest(kind, mode, extraFiles, *options)
802     if kind
803         kind = "mozilla-" + kind
804     else
805         kind = "mozilla"
806     end
807     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
808     args = [pathToVM.to_s] + NO_FTL_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
809     case mode
810     when :normal
811         errorHandler = mozillaErrorHandler
812     when :negative
813         errorHandler = mozillaExit3ErrorHandler
814     when :fail
815         errorHandler = mozillaFailErrorHandler
816     when :skip
817         return
818     else
819         raise "Invalid mode: #{mode}"
820     end
821     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
822 end
823
824 def runMozillaTestDefault(mode, *extraFiles)
825     runMozillaTest(nil, mode, extraFiles)
826 end
827
828 def runMozillaTestDefaultFTL(mode, *extraFiles)
829     runMozillaTest("ftl", mode, extraFiles, *FTL_OPTIONS) if $enableFTL
830 end
831
832 def runMozillaTestLLInt(mode, *extraFiles)
833     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
834 end
835
836 def runMozillaTestBaselineJIT(mode, *extraFiles)
837     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
838 end
839
840 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
841     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(EAGER_OPTIONS + NO_CJIT_OPTIONS))
842 end
843
844 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
845     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + EAGER_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
846 end
847
848 def defaultRunMozillaTest(mode, *extraFiles)
849     runMozillaTestDefault(mode, *extraFiles)
850     runMozillaTestLLInt(mode, *extraFiles)
851     runMozillaTestBaselineJIT(mode, *extraFiles)
852     runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
853     runMozillaTestDefaultFTL(mode, *extraFiles)
854     runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
855 end
856
857 def skip
858     $didAddRunCommand = true
859     puts "Skipping #{$collectionName}/#{$benchmark}"
860 end
861
862 def allJSFiles(path)
863     if path.file?
864         [path]
865     else
866         result = []
867         Dir.foreach(path) {
868             | filename |
869             next unless filename =~ /\.js$/
870             next unless (path + filename).file?
871             result << path + filename
872         }
873         result
874     end
875 end
876
877 def uniqueifyName(names, name)
878     result = name.to_s
879     toAdd = 1
880     while names[result]
881         result = "#{name}-#{toAdd}"
882         toAdd += 1
883     end
884     names[result] = true
885     result
886 end
887
888 def simplifyCollectionName(collectionPath)
889     outerDir = collectionPath.dirname
890     name = collectionPath.basename
891     lastName = name
892     if collectionPath.directory?
893         while lastName.to_s =~ /test/
894             lastName = outerDir.basename
895             name = lastName + name
896             outerDir = outerDir.dirname
897         end
898     end
899     uniqueifyName($collectionNames, name)
900 end
901
902 def prepareCollection(name)
903     FileUtils.mkdir_p $outputDir + name
904
905     absoluteCollection = $collection.realpath
906
907     Dir.chdir($outputDir) {
908         bundleDir = baseDirForCollection(name)
909
910         # Create the proper directory structures.
911         FileUtils.mkdir_p bundleDir
912         if bundleDir.basename == $collection.basename
913             FileUtils.cp_r absoluteCollection, bundleDir.dirname
914         else
915             FileUtils.cp_r absoluteCollection, bundleDir
916         end
917
918         $extraFilesBaseDir = absoluteCollection
919
920         # Redirect the collection's location to the newly constructed bundle.
921         if absoluteCollection.directory?
922             $collection = bundleDir
923         else
924             $collection = bundleDir + $collection.basename
925         end
926     }
927 end
928
929 $collectionNames = {}
930
931 def handleCollectionFile(collection)
932     collectionName = simplifyCollectionName(collection)
933    
934     paths = {}
935     subCollections = []
936     YAML::load(IO::read(collection)).each {
937         | entry |
938         if entry["collection"]
939             subCollections << entry["collection"]
940             next
941         end
942         
943         if Pathname.new(entry["path"]).absolute?
944             raise "Absolute path: " + entry["path"] + " in #{collection}"
945         end
946         
947         if paths[entry["path"]]
948             raise "Duplicate path: " + entry["path"] + " in #{collection}"
949         end
950         
951         subCollection = collection.dirname + entry["path"]
952         
953         if subCollection.file?
954             subCollectionName = Pathname.new(entry["path"]).dirname
955         else
956             subCollectionName = entry["path"]
957         end
958         
959         $collection = subCollection
960         $collectionName = Pathname.new(collectionName)
961         Pathname.new(subCollectionName).each_filename {
962             | filename |
963             next if filename =~ /^\./
964             $collectionName += filename
965         }
966         $collectionName = $collectionName.to_s
967         
968         prepareCollection($collectionName)
969       
970         Dir.chdir($outputDir) {
971             directoryToSearch = $collection
972             if entry["tests"]
973                 directoryToSearch += entry["tests"]
974             end
975             allJSFiles(directoryToSearch).each {
976                 | path |
977                
978                 $benchmark = path.basename
979                 $benchmarkDirectory = path.dirname
980                 
981                 $runCommandOptions = {}
982                 eval entry["cmd"]
983             }
984         }
985     }
986     
987     subCollections.each {
988         | subCollection |
989         handleCollection(collection.dirname + subCollection)
990     }
991 end
992
993 def handleCollectionDirectory(collection)
994     collectionName = simplifyCollectionName(collection)
995     
996     $collection = collection
997     $collectionName = collectionName
998     prepareCollection(collectionName)
999    
1000     Dir.chdir($outputDir) {
1001         $benchmarkDirectory = $collection
1002         allJSFiles($collection).each {
1003             | path |
1004             
1005             $benchmark = path.basename
1006             
1007             $runCommandOptions = {}
1008             defaultRun unless parseRunCommands
1009         }
1010     }
1011 end
1012
1013 def handleCollection(collection)
1014     collection = Pathname.new(collection)
1015     
1016     if collection.file?
1017         handleCollectionFile(collection)
1018     else
1019         handleCollectionDirectory(collection)
1020     end
1021 end
1022
1023 def appendFailure(plan)
1024     File.open($outputDir + "failed", "a") {
1025         | outp |
1026         outp.puts plan.name
1027     }
1028     $numFailures += 1
1029 end
1030
1031 def prepareBundle
1032     raise if $bundle
1033
1034     if $copy
1035         copyVMToBundle
1036     else
1037         if !$remote
1038             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1039             $jscPath = Pathname.new($jscPath).realpath
1040         else
1041             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1042         end
1043     end
1044
1045     ARGV.each {
1046         | collection |
1047         handleCollection(collection)
1048     }
1049
1050     puts
1051 end
1052
1053 def cleanOldResults
1054     raise unless $bundle
1055
1056     eachResultFile($outputDir) {
1057         | path |
1058         FileUtils.rm_f path
1059     }
1060 end
1061
1062 def cleanEmptyResultFiles
1063     eachResultFile($outputDir) {
1064         | path |
1065         next unless path.basename.to_s =~ /\.out$/
1066         next unless FileTest.size(path) == 0
1067         FileUtils.rm_f path
1068     }
1069 end
1070
1071 def eachResultFile(startingDir, &block)
1072     dirsToClean = [startingDir]
1073     until dirsToClean.empty?
1074         nextDir = dirsToClean.pop
1075         Dir.foreach(nextDir) {
1076             | entry |
1077             next if entry =~ /^\./
1078             path = nextDir + entry
1079             if path.directory?
1080                 dirsToClean.push(path)
1081             else
1082                 block.call(path)
1083             end
1084         }
1085     end
1086 end
1087
1088 def prepareTestRunner
1089     raise if $bundle
1090
1091     $runlist.each_with_index {
1092         | plan, index |
1093         plan.index = index
1094     }
1095
1096     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1097     toDelete = []
1098     Dir.foreach($runnerDir) {
1099         | filename |
1100         if filename =~ /^test_/
1101             toDelete << filename
1102         end
1103     }
1104     
1105     toDelete.each {
1106         | filename |
1107         File.unlink($runnerDir + filename)
1108     }
1109
1110     $runlist.each {
1111         | plan |
1112         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1113     }
1114
1115     case $testRunnerType
1116     when :make
1117         prepareMakeTestRunner
1118     when :shell
1119         prepareShellTestRunner
1120     else
1121         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1122     end
1123 end
1124
1125 def prepareShellTestRunner
1126     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
1127 end
1128
1129 def prepareMakeTestRunner
1130     # The goals of our parallel test runner are scalability and simplicity. The
1131     # simplicity part is particularly important. We don't want to have to have
1132     # a full-time contributor just philosophising about parallel testing.
1133     #
1134     # As such, we just pass off all of the hard work to 'make'. This creates a
1135     # dummy directory ("$outputDir/.runner") in which we create a dummy
1136     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
1137     # That is, for each test we know we will run, there is a rule in the
1138     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
1139     # Makefile results in 'make' doing all of the hard work:
1140     #
1141     # - Load balancing just works. Most systems have a great load balancer in
1142     #   'make'. If your system doesn't then just install a real 'make'.
1143     #
1144     # - Interruptions just work. For example Ctrl-C handling in 'make' is
1145     #   exactly right. You don't have to worry about zombie processes.
1146     #
1147     # We then do some tricks to make failure detection work and to make this
1148     # totally sound. If a test fails, we don't want the whole 'make' job to
1149     # stop. We also don't have any facility for makefile-escaping of path names.
1150     # We do have such a thing for shell-escaping, though. We fix both problems
1151     # by having the actual work for each of the test rules be done in a shell
1152     # script on the side. There is one such script per test. The script responds
1153     # to failure by printing something on the console and then touching a
1154     # failure file for that test, but then still returns 0. This makes 'make'
1155     # continue past that failure and complete all the tests anyway.
1156     #
1157     # In the end, this script collects all of the failures by searching for
1158     # files in the .runner directory whose name matches /^test_fail_/, where
1159     # the thing after the 'fail_' is the test index. Those are the files that
1160     # would be created by the test scripts if they detect failure. We're
1161     # basically using the filesystem as a concurrent database of test failures.
1162     # Even if two tests fail at the same time, since they're touching different
1163     # files we won't miss any failures.
1164     runIndices = []
1165     $runlist.each {
1166         | plan |
1167         runIndices << plan.index
1168     }
1169     
1170     File.open($runnerDir + "Makefile", "w") {
1171         | outp |
1172         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1173         runIndices.each {
1174             | index |
1175             plan = $runlist[index]
1176             outp.puts "test_done_#{index}:"
1177             outp.puts "\tsh test_script_#{plan.index}"
1178         }
1179     }
1180 end
1181
1182 if $enableFTL and ENV["JSC_timeout"]
1183     # Currently, using the FTL is a performance regression particularly in real
1184     # (i.e. non-loopy) benchmarks. Account for this in the timeout.
1185     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i * 2).to_s
1186 end
1187
1188 if ENV["JSC_timeout"]
1189     # In the worst case, the processors just interfere with each other.
1190     # Increase the timeout proportionally to the number of processors.
1191     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i.to_f * Math.sqrt($numProcessors)).to_i.to_s
1192 end
1193     
1194 puts
1195
1196 def cleanRunnerDirectory
1197     raise unless $bundle
1198     Dir.foreach($runnerDir) {
1199         | filename |
1200         next unless filename =~ /^test_fail/
1201         FileUtils.rm_f $runnerDir + filename
1202     }
1203 end
1204
1205 def runTestRunner
1206     case $testRunnerType
1207     when :shell
1208         runShellTestRunner
1209     when :make
1210         runMakeTestRunner
1211     else
1212         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1213     end
1214 end
1215
1216 def sshRead(cmd)
1217     raise unless $remote
1218
1219     result = ""
1220     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1221       | inp |
1222       inp.each_line {
1223         | line |
1224         result += line
1225       }
1226     }
1227     raise "#{$?}" unless $?.success?
1228     result
1229 end
1230
1231 def runAndMonitorTestRunnerCommand(numberOfTests, *cmd)
1232     unless $progressMeter
1233         mysys(cmd.join(' '))
1234     else
1235        running = {}
1236        didRun = {}
1237        didFail = {}
1238        blankLine = true
1239        prevStringLength = 0
1240        IO.popen(cmd.join(' '), mode="r") {
1241            | inp |
1242            inp.each_line {
1243                | line |
1244                line.chomp!
1245                if line =~ /^Running /
1246                    running[$~.post_match] = true
1247                elsif line =~ /^PASS: /
1248                    didRun[$~.post_match] = true
1249                elsif line =~ /^FAIL: /
1250                    didRun[$~.post_match] = true
1251                    didFail[$~.post_match] = true
1252                else
1253                    unless blankLine
1254                        print("\r" + " " * prevStringLength + "\r")
1255                    end
1256                    puts line
1257                    blankLine = true
1258                end
1259
1260                def lpad(str, chars)
1261                    str = str.to_s
1262                    if str.length > chars
1263                        str
1264                    else
1265                       "%#{chars}s"%(str)
1266                    end
1267                end
1268
1269                string  = ""
1270                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1271                unless didFail.empty?
1272                    string += " (failed #{didFail.size})"
1273                end
1274                string += " "
1275                (running.size - didRun.size).times {
1276                    string += "."
1277                }
1278                if string.length < prevStringLength
1279                    print string
1280                    print(" " * (prevStringLength - string.length))
1281                end
1282                print string
1283                prevStringLength = string.length
1284                blankLine = false
1285                $stdout.flush
1286            }
1287        }
1288        puts
1289        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1290     end
1291 end
1292
1293 def runShellTestRunner
1294     if $remote
1295         numberOfTests = 0
1296         Dir.chdir($runnerDir) {
1297             # -1 for the runscript, and -2 for '..' and '.'
1298             numberOfTests = Dir.entries(".").count - 3
1299         }
1300
1301         $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1302         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + "payload.tar.gz").to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1303         remoteScript = "\""
1304         remoteScript += "cd #{$remoteDirectory} && "
1305         remoteScript += "rm -rf #{$outputDir.basename} && "
1306         remoteScript += "tar xzf payload.tar.gz && "
1307         remoteScript += "cd #{$outputDir.basename}/.runner && "
1308         remoteScript += "DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" sh runscript"
1309         remoteScript += "\""
1310         runAndMonitorTestRunnerCommand(numberOfTests, "ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1311     else
1312         Dir.chdir($runnerDir) {
1313             # -1 for the runscript, and -2 for '..' and '.'
1314             numberOfTests = Dir.entries(".").count - 3
1315             runAndMonitorTestRunnerCommand(numberOfTests, "sh", "runscript")
1316         }
1317     end
1318 end
1319
1320 def runMakeTestRunner
1321     raise if $remote
1322     Dir.chdir($runnerDir) {
1323         # -1 for the Makefile, and -2 for '..' and '.'
1324         numberOfTests = Dir.entries(".").count - 3
1325         runAndMonitorTestRunnerCommand(numberOfTests, "make", "-j", $numProcessors.to_s, "-s", "-f", "Makefile")
1326     }
1327 end
1328
1329 def detectFailures
1330     raise if $bundle
1331
1332     if $remote
1333         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -name \"test_fail_*\" -depth 1")
1334         output.split(/\n/).each {
1335             | line |
1336             next unless line =~ /test_fail_/
1337             appendFailure($runlist[$~.post_match.to_i])
1338         }
1339     else
1340         Dir.foreach($runnerDir) {
1341             | filename |
1342             next unless filename =~ /test_fail_/
1343             appendFailure($runlist[$~.post_match.to_i])
1344         }
1345     end
1346 end
1347
1348 def compressBundle
1349     cmd = "cd #{$outputDir}/.. && tar -czf payload.tar.gz #{$outputDir.basename}"
1350     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1351     raise unless system(cmd)
1352 end
1353
1354 def clean(file)
1355     FileUtils.rm_rf file unless $bundle
1356 end
1357
1358 clean($outputDir + "failed")
1359 clean($outputDir + ".vm")
1360 clean($outputDir + ".runner")
1361 clean($outputDir + ".tests")
1362 clean($outputDir + "_payload")
1363
1364 Dir.mkdir($outputDir) unless $outputDir.directory?
1365
1366 $outputDir = $outputDir.realpath
1367 $runnerDir = $outputDir + ".runner"
1368
1369 def runBundle
1370     raise unless $bundle
1371
1372     cleanRunnerDirectory
1373     cleanOldResults
1374     runTestRunner
1375     cleanEmptyResultFiles
1376 end
1377
1378 def runNormal
1379     raise if $bundle or $tarball
1380
1381     prepareBundle
1382     prepareTestRunner
1383     runTestRunner
1384     cleanEmptyResultFiles
1385     detectFailures
1386 end
1387
1388 def runTarball
1389     raise unless $tarball
1390
1391     prepareBundle 
1392     prepareTestRunner
1393     compressBundle
1394 end
1395
1396 def runRemote
1397     raise unless $remote
1398
1399     prepareBundle
1400     prepareTestRunner
1401     compressBundle
1402     runTestRunner
1403     detectFailures
1404 end
1405
1406 if $bundle
1407     runBundle
1408 elsif $remote
1409     runRemote
1410 elsif $tarball
1411     runTarball
1412 else
1413     runNormal
1414 end