Disable function.arguments
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013, 2014 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
34 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
35 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
36 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
37 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
38 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
39
40 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
41
42 IMPORTANT_ENVS = ["JSC_timeout"]
43
44 begin
45     require 'shellwords'
46 rescue Exception => e
47     $stderr.puts "Warning: did not find shellwords, not running any tests."
48     exit 0
49 end
50
51 $canRunDisplayProfilerOutput = false
52
53 begin
54     require 'rubygems'
55     require 'json'
56     require 'highline'
57     $canRunDisplayProfilerOutput = true
58 rescue Exception => e
59     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
60     $stderr.puts "Error: #{e.inspect}"
61 end
62
63 def printCommandArray(*cmd)
64     begin
65         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
66     rescue
67         commandArray = cmd.join(' ')
68     end
69     $stderr.puts ">> #{commandArray}"
70 end
71
72 def mysys(*cmd)
73     printCommandArray(*cmd) if $verbosity >= 1
74     raise "Command failed: #{$?.inspect}" unless system(*cmd)
75 end
76
77 def escapeAll(array)
78     array.map {
79         | v |
80         raise "Detected a non-string in #{inspect}" unless v.is_a? String
81         Shellwords.shellescape(v)
82     }.join(' ')
83 end
84
85 begin
86     $numProcessors = `sysctl -n hw.activecpu 2>/dev/null`.to_i
87 rescue
88     $numProcessors = 0
89 end
90
91 if $numProcessors == 0
92     $numProcessors = `nproc --all 2>/dev/null`.to_i
93 end
94 if $numProcessors == 0
95     $numProcessors = 1
96 end
97
98 if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
99     $numProcessors = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
100 end
101
102 $jscPath = nil
103 $doNotMessWithVMPath = false
104 $enableFTL = false
105 $memoryLimited = false
106 $outputDir = Pathname.new("results")
107 $verbosity = 0
108 $bundle = nil
109 $tarball = false
110 $copyVM = false
111 $testRunnerType = :make
112 $remoteUser = nil
113 $remoteHost = nil
114 $remotePort = nil
115 $remoteDirectory = nil
116
117 def usage
118     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
119     puts
120     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
121     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
122     puts "                            --jsc specifies an already present JavaScriptCore to test."
123     puts "--ftl-jit                   Indicate that we have the FTL JIT."
124     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
125     puts "                            Skip tests tagged with //@large-heap"
126     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
127     puts "--verbose            (-v)   Print more things while running."
128     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
129     puts "--tarball                   Creates a tarball of the final bundle."
130     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
131     puts "                            In general the shell runner is slower than the make runner."
132     puts "--remote                    Specify a remote host on which to run tests."
133     puts "--child-processes    (-c)   Specify the number of child processes."
134     puts "--help               (-h)   Print this message."
135     exit 1
136 end
137
138 jscArg = nil
139
140 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
141                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
142                ['--no-copy', GetoptLong::NO_ARGUMENT],
143                ['--ftl-jit', GetoptLong::NO_ARGUMENT],
144                ['--memory-limited', GetoptLong::NO_ARGUMENT],
145                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
146                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
147                ['--tarball', GetoptLong::NO_ARGUMENT],
148                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
149                ['--shell-runner', GetoptLong::NO_ARGUMENT],
150                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
151                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
152                ['--verbose', '-v', GetoptLong::NO_ARGUMENT]).each {
153     | opt, arg |
154     case opt
155     when '--help'
156         usage
157     when '--jsc'
158         jscArg = arg
159     when '--no-copy'
160         $doNotMessWithVMPath = true
161     when '--output-dir'
162         $outputDir = Pathname.new(arg)
163     when '--ftl-jit'
164         $enableFTL = true
165     when '--memory-limited'
166         $memoryLimited = true
167     when '--verbose'
168         $verbosity += 1
169     when '--run-bundle'
170         $bundle = Pathname.new(arg)
171     when '--tarball'
172         $tarball = true
173         $copyVM = true
174     when '--force-vm-copy'
175         $copyVM = true
176     when '--shell-runner'
177         $testRunnerType = :shell
178     when '--remote'
179         $copyVM = true
180         $testRunnerType = :shell
181         $tarball = true
182         $remote = true
183         uri = URI("ftp://" + arg)
184         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
185     when '--child-processes'
186         $numProcessors = arg.to_i
187     end
188 }
189
190 unless jscArg
191     $stderr.puts "Error: must specify -jsc <path>"
192     exit 1
193 end
194
195 if $doNotMessWithVMPath
196     $jscPath = Pathname.new(jscArg)
197 else
198     $jscPath = Pathname.new(jscArg).realpath
199 end
200
201 $progressMeter = ($verbosity == 0 and $stdout.tty?)
202
203 if $bundle
204     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
205     $outputDir = $bundle
206 end
207
208 # Try to determine architecture. Return nil on failure.
209 def machOArchitectureCode
210     begin 
211         otoolLines = `otool -aSfh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
212         otoolLines.each_with_index {
213             | value, index |
214             if value =~ /magic/ and value =~ /cputype/
215                 return otoolLines[index + 1].split[1].to_i
216             end
217         }
218     rescue
219         $stderr.puts "Warning: unable to execute otool."
220     end
221     $stderr.puts "Warning: unable to determine architecture."
222     nil
223 end
224
225 def determineArchitecture
226     code = machOArchitectureCode
227     return nil unless code
228     is64BitFlag = 0x01000000
229     case code
230     when 7
231         "x86"
232     when 7 | is64BitFlag
233         "x86-64"
234     when 12
235         "arm"
236     when 12 | is64BitFlag
237         "arm64"
238     else
239         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
240         nil
241     end
242 end
243
244 def determineOS
245     case RbConfig::CONFIG["host_os"]
246     when /darwin/i
247         "darwin"
248     when /linux/i
249         "linux"
250     when /mswin|mingw|cygwin/
251         "windows"
252     else
253         $stderr.puts "Warning: unable to determine host operating system"
254         nil
255     end
256 end
257
258 $architecture = determineArchitecture
259 $hostOS = determineOS
260
261 $numFailures = 0
262
263 BASE_OPTIONS = ["--useFTLJIT=false", "--enableFunctionDotArguments=true"]
264 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20"]
265 NO_CJIT_OPTIONS = ["--enableConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100"]
266 FTL_OPTIONS = ["--useFTLJIT=true", "--enableExperimentalFTLCoverage=true"]
267
268 $runlist = []
269
270 def frameworkFromJSCPath(jscPath)
271     parentDirectory = jscPath.dirname
272     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
273         parentDirectory.dirname
274     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
275         jscPath.dirname + "JavaScriptCore.framework"
276     else
277         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
278         nil
279     end
280 end
281
282 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
283     dir = Pathname.new(".")
284     $benchmarkDirectory.each_filename {
285         | pathComponent |
286         dir += ".."
287     }
288     dir + resourcePath
289 end
290
291 def pathToVM
292     pathToBundleResourceFromBenchmarkDirectory($jscPath)
293 end
294
295 def pathToHelpers
296     pathToBundleResourceFromBenchmarkDirectory(".helpers")
297 end
298
299 def prefixCommand(prefix)
300     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
301 end
302
303 def redirectAndPrefixCommand(prefix)
304     prefixCommand(prefix) + " 2>&1"
305 end
306
307 def pipeAndPrefixCommand(outputFilename, prefix)
308     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
309 end
310
311 # Output handler for tests that are expected to be silent.
312 def silentOutputHandler
313     Proc.new {
314         | name |
315         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
316     }
317 end
318
319 # Output handler for tests that are expected to produce meaningful output.
320 def noisyOutputHandler
321     Proc.new {
322         | name |
323         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
324     }
325 end
326
327 # Error handler for tests that fail exactly when they return non-zero exit status.
328 def simpleErrorHandler
329     Proc.new {
330         | outp, plan |
331         outp.puts "if test -e #{plan.failFile}"
332         outp.puts "then"
333         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
334         outp.puts "    " + plan.failCommand
335         outp.puts "else"
336         outp.puts "    " + plan.successCommand
337         outp.puts "fi"
338     }
339 end
340
341 # Error handler for tests that diff their output with some expectation.
342 def diffErrorHandler(expectedFilename)
343     Proc.new {
344         | outp, plan |
345         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
346         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
347         
348         outp.puts "if test -e #{plan.failFile}"
349         outp.puts "then"
350         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
351         outp.puts "    " + plan.failCommand
352         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
353         outp.puts "then"
354         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
355         outp.puts "    if [ $? -eq 0 ]"
356         outp.puts "    then"
357         outp.puts "    " + plan.successCommand
358         outp.puts "    else"
359         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
360         outp.puts "        " + plan.failCommand
361         outp.puts "    fi"
362         outp.puts "else"
363         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
364         outp.puts "    " + plan.failCommand
365         outp.puts "fi"
366     }
367 end
368
369 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
370 # tests.
371 def mozillaErrorHandler
372     Proc.new {
373         | outp, plan |
374         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
375
376         outp.puts "if test -e #{plan.failFile}"
377         outp.puts "then"
378         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
379         outp.puts "    " + plan.failCommand
380         outp.puts "elif grep -i -q failed! #{outputFilename}"
381         outp.puts "then"
382         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
383         outp.puts "    " + plan.failCommand
384         outp.puts "else"
385         outp.puts "    " + plan.successCommand
386         outp.puts "fi"
387     }
388 end
389
390 # Error handler for tests that report error by saying "failed!", and are expected to
391 # fail. This is used by Mozilla tests.
392 def mozillaFailErrorHandler
393     Proc.new {
394         | outp, plan |
395         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
396
397         outp.puts "if test -e #{plan.failFile}"
398         outp.puts "then"
399         outp.puts "    " + plan.successCommand
400         outp.puts "elif grep -i -q failed! #{outputFilename}"
401         outp.puts "then"
402         outp.puts "    " + plan.successCommand
403         outp.puts "else"
404         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
405         outp.puts "    " + plan.failCommand
406         outp.puts "fi"
407     }
408 end
409
410 # Error handler for tests that report error by saying "failed!", and are expected to have
411 # an exit code of 3.
412 def mozillaExit3ErrorHandler
413     Proc.new {
414         | outp, plan |
415         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
416
417         outp.puts "if test -e #{plan.failFile}"
418         outp.puts "then"
419         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
420         outp.puts "    then"
421         outp.puts "        if grep -i -q failed! #{outputFilename}"
422         outp.puts "        then"
423         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
424         outp.puts "            " + plan.failCommand
425         outp.puts "        else"
426         outp.puts "            " + plan.successCommand
427         outp.puts "        fi"
428         outp.puts "    else"
429         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
430         outp.puts "        " + plan.failCommand
431         outp.puts "    fi"
432         outp.puts "else"
433         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
434         outp.puts "    " + plan.failCommand
435         outp.puts "fi"
436     }
437 end
438
439 $runCommandOptions = {}
440
441 class Plan
442     attr_reader :directory, :arguments, :name, :outputHandler, :errorHandler
443     attr_accessor :index
444     
445     def initialize(directory, arguments, name, outputHandler, errorHandler)
446         @directory = directory
447         @arguments = arguments
448         @name = name
449         @outputHandler = outputHandler
450         @errorHandler = errorHandler
451         @isSlow = !!$runCommandOptions[:isSlow]
452     end
453     
454     def shellCommand
455         # It's important to remember that the test is actually run in a subshell, so if we change directory
456         # in the subshell when we return we will be in our original directory. This is nice because we don't
457         # have to bend over backwards to do things relative to the root.
458         "(cd ../#{Shellwords.shellescape(@directory.to_s)} && \"$@\" " + escapeAll(@arguments) + ")"
459     end
460     
461     def reproScriptCommand
462         # We have to find our way back to the .runner directory since that's where all of the relative
463         # paths assume they start out from.
464         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
465         script += "cd $CURRENT_DIR\n"
466         Pathname.new(@name).dirname.each_filename {
467             | pathComponent |
468             script += "cd ..\n"
469         }
470         script += "cd .runner\n"
471
472         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
473         IMPORTANT_ENVS.each {
474             | key |
475             if ENV[key]
476                 script += "export #{key}=#{Shellwords.shellescape(ENV[key])}\n"
477             end
478         }
479         script += "#{shellCommand} || exit 1"
480         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
481     end
482     
483     def failCommand
484         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
485     end
486     
487     def successCommand
488         if $progressMeter or $verbosity >= 2
489             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
490         else
491             "rm -f #{failFile}"
492         end
493     end
494     
495     def failFile
496         "test_fail_#{@index}"
497     end
498     
499     def writeRunScript(filename)
500         File.open(filename, "w") {
501             | outp |
502             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
503             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
504             cmd += @outputHandler.call(@name)
505             if $verbosity >= 3
506                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
507             end
508             outp.puts cmd
509             @errorHandler.call(outp, self)
510         }
511     end
512 end
513
514 $uniqueFilenameCounter = 0
515 def uniqueFilename(extension)
516     payloadDir = $outputDir + "_payload"
517     Dir.mkdir payloadDir unless payloadDir.directory?
518     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
519     $uniqueFilenameCounter += 1
520     result
521 end
522
523 def baseOutputName(kind)
524     "#{$collectionName}/#{$benchmark}.#{kind}"
525 end
526
527 def addRunCommand(kind, command, outputHandler, errorHandler)
528     $didAddRunCommand = true
529     plan = Plan.new($benchmarkDirectory, command, baseOutputName(kind), outputHandler, errorHandler)
530     if $numProcessors > 1 and $runCommandOptions[:isSlow]
531         $runlist.unshift plan
532     else
533         $runlist << plan
534     end
535 end
536
537 # Returns true if there were run commands found in the file ($benchmarkDirectory +
538 # $benchmark), in which case those run commands have already been executed. Otherwise
539 # returns false, in which case you're supposed to add your own run commands.
540 def parseRunCommands
541     oldDidAddRunCommand = $didAddRunCommand
542     $didAddRunCommand = false
543
544     Dir.chdir($outputDir) {
545         File.open($benchmarkDirectory + $benchmark) {
546             | inp |
547             inp.each_line {
548                 | line |
549                 begin
550                     doesMatch = line =~ /^\/\/@/
551                 rescue Exception => e
552                     # Apparently this happens in the case of some UTF8 stuff in some files, where
553                     # Ruby tries to be strict and throw exceptions.
554                     next
555                 end
556                 next unless doesMatch
557                 eval $~.post_match
558             }
559         }
560     }
561
562     result = $didAddRunCommand
563     $didAddRunCommand = result or oldDidAddRunCommand
564     result
565 end
566
567 def slow!
568     $runCommandOptions[:isSlow] = true
569 end
570
571 def run(kind, *options)
572     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
573 end
574
575 def runDefault
576     run("default")
577 end
578
579 def runNoLLInt
580     run("no-llint", "--useLLInt=false")
581 end
582
583 def runNoCJITValidate
584     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *NO_CJIT_OPTIONS)
585 end
586
587 def runNoCJITValidatePhases
588     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
589 end
590
591 def runDefaultFTL
592     run("default-ftl", *FTL_OPTIONS) if $enableFTL
593 end
594
595 def runFTLNoCJIT
596     run("ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
597 end
598
599 def runFTLNoCJITValidate
600     run("ftl-no-cjit-validate", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
601 end
602
603 def runFTLNoCJITNoInlineValidate
604     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", "--maximumInliningDepthForMustInline=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
605 end
606
607 def runFTLNoCJITOSRValidation
608     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
609 end
610
611 def runDFGEager
612     run("dfg-eager", *EAGER_OPTIONS)
613 end
614
615 def runDFGEagerNoCJITValidate
616     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
617 end
618
619 def runFTLEager
620     run("ftl-eager", *(FTL_OPTIONS + EAGER_OPTIONS)) if $enableFTL
621 end
622
623 def runFTLEagerNoCJITValidate
624     run("ftl-eager-no-cjit", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
625 end
626
627 def runFTLEagerNoCJITOSRValidation
628     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
629 end
630
631 def runAlwaysTriggerCopyPhase
632     run("always-trigger-copy-phase", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
633 end
634
635 def runNoCJITNoASO
636     run("no-cjit-no-aso", "--enableArchitectureSpecificOptimizations=false", *NO_CJIT_OPTIONS)
637 end
638
639 def runFTLNoCJITNoSimpleOpt
640     run("ftl-no-cjit-no-simple-opt", "--llvmSimpleOpt=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
641 end
642
643 def runNoCJITNoAccessInlining
644     run("no-cjit-no-access-inlining", "--enableAccessInlining=false", *NO_CJIT_OPTIONS)
645 end
646
647 def runFTLNoCJITNoAccessInlining
648     run("ftl-no-cjit-no-access-inlining", "--enableAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
649 end
650
651 def defaultRun
652     runDefault
653     runNoLLInt
654     runAlwaysTriggerCopyPhase
655     runNoCJITValidatePhases
656     runDFGEager
657     runDFGEagerNoCJITValidate
658     runDefaultFTL
659     runFTLNoCJITValidate
660     runFTLNoCJITNoInlineValidate
661     runFTLEager
662     runFTLEagerNoCJITValidate
663 end
664
665 def defaultQuickRun
666     if $enableFTL
667         runDefaultFTL
668         runFTLNoCJITValidate
669     else
670         runDefault
671         runNoCJITValidate
672     end
673 end
674
675 def defaultSpotCheck
676     defaultQuickRun
677     runFTLNoCJITNoSimpleOpt
678     runFTLNoCJITOSRValidation
679     runNoCJITNoAccessInlining
680     runFTLNoCJITNoAccessInlining
681 end
682
683 # This is expected to not do eager runs because eager runs can have a lot of recompilations
684 # for reasons that don't arise in the real world. It's used for tests that assert convergence
685 # by counting recompilations.
686 def defaultNoEagerRun
687     runDefault
688     runNoLLInt
689     runAlwaysTriggerCopyPhase
690     runNoCJITValidatePhases
691     runDefaultFTL
692     runFTLNoCJITValidate
693     runFTLNoCJITNoInlineValidate
694 end
695
696 def runProfiler
697     profilerOutput = uniqueFilename(".json")
698     if $canRunDisplayProfilerOutput
699         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
700     else
701         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
702         run("profiler-simple", "-p", profilerOutput.to_s)
703     end
704 end
705
706 def runExceptionFuzz
707     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
708     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
709 end
710
711 def runTypeProfiler
712     if $enableFTL
713         run("ftl-no-cjit-type-profiler", "--enableTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
714     else
715        run("no-cjit-type-profiler", "--enableTypeProfiler=true", *NO_CJIT_OPTIONS)
716     end
717 end
718
719 def runLayoutTest(kind, *options)
720     raise unless $benchmark.to_s =~ /\.js$/
721     testName = $~.pre_match
722     if kind
723         kind = "layout-" + kind
724     else
725         kind = "layout"
726     end
727
728     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
729     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
730
731     args = [pathToVM.to_s] + BASE_OPTIONS + options +
732         [(Pathname.new("resources") + "standalone-pre.js").to_s,
733          $benchmark.to_s,
734          (Pathname.new("resources") + "standalone-post.js").to_s]
735     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
736 end
737
738 def runLayoutTestDefault
739     runLayoutTest(nil)
740 end
741
742 def runLayoutTestNoLLInt
743     runLayoutTest("no-llint", "--useLLInt=false")
744 end
745
746 def runLayoutTestNoCJIT
747     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
748 end
749
750 def runLayoutTestDFGEagerNoCJIT
751     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
752 end
753
754 def runLayoutTestDefaultFTL
755     runLayoutTest("ftl", "--testTheFTL=true", *FTL_OPTIONS) if $enableFTL
756 end
757
758 def runLayoutTestFTLNoCJIT
759     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
760 end
761
762 def runLayoutTestFTLEagerNoCJIT
763     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
764 end
765
766 def noFTLRunLayoutTest
767     runLayoutTestDefault
768     runLayoutTestNoLLInt
769     runLayoutTestNoCJIT
770     runLayoutTestDFGEagerNoCJIT
771 end
772
773 def defaultRunLayoutTest
774     noFTLRunLayoutTest
775     runLayoutTestDefaultFTL
776     runLayoutTestFTLNoCJIT
777     runLayoutTestFTLEagerNoCJIT
778 end
779
780 def prepareExtraRelativeFiles(extraFiles, destination)
781     Dir.chdir($outputDir) {
782         extraFiles.each {
783             | file |
784             FileUtils.cp $extraFilesBaseDir + file, destination + file
785         }
786     }
787 end
788
789 def baseDirForCollection(collectionName)
790     Pathname(".tests") + collectionName
791 end
792
793 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
794     raise unless absoluteBase.absolute?
795     Dir.chdir($outputDir) {
796         collectionBaseDir = baseDirForCollection($collectionName)
797         extraFiles.each {
798             | file |
799             destination = collectionBaseDir + file
800             FileUtils.mkdir_p destination.dirname unless destination.directory?
801             FileUtils.cp absoluteBase + file, destination
802         }
803     }
804 end
805
806 def runMozillaTest(kind, mode, extraFiles, *options)
807     if kind
808         kind = "mozilla-" + kind
809     else
810         kind = "mozilla"
811     end
812     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
813     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
814     case mode
815     when :normal
816         errorHandler = mozillaErrorHandler
817     when :negative
818         errorHandler = mozillaExit3ErrorHandler
819     when :fail
820         errorHandler = mozillaFailErrorHandler
821     when :skip
822         return
823     else
824         raise "Invalid mode: #{mode}"
825     end
826     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
827 end
828
829 def runMozillaTestDefault(mode, *extraFiles)
830     runMozillaTest(nil, mode, extraFiles)
831 end
832
833 def runMozillaTestDefaultFTL(mode, *extraFiles)
834     runMozillaTest("ftl", mode, extraFiles, *FTL_OPTIONS) if $enableFTL
835 end
836
837 def runMozillaTestLLInt(mode, *extraFiles)
838     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
839 end
840
841 def runMozillaTestBaselineJIT(mode, *extraFiles)
842     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
843 end
844
845 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
846     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
847 end
848
849 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
850     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
851 end
852
853 def defaultRunMozillaTest(mode, *extraFiles)
854     runMozillaTestDefault(mode, *extraFiles)
855     runMozillaTestLLInt(mode, *extraFiles)
856     runMozillaTestBaselineJIT(mode, *extraFiles)
857     runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
858     runMozillaTestDefaultFTL(mode, *extraFiles)
859     runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
860 end
861
862 def skip
863     $didAddRunCommand = true
864     puts "Skipping #{$collectionName}/#{$benchmark}"
865 end
866
867 def largeHeap
868     if $memoryLimited
869         $didAddRunCommand = true
870         puts "Skipping #{$collectionName}/#{$benchmark}"
871     end
872 end
873
874 def allJSFiles(path)
875     if path.file?
876         [path]
877     else
878         result = []
879         Dir.foreach(path) {
880             | filename |
881             next unless filename =~ /\.js$/
882             next unless (path + filename).file?
883             result << path + filename
884         }
885         result
886     end
887 end
888
889 def uniqueifyName(names, name)
890     result = name.to_s
891     toAdd = 1
892     while names[result]
893         result = "#{name}-#{toAdd}"
894         toAdd += 1
895     end
896     names[result] = true
897     result
898 end
899
900 def simplifyCollectionName(collectionPath)
901     outerDir = collectionPath.dirname
902     name = collectionPath.basename
903     lastName = name
904     if collectionPath.directory?
905         while lastName.to_s =~ /test/
906             lastName = outerDir.basename
907             name = lastName + name
908             outerDir = outerDir.dirname
909         end
910     end
911     uniqueifyName($collectionNames, name)
912 end
913
914 def prepareCollection(name)
915     FileUtils.mkdir_p $outputDir + name
916
917     absoluteCollection = $collection.realpath
918
919     Dir.chdir($outputDir) {
920         bundleDir = baseDirForCollection(name)
921
922         # Create the proper directory structures.
923         FileUtils.mkdir_p bundleDir
924         if bundleDir.basename == $collection.basename
925             FileUtils.cp_r absoluteCollection, bundleDir.dirname
926         else
927             FileUtils.cp_r absoluteCollection, bundleDir
928         end
929
930         $extraFilesBaseDir = absoluteCollection
931
932         # Redirect the collection's location to the newly constructed bundle.
933         if absoluteCollection.directory?
934             $collection = bundleDir
935         else
936             $collection = bundleDir + $collection.basename
937         end
938     }
939 end
940
941 $collectionNames = {}
942
943 def handleCollectionFile(collection)
944     collectionName = simplifyCollectionName(collection)
945    
946     paths = {}
947     subCollections = []
948     YAML::load(IO::read(collection)).each {
949         | entry |
950         if entry["collection"]
951             subCollections << entry["collection"]
952             next
953         end
954         
955         if Pathname.new(entry["path"]).absolute?
956             raise "Absolute path: " + entry["path"] + " in #{collection}"
957         end
958         
959         if paths[entry["path"]]
960             raise "Duplicate path: " + entry["path"] + " in #{collection}"
961         end
962         
963         subCollection = collection.dirname + entry["path"]
964         
965         if subCollection.file?
966             subCollectionName = Pathname.new(entry["path"]).dirname
967         else
968             subCollectionName = entry["path"]
969         end
970         
971         $collection = subCollection
972         $collectionName = Pathname.new(collectionName)
973         Pathname.new(subCollectionName).each_filename {
974             | filename |
975             next if filename =~ /^\./
976             $collectionName += filename
977         }
978         $collectionName = $collectionName.to_s
979         
980         prepareCollection($collectionName)
981       
982         Dir.chdir($outputDir) {
983             directoryToSearch = $collection
984             if entry["tests"]
985                 directoryToSearch += entry["tests"]
986             end
987             allJSFiles(directoryToSearch).each {
988                 | path |
989                
990                 $benchmark = path.basename
991                 $benchmarkDirectory = path.dirname
992                 
993                 $runCommandOptions = {}
994                 eval entry["cmd"]
995             }
996         }
997     }
998     
999     subCollections.each {
1000         | subCollection |
1001         handleCollection(collection.dirname + subCollection)
1002     }
1003 end
1004
1005 def handleCollectionDirectory(collection)
1006     collectionName = simplifyCollectionName(collection)
1007     
1008     $collection = collection
1009     $collectionName = collectionName
1010     prepareCollection(collectionName)
1011    
1012     Dir.chdir($outputDir) {
1013         $benchmarkDirectory = $collection
1014         allJSFiles($collection).each {
1015             | path |
1016             
1017             $benchmark = path.basename
1018             
1019             $runCommandOptions = {}
1020             defaultRun unless parseRunCommands
1021         }
1022     }
1023 end
1024
1025 def handleCollection(collection)
1026     collection = Pathname.new(collection)
1027     
1028     if collection.file?
1029         handleCollectionFile(collection)
1030     else
1031         handleCollectionDirectory(collection)
1032     end
1033 end
1034
1035 def appendFailure(plan)
1036     File.open($outputDir + "failed", "a") {
1037         | outp |
1038         outp.puts plan.name
1039     }
1040     $numFailures += 1
1041 end
1042
1043 def prepareBundle
1044     raise if $bundle
1045
1046     if $doNotMessWithVMPath
1047         if !$remote and !$tarball
1048             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1049             $jscPath = Pathname.new($jscPath).realpath
1050         else
1051             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1052         end
1053     else
1054         originalJSCPath = $jscPath
1055         vmDir = $outputDir + ".vm"
1056         FileUtils.mkdir_p vmDir
1057         
1058         frameworkPath = frameworkFromJSCPath($jscPath)
1059         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1060         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1061         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1062
1063         if frameworkPath
1064             source = frameworkPath
1065             destination = Pathname.new(".vm")
1066         else
1067             source = originalJSCPath
1068             destination = $jscPath
1069
1070             Dir.chdir($outputDir) {
1071                 FileUtils.mkdir_p $jscPath.dirname
1072             }
1073         end
1074
1075         Dir.chdir($outputDir) {
1076             if $copyVM
1077                 FileUtils.cp_r source, destination
1078             else
1079                 begin 
1080                     FileUtils.ln_s source, destination
1081                 rescue
1082                     FileUtils.cp_r source, destination
1083                 end
1084             end
1085         }
1086     end
1087     
1088     Dir.chdir($outputDir) {
1089         FileUtils.cp_r HELPERS_PATH, ".helpers"
1090     }
1091
1092     ARGV.each {
1093         | collection |
1094         handleCollection(collection)
1095     }
1096
1097     puts
1098 end
1099
1100 def cleanOldResults
1101     raise unless $bundle
1102
1103     eachResultFile($outputDir) {
1104         | path |
1105         FileUtils.rm_f path
1106     }
1107 end
1108
1109 def cleanEmptyResultFiles
1110     eachResultFile($outputDir) {
1111         | path |
1112         next unless path.basename.to_s =~ /\.out$/
1113         next unless FileTest.size(path) == 0
1114         FileUtils.rm_f path
1115     }
1116 end
1117
1118 def eachResultFile(startingDir, &block)
1119     dirsToClean = [startingDir]
1120     until dirsToClean.empty?
1121         nextDir = dirsToClean.pop
1122         Dir.foreach(nextDir) {
1123             | entry |
1124             next if entry =~ /^\./
1125             path = nextDir + entry
1126             if path.directory?
1127                 dirsToClean.push(path)
1128             else
1129                 block.call(path)
1130             end
1131         }
1132     end
1133 end
1134
1135 def prepareTestRunner
1136     raise if $bundle
1137
1138     $runlist.each_with_index {
1139         | plan, index |
1140         plan.index = index
1141     }
1142
1143     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1144     toDelete = []
1145     Dir.foreach($runnerDir) {
1146         | filename |
1147         if filename =~ /^test_/
1148             toDelete << filename
1149         end
1150     }
1151     
1152     toDelete.each {
1153         | filename |
1154         File.unlink($runnerDir + filename)
1155     }
1156
1157     $runlist.each {
1158         | plan |
1159         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1160     }
1161
1162     case $testRunnerType
1163     when :make
1164         prepareMakeTestRunner
1165     when :shell
1166         prepareShellTestRunner
1167     else
1168         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1169     end
1170 end
1171
1172 def prepareShellTestRunner
1173     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
1174 end
1175
1176 def prepareMakeTestRunner
1177     # The goals of our parallel test runner are scalability and simplicity. The
1178     # simplicity part is particularly important. We don't want to have to have
1179     # a full-time contributor just philosophising about parallel testing.
1180     #
1181     # As such, we just pass off all of the hard work to 'make'. This creates a
1182     # dummy directory ("$outputDir/.runner") in which we create a dummy
1183     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
1184     # That is, for each test we know we will run, there is a rule in the
1185     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
1186     # Makefile results in 'make' doing all of the hard work:
1187     #
1188     # - Load balancing just works. Most systems have a great load balancer in
1189     #   'make'. If your system doesn't then just install a real 'make'.
1190     #
1191     # - Interruptions just work. For example Ctrl-C handling in 'make' is
1192     #   exactly right. You don't have to worry about zombie processes.
1193     #
1194     # We then do some tricks to make failure detection work and to make this
1195     # totally sound. If a test fails, we don't want the whole 'make' job to
1196     # stop. We also don't have any facility for makefile-escaping of path names.
1197     # We do have such a thing for shell-escaping, though. We fix both problems
1198     # by having the actual work for each of the test rules be done in a shell
1199     # script on the side. There is one such script per test. The script responds
1200     # to failure by printing something on the console and then touching a
1201     # failure file for that test, but then still returns 0. This makes 'make'
1202     # continue past that failure and complete all the tests anyway.
1203     #
1204     # In the end, this script collects all of the failures by searching for
1205     # files in the .runner directory whose name matches /^test_fail_/, where
1206     # the thing after the 'fail_' is the test index. Those are the files that
1207     # would be created by the test scripts if they detect failure. We're
1208     # basically using the filesystem as a concurrent database of test failures.
1209     # Even if two tests fail at the same time, since they're touching different
1210     # files we won't miss any failures.
1211     runIndices = []
1212     $runlist.each {
1213         | plan |
1214         runIndices << plan.index
1215     }
1216     
1217     File.open($runnerDir + "Makefile", "w") {
1218         | outp |
1219         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1220         runIndices.each {
1221             | index |
1222             plan = $runlist[index]
1223             outp.puts "test_done_#{index}:"
1224             outp.puts "\tsh test_script_#{plan.index}"
1225         }
1226     }
1227 end
1228
1229 if $enableFTL and ENV["JSC_timeout"]
1230     # Currently, using the FTL is a performance regression particularly in real
1231     # (i.e. non-loopy) benchmarks. Account for this in the timeout.
1232     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i * 2).to_s
1233 end
1234
1235 if ENV["JSC_timeout"]
1236     # In the worst case, the processors just interfere with each other.
1237     # Increase the timeout proportionally to the number of processors.
1238     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i.to_f * Math.sqrt($numProcessors)).to_i.to_s
1239 end
1240     
1241 puts
1242
1243 def cleanRunnerDirectory
1244     raise unless $bundle
1245     Dir.foreach($runnerDir) {
1246         | filename |
1247         next unless filename =~ /^test_fail/
1248         FileUtils.rm_f $runnerDir + filename
1249     }
1250 end
1251
1252 def runTestRunner
1253     case $testRunnerType
1254     when :shell
1255         runShellTestRunner
1256     when :make
1257         runMakeTestRunner
1258     else
1259         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1260     end
1261 end
1262
1263 def sshRead(cmd)
1264     raise unless $remote
1265
1266     result = ""
1267     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1268       | inp |
1269       inp.each_line {
1270         | line |
1271         result += line
1272       }
1273     }
1274     raise "#{$?}" unless $?.success?
1275     result
1276 end
1277
1278 def runAndMonitorTestRunnerCommand(numberOfTests, *cmd)
1279     unless $progressMeter
1280         mysys(cmd.join(' '))
1281     else
1282        running = {}
1283        didRun = {}
1284        didFail = {}
1285        blankLine = true
1286        prevStringLength = 0
1287        IO.popen(cmd.join(' '), mode="r") {
1288            | inp |
1289            inp.each_line {
1290                | line |
1291                line.chomp!
1292                if line =~ /^Running /
1293                    running[$~.post_match] = true
1294                elsif line =~ /^PASS: /
1295                    didRun[$~.post_match] = true
1296                elsif line =~ /^FAIL: /
1297                    didRun[$~.post_match] = true
1298                    didFail[$~.post_match] = true
1299                else
1300                    unless blankLine
1301                        print("\r" + " " * prevStringLength + "\r")
1302                    end
1303                    puts line
1304                    blankLine = true
1305                end
1306
1307                def lpad(str, chars)
1308                    str = str.to_s
1309                    if str.length > chars
1310                        str
1311                    else
1312                       "%#{chars}s"%(str)
1313                    end
1314                end
1315
1316                string  = ""
1317                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1318                unless didFail.empty?
1319                    string += " (failed #{didFail.size})"
1320                end
1321                string += " "
1322                (running.size - didRun.size).times {
1323                    string += "."
1324                }
1325                if string.length < prevStringLength
1326                    print string
1327                    print(" " * (prevStringLength - string.length))
1328                end
1329                print string
1330                prevStringLength = string.length
1331                blankLine = false
1332                $stdout.flush
1333            }
1334        }
1335        puts
1336        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1337     end
1338 end
1339
1340 def runShellTestRunner
1341     if $remote
1342         numberOfTests = 0
1343         Dir.chdir($runnerDir) {
1344             # -1 for the runscript, and -2 for '..' and '.'
1345             numberOfTests = Dir.entries(".").count - 3
1346         }
1347
1348         $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1349         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + "payload.tar.gz").to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1350         remoteScript = "\""
1351         remoteScript += "cd #{$remoteDirectory} && "
1352         remoteScript += "rm -rf #{$outputDir.basename} && "
1353         remoteScript += "tar xzf payload.tar.gz && "
1354         remoteScript += "cd #{$outputDir.basename}/.runner && "
1355         remoteScript += "DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" sh runscript"
1356         remoteScript += "\""
1357         runAndMonitorTestRunnerCommand(numberOfTests, "ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1358     else
1359         Dir.chdir($runnerDir) {
1360             # -1 for the runscript, and -2 for '..' and '.'
1361             numberOfTests = Dir.entries(".").count - 3
1362             runAndMonitorTestRunnerCommand(numberOfTests, "sh", "runscript")
1363         }
1364     end
1365 end
1366
1367 def runMakeTestRunner
1368     raise if $remote
1369     Dir.chdir($runnerDir) {
1370         # -1 for the Makefile, and -2 for '..' and '.'
1371         numberOfTests = Dir.entries(".").count - 3
1372         runAndMonitorTestRunnerCommand(numberOfTests, "make", "-j", $numProcessors.to_s, "-s", "-f", "Makefile")
1373     }
1374 end
1375
1376 def detectFailures
1377     raise if $bundle
1378
1379     if $remote
1380         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -name \"test_fail_*\" -depth 1")
1381         output.split(/\n/).each {
1382             | line |
1383             next unless line =~ /test_fail_/
1384             appendFailure($runlist[$~.post_match.to_i])
1385         }
1386     else
1387         Dir.foreach($runnerDir) {
1388             | filename |
1389             next unless filename =~ /test_fail_/
1390             appendFailure($runlist[$~.post_match.to_i])
1391         }
1392     end
1393 end
1394
1395 def compressBundle
1396     cmd = "cd #{$outputDir}/.. && tar -czf payload.tar.gz #{$outputDir.basename}"
1397     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1398     raise unless system(cmd)
1399 end
1400
1401 def clean(file)
1402     FileUtils.rm_rf file unless $bundle
1403 end
1404
1405 clean($outputDir + "failed")
1406 clean($outputDir + ".vm")
1407 clean($outputDir + ".helpers")
1408 clean($outputDir + ".runner")
1409 clean($outputDir + ".tests")
1410 clean($outputDir + "_payload")
1411
1412 Dir.mkdir($outputDir) unless $outputDir.directory?
1413
1414 $outputDir = $outputDir.realpath
1415 $runnerDir = $outputDir + ".runner"
1416
1417 def runBundle
1418     raise unless $bundle
1419
1420     cleanRunnerDirectory
1421     cleanOldResults
1422     runTestRunner
1423     cleanEmptyResultFiles
1424 end
1425
1426 def runNormal
1427     raise if $bundle or $tarball
1428
1429     prepareBundle
1430     prepareTestRunner
1431     runTestRunner
1432     cleanEmptyResultFiles
1433     detectFailures
1434 end
1435
1436 def runTarball
1437     raise unless $tarball
1438
1439     prepareBundle 
1440     prepareTestRunner
1441     compressBundle
1442 end
1443
1444 def runRemote
1445     raise unless $remote
1446
1447     prepareBundle
1448     prepareTestRunner
1449     compressBundle
1450     runTestRunner
1451     detectFailures
1452 end
1453
1454 if $bundle
1455     runBundle
1456 elsif $remote
1457     runRemote
1458 elsif $tarball
1459     runTarball
1460 else
1461     runNormal
1462 end