83f88ac66384c195c889d4df4d20b3578a825dc3
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'uri'
30 require 'yaml'
31
32 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
33 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
34 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
35 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
36 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
37 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
38
39 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
40
41 IMPORTANT_ENVS = ["JSC_timeout"]
42
43 begin
44     require 'shellwords'
45 rescue Exception => e
46     $stderr.puts "Warning: did not find shellwords, not running any tests."
47     exit 0
48 end
49
50 $canRunDisplayProfilerOutput = false
51
52 begin
53     require 'rubygems'
54     require 'json'
55     require 'highline'
56     $canRunDisplayProfilerOutput = true
57 rescue Exception => e
58     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
59     $stderr.puts "Error: #{e.inspect}"
60 end
61
62 def printCommandArray(*cmd)
63     begin
64         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
65     rescue
66         commandArray = cmd.join(' ')
67     end
68     $stderr.puts ">> #{commandArray}"
69 end
70
71 def mysys(*cmd)
72     printCommandArray(*cmd) if $verbosity >= 1
73     raise "Command failed: #{$?.inspect}" unless system(*cmd)
74 end
75
76 begin
77     $numProcessors = `sysctl -n hw.activecpu`.to_i
78 rescue
79     $numProcessors = 0
80 end
81
82 if $numProcessors == 0
83     $numProcessors = `nproc --all 2>/dev/null`.to_i
84 end
85 if $numProcessors == 0
86     $numProcessors = 1
87 end
88
89 $jscPath = nil
90 $enableFTL = false
91 $outputDir = Pathname.new("results")
92 $verbosity = 0
93 $bundle = nil
94 $tarball = false
95 $copyVM = false
96 $testRunnerType = :make
97 $remoteUser = nil
98 $remoteHost = nil
99 $remotePort = nil
100 $remoteDirectory = nil
101
102 def usage
103     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
104     puts
105     puts "--jsc                (-j)   Path to JavaScriptCore. This option is required."
106     puts "--ftl-jit                   Indicate that we have the FTL JIT."
107     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
108     puts "--verbose            (-v)   Print more things while running."
109     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
110     puts "--tarball                   Creates a tarball of the final bundle."
111     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
112     puts "                            In general the shell runner is slower than the make runner."
113     puts "--remote                    Specify a remote host on which to run tests."
114     puts "--help               (-h)   Print this message."
115     exit 1
116 end
117
118 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
119                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
120                ['--ftl-jit', GetoptLong::NO_ARGUMENT],
121                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
122                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
123                ['--tarball', GetoptLong::NO_ARGUMENT],
124                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
125                ['--shell-runner', GetoptLong::NO_ARGUMENT],
126                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
127                ['--verbose', '-v', GetoptLong::NO_ARGUMENT]).each {
128     | opt, arg |
129     case opt
130     when '--help'
131         usage
132     when '--jsc'
133         $jscPath = Pathname.new(arg).realpath
134     when '--output-dir'
135         $outputDir = Pathname.new(arg)
136     when '--ftl-jit'
137         $enableFTL = true
138     when '--verbose'
139         $verbosity += 1
140     when '--run-bundle'
141         $bundle = Pathname.new(arg)
142     when '--tarball'
143         $tarball = true
144         $copyVM = true
145     when '--force-vm-copy'
146         $copyVM = true
147     when '--shell-runner'
148         $testRunnerType = :shell
149     when '--remote'
150         $copyVM = true
151         $testRunnerType = :shell
152         $tarball = true
153         $remote = true
154         uri = URI("ftp://" + arg)
155         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
156     end
157 }
158
159 $progressMeter = ($verbosity == 0 and $stdin.tty?)
160
161 if $bundle
162     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
163     $outputDir = $bundle
164 end
165
166 unless $jscPath
167     $stderr.puts "Error: must specify -j <path>."
168     exit 1
169 end
170
171 $numFailures = 0
172
173 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20"]
174
175 $runlist = []
176
177 def frameworkFromJSCPath(jscPath)
178     parentDirectory = jscPath.dirname
179     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
180         parentDirectory.dirname
181     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
182         jscPath.dirname + "JavaScriptCore.framework"
183     else
184         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
185         nil
186     end
187 end
188
189 def prepareFramework(jscPath)
190     frameworkPath = frameworkFromJSCPath(jscPath)
191     $frameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
192     $jscPath = $frameworkPath + "Resources" + "jsc"
193
194     if frameworkPath
195         source = frameworkPath
196         destination = Pathname.new(".vm")
197     else
198         source = jscPath
199         destination = $jscPath
200
201         Dir.chdir($outputDir) {
202             FileUtils.mkdir_p $jscPath.dirname
203         }
204     end
205
206     Dir.chdir($outputDir) {
207         if $copyVM
208             FileUtils.cp_r source, destination
209         else
210             begin 
211                 FileUtils.ln_s source, destination
212             rescue
213                 FileUtils.cp_r source, destination
214             end
215         end
216     }
217 end
218
219 def copyVMToBundle
220     raise if $bundle
221
222     vmDir = $outputDir + ".vm"
223     FileUtils.mkdir_p vmDir
224    
225     prepareFramework($jscPath) 
226 end
227
228 def pathToVM
229     dir = Pathname.new(".")
230     $benchmarkDirectory.each_filename {
231         | pathComponent |
232         dir += ".."
233     }
234     dir + $jscPath
235 end
236
237 def prefixCommand(prefix)
238     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
239 end
240
241 def pipeAndPrefixCommand(outputFilename, prefix)
242     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
243 end
244
245 # Output handler for tests that are expected to be silent.
246 def silentOutputHandler
247     Proc.new {
248         | name |
249         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
250     }
251 end
252
253 # Output handler for tests that are expected to produce meaningful output.
254 def noisyOutputHandler
255     Proc.new {
256         | name |
257         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
258     }
259 end
260
261 # Error handler for tests that fail exactly when they return non-zero exit status.
262 def simpleErrorHandler
263     Proc.new {
264         | outp, plan |
265         outp.puts "if test -e #{plan.failFile}"
266         outp.puts "then"
267         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + prefixCommand(plan.name)
268         outp.puts "    " + plan.failCommand
269         outp.puts "else"
270         outp.puts "    " + plan.successCommand
271         outp.puts "fi"
272     }
273 end
274
275 # Error handler for tests that diff their output with some expectation.
276 def diffErrorHandler(expectedFilename)
277     Proc.new {
278         | outp, plan |
279         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
280         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
281         
282         outp.puts "if test -e #{plan.failFile}"
283         outp.puts "then"
284         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + prefixCommand(plan.name)
285         outp.puts "    " + plan.failCommand
286         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
287         outp.puts "then"
288         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
289         outp.puts "    if [ $? -eq 0 ]"
290         outp.puts "    then"
291         outp.puts "    " + plan.successCommand
292         outp.puts "    else"
293         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + prefixCommand(plan.name)
294         outp.puts "        " + plan.failCommand
295         outp.puts "    fi"
296         outp.puts "else"
297         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + prefixCommand(plan.name)
298         outp.puts "    " + plan.failCommand
299         outp.puts "fi"
300     }
301 end
302
303 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
304 # tests.
305 def mozillaErrorHandler
306     Proc.new {
307         | outp, plan |
308         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
309
310         outp.puts "if test -e #{plan.failFile}"
311         outp.puts "then"
312         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + prefixCommand(plan.name)
313         outp.puts "    " + plan.failCommand
314         outp.puts "elif grep -i -q failed! #{outputFilename}"
315         outp.puts "then"
316         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + prefixCommand(plan.name)
317         outp.puts "    " + plan.failCommand
318         outp.puts "else"
319         outp.puts "    " + plan.successCommand
320         outp.puts "fi"
321     }
322 end
323
324 # Error handler for tests that report error by saying "failed!", and are expected to
325 # fail. This is used by Mozilla tests.
326 def mozillaFailErrorHandler
327     Proc.new {
328         | outp, plan |
329         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
330
331         outp.puts "if test -e #{plan.failFile}"
332         outp.puts "then"
333         outp.puts "    " + plan.successCommand
334         outp.puts "elif grep -i -q failed! #{outputFilename}"
335         outp.puts "then"
336         outp.puts "    " + plan.successCommand
337         outp.puts "else"
338         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + prefixCommand(plan.name)
339         outp.puts "    " + plan.failCommand
340         outp.puts "fi"
341     }
342 end
343
344 # Error handler for tests that report error by saying "failed!", and are expected to have
345 # an exit code of 3.
346 def mozillaExit3ErrorHandler
347     Proc.new {
348         | outp, plan |
349         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
350
351         outp.puts "if test -e #{plan.failFile}"
352         outp.puts "then"
353         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
354         outp.puts "    then"
355         outp.puts "        if grep -i -q failed! #{outputFilename}"
356         outp.puts "        then"
357         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + prefixCommand(plan.name)
358         outp.puts "            " + plan.failCommand
359         outp.puts "        else"
360         outp.puts "            " + plan.successCommand
361         outp.puts "        fi"
362         outp.puts "    else"
363         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + prefixCommand(plan.name)
364         outp.puts "        " + plan.failCommand
365         outp.puts "    fi"
366         outp.puts "else"
367         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + prefixCommand(plan.name)
368         outp.puts "    " + plan.failCommand
369         outp.puts "fi"
370     }
371 end
372
373 $runCommandOptions = {}
374
375 class Plan
376     attr_reader :directory, :arguments, :name, :outputHandler, :errorHandler
377     attr_accessor :index
378     
379     def initialize(directory, arguments, name, outputHandler, errorHandler)
380         @directory = directory
381         @arguments = arguments
382         @name = name
383         @outputHandler = outputHandler
384         @errorHandler = errorHandler
385         @isSlow = !!$runCommandOptions[:isSlow]
386     end
387     
388     def shellCommand
389         # It's important to remember that the test is actually run in a subshell, so if we change directory
390         # in the subshell when we return we will be in our original directory. This is nice because we don't
391         # have to bend over backwards to do things relative to the root.
392         "(cd ../#{Shellwords.shellescape(@directory.to_s)} && \"$@\" " + @arguments.map{
393             | v |
394             raise "Detected a non-string in #{inspect}" unless v.is_a? String
395             Shellwords.shellescape(v)
396         }.join(' ') + ")"
397     end
398     
399     def reproScriptCommand
400         # We have to find our way back to the .runner directory since that's where all of the relative
401         # paths assume they start out from.
402         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
403         script += "cd $CURRENT_DIR\n"
404         Pathname.new(@name).dirname.each_filename {
405             | pathComponent |
406             script += "cd ..\n"
407         }
408         script += "cd .runner\n"
409
410         script += "export DYLD_FRAMEWORK_PATH=$(cd ../#{$frameworkPath.dirname}; pwd)\n"
411         IMPORTANT_ENVS.each {
412             | key |
413             if ENV[key]
414                 script += "export #{key}=#{Shellwords.shellescape(ENV[key])}\n"
415             end
416         }
417         script += "#{shellCommand} || exit 1"
418         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
419     end
420     
421     def failCommand
422         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
423     end
424     
425     def successCommand
426         if $progressMeter or $verbosity >= 2
427             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
428         else
429             "rm -f #{failFile}"
430         end
431     end
432     
433     def failFile
434         "test_fail_#{@index}"
435     end
436     
437     def writeRunScript(filename)
438         File.open(filename, "w") {
439             | outp |
440             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
441             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
442             cmd += @outputHandler.call(@name)
443             if $verbosity >= 3
444                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
445             end
446             outp.puts cmd
447             @errorHandler.call(outp, self)
448         }
449     end
450 end
451
452 $uniqueFilenameCounter = 0
453 def uniqueFilename(extension)
454     payloadDir = $outputDir + "_payload"
455     Dir.mkdir payloadDir unless payloadDir.directory?
456     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
457     $uniqueFilenameCounter += 1
458     result
459 end
460
461 def baseOutputName(kind)
462     "#{$collectionName}/#{$benchmark}.#{kind}"
463 end
464
465 def addRunCommand(kind, command, outputHandler, errorHandler)
466     plan = Plan.new($benchmarkDirectory, command, baseOutputName(kind), outputHandler, errorHandler)
467     if $numProcessors > 1 and $runCommandOptions[:isSlow]
468         $runlist.unshift plan
469     else
470         $runlist << plan
471     end
472 end
473
474 # Returns true if there were run commands found in the file ($benchmarkDirectory +
475 # $benchmark), in which case those run commands have already been executed. Otherwise
476 # returns false, in which case you're supposed to add your own run commands.
477 def parseRunCommands
478     didRun = false
479
480     Dir.chdir($outputDir) {
481         File.open($benchmarkDirectory + $benchmark) {
482             | inp |
483             inp.each_line {
484                 | line |
485                 begin
486                     doesMatch = line =~ /^\/\/@/
487                 rescue Exception => e
488                     # Apparently this happens in the case of some UTF8 stuff in some files, where
489                     # Ruby tries to be strict and throw exceptions.
490                     next
491                 end
492                 next unless doesMatch
493                 eval $~.post_match
494                 didRun = true
495             }
496         }
497     }
498
499     didRun
500 end
501
502 def slow!
503     $runCommandOptions[:isSlow] = true
504 end
505
506 def run(kind, *options)
507     addRunCommand(kind, [pathToVM.to_s] + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
508 end
509
510 def runDefault
511     run("default")
512 end
513
514 def runNoLLInt
515     run("no-llint", "--useLLInt=false")
516 end
517
518 def runNoCJITValidate
519     run("no-cjit", "--enableConcurrentJIT=false", "--validateBytecode=true", "--validateGraph=true")
520 end
521
522 def runNoCJITValidatePhases
523     run("no-cjit-validate-phases", "--enableConcurrentJIT=false", "--validateBytecode=true", "--validateGraphAtEachPhase=true")
524 end
525
526 def runDefaultFTL
527     run("default-ftl", "--useExperimentalFTL=true")
528 end
529
530 def runFTLNoCJIT
531     run("ftl-no-cjit", "--enableConcurrentJIT=false", "--useExperimentalFTL=true")
532 end
533
534 def runFTLNoCJITValidate
535     run("ftl-no-cjit-validate", "--enableConcurrentJIT=false", "--useExperimentalFTL=true", "--validateGraph=true")
536 end
537
538 def runFTLNoCJITOSRValidation
539     run("ftl-no-cjit-osr-validation", "--enableConcurrentJIT=false", "--useExperimentalFTL=true", "--validateFTLOSRExitLiveness=true")
540 end
541
542 def runDFGEager
543     run("dfg-eager", *EAGER_OPTIONS)
544 end
545
546 def runDFGEagerNoCJITValidate
547     run("dfg-eager-no-cjit-validate", "--enableConcurrentJIT=false", "--validateGraph=true", *EAGER_OPTIONS)
548 end
549
550 def runFTLEager
551     run("ftl-eager", "--useExperimentalFTL=true", *EAGER_OPTIONS)
552 end
553
554 def runFTLEagerNoCJITValidate
555     run("ftl-eager-no-cjit", "--useExperimentalFTL=true", "--enableConcurrentJIT=false", "--validateGraph=true", *EAGER_OPTIONS)
556 end
557
558 def runFTLEagerNoCJITOSRValidation
559     run("ftl-eager-no-cjit-osr-validation", "--useExperimentalFTL=true", "--enableConcurrentJIT=false", "--validateFTLOSRExitLiveness=true", *EAGER_OPTIONS)
560 end
561
562 def runAlwaysTriggerCopyPhase
563     run("always-trigger-copy-phase", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
564 end
565
566 def defaultRun
567     runDefault
568     runNoLLInt
569     runAlwaysTriggerCopyPhase
570     runNoCJITValidatePhases
571     runDFGEager
572     runDFGEagerNoCJITValidate
573     if $enableFTL
574         runDefaultFTL
575         runFTLNoCJITValidate
576         runFTLNoCJITOSRValidation
577         runFTLEager
578         runFTLEagerNoCJITValidate
579         runFTLEagerNoCJITOSRValidation
580     end
581 end
582
583 def defaultQuickRun
584     runDefault
585     runNoCJITValidate
586     if $enableFTL
587         runDefaultFTL
588         runFTLNoCJIT
589     end
590 end
591
592 def runProfiler
593     profilerOutput = uniqueFilename(".json")
594     if $canRunDisplayProfilerOutput
595         addRunCommand("profiler", ["ruby", (HELPERS_PATH + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
596     else
597         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
598         run("profiler-simple", "-p", profilerOutput.to_s)
599     end
600 end
601
602 def runLayoutTest(kind, *options)
603     raise unless $benchmark.to_s =~ /\.js$/
604     testName = $~.pre_match
605     if kind
606         kind = "layout-" + kind
607     else
608         kind = "layout"
609     end
610
611     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
612     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
613
614     args = [pathToVM.to_s] + options +
615         [(Pathname.new("resources") + "standalone-pre.js").to_s,
616          $benchmark.to_s,
617          (Pathname.new("resources") + "standalone-post.js").to_s]
618     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
619 end
620
621 def runLayoutTestDefault
622     runLayoutTest(nil)
623 end
624
625 def runLayoutTestNoLLInt
626     runLayoutTest("no-llint", "--useLLInt=false")
627 end
628
629 def runLayoutTestNoCJIT
630     runLayoutTest("no-cjit", "--enableConcurrentJIT=false")
631 end
632
633 def runLayoutTestDFGEagerNoCJIT
634     runLayoutTest("dfg-eager-no-cjit", "--enableConcurrentJIT=false", *EAGER_OPTIONS)
635 end
636
637 def defaultRunLayoutTest
638     runLayoutTestDefault
639     runLayoutTestNoLLInt
640     runLayoutTestNoCJIT
641     runLayoutTestDFGEagerNoCJIT
642 end
643
644 def prepareExtraRelativeFiles(extraFiles, destination)
645     Dir.chdir($outputDir) {
646         extraFiles.each {
647             | file |
648             FileUtils.cp $extraFilesBaseDir + file, destination + file
649         }
650     }
651 end
652
653 def baseDirForCollection(collectionName)
654     Pathname(".tests") + collectionName
655 end
656
657 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
658     raise unless absoluteBase.absolute?
659     Dir.chdir($outputDir) {
660         collectionBaseDir = baseDirForCollection($collectionName)
661         extraFiles.each {
662             | file |
663             destination = collectionBaseDir + file
664             FileUtils.mkdir_p destination.dirname unless destination.directory?
665             FileUtils.cp absoluteBase + file, destination
666         }
667     }
668 end
669
670 def runMozillaTest(kind, mode, extraFiles, *options)
671     if kind
672         kind = "mozilla-" + kind
673     else
674         kind = "mozilla"
675     end
676     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
677     args = [pathToVM.to_s] + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
678     case mode
679     when :normal
680         errorHandler = mozillaErrorHandler
681     when :negative
682         errorHandler = mozillaExit3ErrorHandler
683     when :fail
684         errorHandler = mozillaFailErrorHandler
685     when :skip
686         return
687     else
688         raise "Invalid mode: #{mode}"
689     end
690     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
691 end
692
693 def runMozillaTestDefault(mode, *extraFiles)
694     runMozillaTest(nil, mode, extraFiles)
695 end
696
697 def runMozillaTestLLInt(mode, *extraFiles)
698     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
699 end
700
701 def runMozillaTestBaselineJIT(mode, *extraFiles)
702     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
703 end
704
705 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
706     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--enableConcurrentJIT=false", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *EAGER_OPTIONS)
707 end
708
709 def defaultRunMozillaTest(mode, *extraFiles)
710     runMozillaTestDefault(mode, *extraFiles)
711     runMozillaTestLLInt(mode, *extraFiles)
712     runMozillaTestBaselineJIT(mode, *extraFiles)
713     runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
714 end
715
716 def skip
717     puts "Skipping #{$collectionName}/#{$benchmark}"
718 end
719
720 def allJSFiles(path)
721     if path.file?
722         [path]
723     else
724         result = []
725         Dir.foreach(path) {
726             | filename |
727             next unless filename =~ /\.js$/
728             next unless (path + filename).file?
729             result << path + filename
730         }
731         result
732     end
733 end
734
735 def uniqueifyName(names, name)
736     result = name.to_s
737     toAdd = 1
738     while names[result]
739         result = "#{name}-#{toAdd}"
740         toAdd += 1
741     end
742     names[result] = true
743     result
744 end
745
746 def simplifyCollectionName(collectionPath)
747     outerDir = collectionPath.dirname
748     name = collectionPath.basename
749     lastName = name
750     if collectionPath.directory?
751         while lastName.to_s =~ /test/
752             lastName = outerDir.basename
753             name = lastName + name
754             outerDir = outerDir.dirname
755         end
756     end
757     uniqueifyName($collectionNames, name)
758 end
759
760 def prepareCollection(name)
761     FileUtils.mkdir_p $outputDir + name
762
763     absoluteCollection = $collection.realpath
764
765     Dir.chdir($outputDir) {
766         bundleDir = baseDirForCollection(name)
767
768         # Create the proper directory structures.
769         FileUtils.mkdir_p bundleDir
770         if bundleDir.basename == $collection.basename
771             FileUtils.cp_r absoluteCollection, bundleDir.dirname
772         else
773             FileUtils.cp_r absoluteCollection, bundleDir
774         end
775
776         $extraFilesBaseDir = absoluteCollection
777
778         # Redirect the collection's location to the newly constructed bundle.
779         if absoluteCollection.directory?
780             $collection = bundleDir
781         else
782             $collection = bundleDir + $collection.basename
783         end
784     }
785 end
786
787 $collectionNames = {}
788
789 def handleCollectionFile(collection)
790     collectionName = simplifyCollectionName(collection)
791    
792     paths = {}
793     subCollections = []
794     YAML::load(IO::read(collection)).each {
795         | entry |
796         if entry["collection"]
797             subCollections << entry["collection"]
798             next
799         end
800         
801         if Pathname.new(entry["path"]).absolute?
802             raise "Absolute path: " + entry["path"] + " in #{collection}"
803         end
804         
805         if paths[entry["path"]]
806             raise "Duplicate path: " + entry["path"] + " in #{collection}"
807         end
808         
809         subCollection = collection.dirname + entry["path"]
810         
811         if subCollection.file?
812             subCollectionName = Pathname.new(entry["path"]).dirname
813         else
814             subCollectionName = entry["path"]
815         end
816         
817         $collection = subCollection
818         $collectionName = Pathname.new(collectionName)
819         Pathname.new(subCollectionName).each_filename {
820             | filename |
821             next if filename =~ /^\./
822             $collectionName += filename
823         }
824         $collectionName = $collectionName.to_s
825         
826         prepareCollection($collectionName)
827       
828         Dir.chdir($outputDir) {
829             directoryToSearch = $collection
830             if entry["tests"]
831                 directoryToSearch += entry["tests"]
832             end
833             allJSFiles(directoryToSearch).each {
834                 | path |
835                
836                 $benchmark = path.basename
837                 $benchmarkDirectory = path.dirname
838                 
839                 $runCommandOptions = {}
840                 eval entry["cmd"]
841             }
842         }
843     }
844     
845     subCollections.each {
846         | subCollection |
847         handleCollection(collection.dirname + subCollection)
848     }
849 end
850
851 def handleCollectionDirectory(collection)
852     collectionName = simplifyCollectionName(collection)
853     
854     $collection = collection
855     $collectionName = collectionName
856     prepareCollection(collectionName)
857    
858     Dir.chdir($outputDir) {
859         $benchmarkDirectory = $collection
860         allJSFiles($collection).each {
861             | path |
862             
863             $benchmark = path.basename
864             
865             $runCommandOptions = {}
866             defaultRun unless parseRunCommands
867         }
868     }
869 end
870
871 def handleCollection(collection)
872     collection = Pathname.new(collection)
873     
874     if collection.file?
875         handleCollectionFile(collection)
876     else
877         handleCollectionDirectory(collection)
878     end
879 end
880
881 def appendFailure(plan)
882     File.open($outputDir + "failed", "a") {
883         | outp |
884         outp.puts plan.name
885     }
886     $numFailures += 1
887 end
888
889 def prepareBundle
890     raise if $bundle
891
892     copyVMToBundle
893
894     ARGV.each {
895         | collection |
896         handleCollection(collection)
897     }
898
899     puts
900 end
901
902 def cleanOldResults
903     raise unless $bundle
904
905     eachResultFile($outputDir) {
906         | path |
907         FileUtils.rm_f path
908     }
909 end
910
911 def cleanEmptyResultFiles
912     eachResultFile($outputDir) {
913         | path |
914         next unless path.basename.to_s =~ /\.out$/
915         next unless FileTest.size(path) == 0
916         FileUtils.rm_f path
917     }
918 end
919
920 def eachResultFile(startingDir, &block)
921     dirsToClean = [startingDir]
922     until dirsToClean.empty?
923         nextDir = dirsToClean.pop
924         Dir.foreach(nextDir) {
925             | entry |
926             next if entry =~ /^\./
927             path = nextDir + entry
928             if path.directory?
929                 dirsToClean.push(path)
930             else
931                 block.call(path)
932             end
933         }
934     end
935 end
936
937 def prepareTestRunner
938     raise if $bundle
939
940     $runlist.each_with_index {
941         | plan, index |
942         plan.index = index
943     }
944
945     Dir.mkdir($runnerDir) unless $runnerDir.directory?
946     toDelete = []
947     Dir.foreach($runnerDir) {
948         | filename |
949         if filename =~ /^test_/
950             toDelete << filename
951         end
952     }
953     
954     toDelete.each {
955         | filename |
956         File.unlink($runnerDir + filename)
957     }
958
959     $runlist.each {
960         | plan |
961         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
962     }
963
964     case $testRunnerType
965     when :make
966         prepareMakeTestRunner
967     when :shell
968         prepareShellTestRunner
969     else
970         raise "Unknown test runner type: #{$testRunnerType.to_s}"
971     end
972 end
973
974 def prepareShellTestRunner
975     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
976 end
977
978 def prepareMakeTestRunner
979     # The goals of our parallel test runner are scalability and simplicity. The
980     # simplicity part is particularly important. We don't want to have to have
981     # a full-time contributor just philosophising about parallel testing.
982     #
983     # As such, we just pass off all of the hard work to 'make'. This creates a
984     # dummy directory ("$outputDir/.runner") in which we create a dummy
985     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
986     # That is, for each test we know we will run, there is a rule in the
987     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
988     # Makefile results in 'make' doing all of the hard work:
989     #
990     # - Load balancing just works. Most systems have a great load balancer in
991     #   'make'. If your system doesn't then just install a real 'make'.
992     #
993     # - Interruptions just work. For example Ctrl-C handling in 'make' is
994     #   exactly right. You don't have to worry about zombie processes.
995     #
996     # We then do some tricks to make failure detection work and to make this
997     # totally sound. If a test fails, we don't want the whole 'make' job to
998     # stop. We also don't have any facility for makefile-escaping of path names.
999     # We do have such a thing for shell-escaping, though. We fix both problems
1000     # by having the actual work for each of the test rules be done in a shell
1001     # script on the side. There is one such script per test. The script responds
1002     # to failure by printing something on the console and then touching a
1003     # failure file for that test, but then still returns 0. This makes 'make'
1004     # continue past that failure and complete all the tests anyway.
1005     #
1006     # In the end, this script collects all of the failures by searching for
1007     # files in the .runner directory whose name matches /^test_fail_/, where
1008     # the thing after the 'fail_' is the test index. Those are the files that
1009     # would be created by the test scripts if they detect failure. We're
1010     # basically using the filesystem as a concurrent database of test failures.
1011     # Even if two tests fail at the same time, since they're touching different
1012     # files we won't miss any failures.
1013     runIndices = []
1014     $runlist.each {
1015         | plan |
1016         runIndices << plan.index
1017     }
1018     
1019     File.open($runnerDir + "Makefile", "w") {
1020         | outp |
1021         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1022         runIndices.each {
1023             | index |
1024             plan = $runlist[index]
1025             outp.puts "test_done_#{index}:"
1026             outp.puts "\tsh test_script_#{plan.index}"
1027         }
1028     }
1029 end
1030
1031 if $enableFTL and ENV["JSC_timeout"]
1032     # Currently, using the FTL is a performance regression particularly in real
1033     # (i.e. non-loopy) benchmarks. Account for this in the timeout.
1034     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i * 2).to_s
1035 end
1036
1037 if ENV["JSC_timeout"]
1038     # In the worst case, the processors just interfere with each other.
1039     # Increase the timeout proportionally to the number of processors.
1040     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i.to_f * Math.sqrt($numProcessors)).to_i.to_s
1041 end
1042     
1043 puts
1044
1045 def cleanRunnerDirectory
1046     raise unless $bundle
1047     Dir.foreach($runnerDir) {
1048         | filename |
1049         next unless filename =~ /^test_fail/
1050         FileUtils.rm_f $runnerDir + filename
1051     }
1052 end
1053
1054 def runTestRunner
1055     case $testRunnerType
1056     when :shell
1057         runShellTestRunner
1058     when :make
1059         runMakeTestRunner
1060     else
1061         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1062     end
1063 end
1064
1065 def sshRead(cmd)
1066     raise unless $remote
1067
1068     result = ""
1069     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1070       | inp |
1071       inp.each_line {
1072         | line |
1073         result += line
1074       }
1075     }
1076     raise "#{$?}" unless $?.success?
1077     result
1078 end
1079
1080 def runShellTestRunner
1081     if $remote
1082         $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1083         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + "payload.tar.gz").to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1084         remoteScript = ""
1085         remoteScript += "cd #{$remoteDirectory} && "
1086         remoteScript += "rm -rf #{$outputDir.basename} && "
1087         remoteScript += "tar xzf payload.tar.gz && "
1088         remoteScript += "cd #{$outputDir.basename}/.runner && "
1089         remoteScript += "DYLD_FRAMEWORK_PATH=$(cd ../#{$frameworkPath.dirname}; pwd) sh runscript" 
1090         system("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1091     else
1092         Dir.chdir($runnerDir) {
1093             mysys("sh", "runscript")
1094         }
1095     end
1096 end
1097
1098 def runMakeTestRunner
1099     raise if $remote
1100     Dir.chdir($runnerDir) {
1101         # -1 for the Makefile, and -2 for '..' and '.'
1102         numberOfTests = Dir.entries(".").count - 3
1103         unless $progressMeter
1104             mysys("make", "-j", $numProcessors.to_s, "-s", "-f", "Makefile")
1105         else
1106             cmd = "make -j #{$numProcessors} -s -f Makefile"
1107             running = {}
1108             didRun = {}
1109             didFail = {}
1110             blankLine = true
1111             prevStringLength = 0
1112             IO.popen(cmd, "r") {
1113                 | inp |
1114                 inp.each_line {
1115                     | line |
1116                     line.chomp!
1117                     if line =~ /^Running /
1118                         running[$~.post_match] = true
1119                     elsif line =~ /^PASS: /
1120                         didRun[$~.post_match] = true
1121                     elsif line =~ /^FAIL: /
1122                         didRun[$~.post_match] = true
1123                         didFail[$~.post_match] = true
1124                     else
1125                         unless blankLine
1126                             print("\r" + " " * prevStringLength + "\r")
1127                         end
1128                         puts line
1129                         blankLine = true
1130                     end
1131                     
1132                     def lpad(str, chars)
1133                         str = str.to_s
1134                         if str.length > chars
1135                             str
1136                         else
1137                             "%#{chars}s"%(str)
1138                         end
1139                     end
1140     
1141                     string  = ""
1142                     string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1143                     unless didFail.empty?
1144                         string += " (failed #{didFail.size})"
1145                     end
1146                     string += " "
1147                     (running.size - didRun.size).times {
1148                         string += "."
1149                     }
1150                     if string.length < prevStringLength
1151                         print string
1152                         print(" " * (prevStringLength - string.length))
1153                     end
1154                     print string
1155                     prevStringLength = string.length
1156                     blankLine = false
1157                     $stdout.flush
1158                 }
1159             }
1160             puts
1161             raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1162         end
1163     }
1164 end
1165
1166 def detectFailures
1167     raise if $bundle
1168
1169     if $remote
1170         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && (ls test_fail_* 2> /dev/null || true)")
1171         output.split(/\n/).each {
1172             | line |
1173             next unless line =~ /test_fail_/
1174             appendFailure($runlist[$~.post_match.to_i])
1175         }
1176     else
1177         Dir.foreach($runnerDir) {
1178             | filename |
1179             next unless filename =~ /test_fail_/
1180             appendFailure($runlist[$~.post_match.to_i])
1181         }
1182     end
1183 end
1184
1185 def compressBundle
1186     cmd = "cd #{$outputDir}/.. && tar -czf payload.tar.gz #{$outputDir.basename}"
1187     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1188     raise unless system(cmd)
1189 end
1190
1191 def clean(file)
1192     FileUtils.rm_rf file unless $bundle
1193 end
1194
1195 clean($outputDir + "failed")
1196 clean($outputDir + ".vm")
1197 clean($outputDir + ".runner")
1198 clean($outputDir + ".tests")
1199 clean($outputDir + "_payload")
1200
1201 Dir.mkdir($outputDir) unless $outputDir.directory?
1202
1203 $outputDir = $outputDir.realpath
1204 $runnerDir = $outputDir + ".runner"
1205
1206 def runBundle
1207     raise unless $bundle
1208
1209     cleanRunnerDirectory
1210     cleanOldResults
1211     runTestRunner
1212     cleanEmptyResultFiles
1213 end
1214
1215 def runNormal
1216     raise if $bundle or $tarball
1217
1218     prepareBundle
1219     prepareTestRunner
1220     runTestRunner
1221     cleanEmptyResultFiles
1222     detectFailures
1223 end
1224
1225 def runTarball
1226     raise unless $tarball
1227
1228     prepareBundle 
1229     prepareTestRunner
1230     compressBundle
1231 end
1232
1233 def runRemote
1234     raise unless $remote
1235
1236     prepareBundle
1237     prepareTestRunner
1238     compressBundle
1239     runTestRunner
1240     detectFailures
1241 end
1242
1243 if $bundle
1244     runBundle
1245 elsif $remote
1246     runRemote
1247 elsif $tarball
1248     runTarball
1249 else
1250     runNormal
1251 end