Get rid of forward exit on DoubleAsInt32
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'uri'
30 require 'yaml'
31
32 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
33 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
34 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
35 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
36 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
37 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
38
39 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
40
41 IMPORTANT_ENVS = ["JSC_timeout"]
42
43 begin
44     require 'shellwords'
45 rescue Exception => e
46     $stderr.puts "Warning: did not find shellwords, not running any tests."
47     exit 0
48 end
49
50 $canRunDisplayProfilerOutput = false
51
52 begin
53     require 'rubygems'
54     require 'json'
55     require 'highline'
56     $canRunDisplayProfilerOutput = true
57 rescue Exception => e
58     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
59     $stderr.puts "Error: #{e.inspect}"
60 end
61
62 def printCommandArray(*cmd)
63     begin
64         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
65     rescue
66         commandArray = cmd.join(' ')
67     end
68     $stderr.puts ">> #{commandArray}"
69 end
70
71 def mysys(*cmd)
72     printCommandArray(*cmd) if $verbosity >= 1
73     raise "Command failed: #{$?.inspect}" unless system(*cmd)
74 end
75
76 begin
77     $numProcessors = `sysctl -n hw.activecpu`.to_i
78 rescue
79     $numProcessors = 0
80 end
81
82 if $numProcessors == 0
83     $numProcessors = `nproc --all 2>/dev/null`.to_i
84 end
85 if $numProcessors == 0
86     $numProcessors = 1
87 end
88
89 $jscPath = nil
90 $enableFTL = false
91 $outputDir = Pathname.new("results")
92 $verbosity = 0
93 $bundle = nil
94 $tarball = false
95 $copyVM = false
96 $testRunnerType = :make
97 $remoteUser = nil
98 $remoteHost = nil
99 $remotePort = nil
100 $remoteDirectory = nil
101
102 def usage
103     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
104     puts
105     puts "--jsc                (-j)   Path to JavaScriptCore. This option is required."
106     puts "--ftl-jit                   Indicate that we have the FTL JIT."
107     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
108     puts "--verbose            (-v)   Print more things while running."
109     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
110     puts "--tarball                   Creates a tarball of the final bundle."
111     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
112     puts "                            In general the shell runner is slower than the make runner."
113     puts "--remote                    Specify a remote host on which to run tests."
114     puts "--help               (-h)   Print this message."
115     exit 1
116 end
117
118 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
119                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
120                ['--ftl-jit', GetoptLong::NO_ARGUMENT],
121                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
122                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
123                ['--tarball', GetoptLong::NO_ARGUMENT],
124                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
125                ['--shell-runner', GetoptLong::NO_ARGUMENT],
126                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
127                ['--verbose', '-v', GetoptLong::NO_ARGUMENT]).each {
128     | opt, arg |
129     case opt
130     when '--help'
131         usage
132     when '--jsc'
133         $jscPath = Pathname.new(arg).realpath
134     when '--output-dir'
135         $outputDir = Pathname.new(arg)
136     when '--ftl-jit'
137         $enableFTL = true
138     when '--verbose'
139         $verbosity += 1
140     when '--run-bundle'
141         $bundle = Pathname.new(arg)
142     when '--tarball'
143         $tarball = true
144         $copyVM = true
145     when '--force-vm-copy'
146         $copyVM = true
147     when '--shell-runner'
148         $testRunnerType = :shell
149     when '--remote'
150         $copyVM = true
151         $testRunnerType = :shell
152         $tarball = true
153         $remote = true
154         uri = URI("ftp://" + arg)
155         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
156     end
157 }
158
159 $progressMeter = ($verbosity == 0 and $stdin.tty?)
160
161 if $bundle
162     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
163     $outputDir = $bundle
164 end
165
166 unless $jscPath
167     $stderr.puts "Error: must specify -j <path>."
168     exit 1
169 end
170
171 $numFailures = 0
172
173 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20"]
174
175 $runlist = []
176
177 def frameworkFromJSCPath(jscPath)
178     parentDirectory = jscPath.dirname
179     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
180         parentDirectory.dirname
181     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
182         jscPath.dirname + "JavaScriptCore.framework"
183     else
184         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
185         nil
186     end
187 end
188
189 def prepareFramework(jscPath)
190     frameworkPath = frameworkFromJSCPath(jscPath)
191     $frameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
192     $jscPath = $frameworkPath + "Resources" + "jsc"
193
194     if frameworkPath
195         source = frameworkPath
196         destination = Pathname.new(".vm")
197     else
198         source = jscPath
199         destination = $jscPath
200
201         Dir.chdir($outputDir) {
202             FileUtils.mkdir_p $jscPath.dirname
203         }
204     end
205
206     Dir.chdir($outputDir) {
207         if $copyVM
208             FileUtils.cp_r source, destination
209         else
210             begin 
211                 FileUtils.ln_s source, destination
212             rescue
213                 FileUtils.cp_r source, destination
214             end
215         end
216     }
217 end
218
219 def copyVMToBundle
220     raise if $bundle
221
222     vmDir = $outputDir + ".vm"
223     FileUtils.mkdir_p vmDir
224    
225     prepareFramework($jscPath) 
226 end
227
228 def pathToVM
229     dir = Pathname.new(".")
230     $benchmarkDirectory.each_filename {
231         | pathComponent |
232         dir += ".."
233     }
234     dir + $jscPath
235 end
236
237 def prefixCommand(prefix)
238     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
239 end
240
241 def pipeAndPrefixCommand(outputFilename, prefix)
242     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
243 end
244
245 # Output handler for tests that are expected to be silent.
246 def silentOutputHandler
247     Proc.new {
248         | name |
249         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
250     }
251 end
252
253 # Output handler for tests that are expected to produce meaningful output.
254 def noisyOutputHandler
255     Proc.new {
256         | name |
257         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
258     }
259 end
260
261 # Error handler for tests that fail exactly when they return non-zero exit status.
262 def simpleErrorHandler
263     Proc.new {
264         | outp, plan |
265         outp.puts "if test -e #{plan.failFile}"
266         outp.puts "then"
267         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + prefixCommand(plan.name)
268         outp.puts "    " + plan.failCommand
269         outp.puts "else"
270         outp.puts "    " + plan.successCommand
271         outp.puts "fi"
272     }
273 end
274
275 # Error handler for tests that diff their output with some expectation.
276 def diffErrorHandler(expectedFilename)
277     Proc.new {
278         | outp, plan |
279         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
280         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
281         
282         outp.puts "if test -e #{plan.failFile}"
283         outp.puts "then"
284         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + prefixCommand(plan.name)
285         outp.puts "    " + plan.failCommand
286         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
287         outp.puts "then"
288         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
289         outp.puts "    if [ $? -eq 0 ]"
290         outp.puts "    then"
291         outp.puts "    " + plan.successCommand
292         outp.puts "    else"
293         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + prefixCommand(plan.name)
294         outp.puts "        " + plan.failCommand
295         outp.puts "    fi"
296         outp.puts "else"
297         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + prefixCommand(plan.name)
298         outp.puts "    " + plan.failCommand
299         outp.puts "fi"
300     }
301 end
302
303 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
304 # tests.
305 def mozillaErrorHandler
306     Proc.new {
307         | outp, plan |
308         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
309
310         outp.puts "if test -e #{plan.failFile}"
311         outp.puts "then"
312         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + prefixCommand(plan.name)
313         outp.puts "    " + plan.failCommand
314         outp.puts "elif grep -i -q failed! #{outputFilename}"
315         outp.puts "then"
316         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + prefixCommand(plan.name)
317         outp.puts "    " + plan.failCommand
318         outp.puts "else"
319         outp.puts "    " + plan.successCommand
320         outp.puts "fi"
321     }
322 end
323
324 # Error handler for tests that report error by saying "failed!", and are expected to
325 # fail. This is used by Mozilla tests.
326 def mozillaFailErrorHandler
327     Proc.new {
328         | outp, plan |
329         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
330
331         outp.puts "if test -e #{plan.failFile}"
332         outp.puts "then"
333         outp.puts "    " + plan.successCommand
334         outp.puts "elif grep -i -q failed! #{outputFilename}"
335         outp.puts "then"
336         outp.puts "    " + plan.successCommand
337         outp.puts "else"
338         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + prefixCommand(plan.name)
339         outp.puts "    " + plan.failCommand
340         outp.puts "fi"
341     }
342 end
343
344 # Error handler for tests that report error by saying "failed!", and are expected to have
345 # an exit code of 3.
346 def mozillaExit3ErrorHandler
347     Proc.new {
348         | outp, plan |
349         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
350
351         outp.puts "if test -e #{plan.failFile}"
352         outp.puts "then"
353         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
354         outp.puts "    then"
355         outp.puts "        if grep -i -q failed! #{outputFilename}"
356         outp.puts "        then"
357         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + prefixCommand(plan.name)
358         outp.puts "            " + plan.failCommand
359         outp.puts "        else"
360         outp.puts "            " + plan.successCommand
361         outp.puts "        fi"
362         outp.puts "    else"
363         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + prefixCommand(plan.name)
364         outp.puts "        " + plan.failCommand
365         outp.puts "    fi"
366         outp.puts "else"
367         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + prefixCommand(plan.name)
368         outp.puts "    " + plan.failCommand
369         outp.puts "fi"
370     }
371 end
372
373 $runCommandOptions = {}
374
375 class Plan
376     attr_reader :directory, :arguments, :name, :outputHandler, :errorHandler
377     attr_accessor :index
378     
379     def initialize(directory, arguments, name, outputHandler, errorHandler)
380         @directory = directory
381         @arguments = arguments
382         @name = name
383         @outputHandler = outputHandler
384         @errorHandler = errorHandler
385         @isSlow = !!$runCommandOptions[:isSlow]
386     end
387     
388     def shellCommand
389         # It's important to remember that the test is actually run in a subshell, so if we change directory
390         # in the subshell when we return we will be in our original directory. This is nice because we don't
391         # have to bend over backwards to do things relative to the root.
392         "(cd ../#{Shellwords.shellescape(@directory.to_s)} && \"$@\" " + @arguments.map{
393             | v |
394             raise "Detected a non-string in #{inspect}" unless v.is_a? String
395             Shellwords.shellescape(v)
396         }.join(' ') + ")"
397     end
398     
399     def reproScriptCommand
400         # We have to find our way back to the .runner directory since that's where all of the relative
401         # paths assume they start out from.
402         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
403         script += "cd $CURRENT_DIR\n"
404         Pathname.new(@name).dirname.each_filename {
405             | pathComponent |
406             script += "cd ..\n"
407         }
408         script += "cd .runner\n"
409
410         script += "export DYLD_FRAMEWORK_PATH=$(cd ../#{$frameworkPath.dirname}; pwd)\n"
411         IMPORTANT_ENVS.each {
412             | key |
413             if ENV[key]
414                 script += "export #{key}=#{Shellwords.shellescape(ENV[key])}\n"
415             end
416         }
417         script += "#{shellCommand} || exit 1"
418         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
419     end
420     
421     def failCommand
422         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
423     end
424     
425     def successCommand
426         if $progressMeter or $verbosity >= 2
427             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
428         else
429             "rm -f #{failFile}"
430         end
431     end
432     
433     def failFile
434         "test_fail_#{@index}"
435     end
436     
437     def writeRunScript(filename)
438         File.open(filename, "w") {
439             | outp |
440             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
441             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
442             cmd += @outputHandler.call(@name)
443             if $verbosity >= 3
444                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
445             end
446             outp.puts cmd
447             @errorHandler.call(outp, self)
448         }
449     end
450 end
451
452 $uniqueFilenameCounter = 0
453 def uniqueFilename(extension)
454     payloadDir = $outputDir + "_payload"
455     Dir.mkdir payloadDir unless payloadDir.directory?
456     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
457     $uniqueFilenameCounter += 1
458     result
459 end
460
461 def baseOutputName(kind)
462     "#{$collectionName}/#{$benchmark}.#{kind}"
463 end
464
465 def addRunCommand(kind, command, outputHandler, errorHandler)
466     plan = Plan.new($benchmarkDirectory, command, baseOutputName(kind), outputHandler, errorHandler)
467     if $numProcessors > 1 and $runCommandOptions[:isSlow]
468         $runlist.unshift plan
469     else
470         $runlist << plan
471     end
472 end
473
474 # Returns true if there were run commands found in the file ($benchmarkDirectory +
475 # $benchmark), in which case those run commands have already been executed. Otherwise
476 # returns false, in which case you're supposed to add your own run commands.
477 def parseRunCommands
478     didRun = false
479
480     Dir.chdir($outputDir) {
481         File.open($benchmarkDirectory + $benchmark) {
482             | inp |
483             inp.each_line {
484                 | line |
485                 begin
486                     doesMatch = line =~ /^\/\/@/
487                 rescue Exception => e
488                     # Apparently this happens in the case of some UTF8 stuff in some files, where
489                     # Ruby tries to be strict and throw exceptions.
490                     next
491                 end
492                 next unless doesMatch
493                 eval $~.post_match
494                 didRun = true
495             }
496         }
497     }
498
499     didRun
500 end
501
502 def slow!
503     $runCommandOptions[:isSlow] = true
504 end
505
506 def run(kind, *options)
507     addRunCommand(kind, [pathToVM.to_s] + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
508 end
509
510 def runDefault
511     run("default")
512 end
513
514 def runNoLLInt
515     run("no-llint", "--useLLInt=false")
516 end
517
518 def runNoCJITValidate
519     run("no-cjit", "--enableConcurrentJIT=false", "--validateBytecode=true", "--validateGraph=true")
520 end
521
522 def runNoCJITValidatePhases
523     run("no-cjit-validate-phases", "--enableConcurrentJIT=false", "--validateBytecode=true", "--validateGraphAtEachPhase=true")
524 end
525
526 def runDefaultFTL
527     run("default-ftl", "--useExperimentalFTL=true")
528 end
529
530 def runFTLNoCJIT
531     run("ftl-no-cjit", "--enableConcurrentJIT=false", "--useExperimentalFTL=true")
532 end
533
534 def runFTLNoCJITValidate
535     run("ftl-no-cjit-validate", "--enableConcurrentJIT=false", "--useExperimentalFTL=true", "--validateGraph=true")
536 end
537
538 def runFTLNoCJITOSRValidation
539     run("ftl-no-cjit-osr-validation", "--enableConcurrentJIT=false", "--useExperimentalFTL=true", "--validateFTLOSRExitLiveness=true")
540 end
541
542 def runDFGEager
543     run("dfg-eager", *EAGER_OPTIONS)
544 end
545
546 def runDFGEagerNoCJITValidate
547     run("dfg-eager-no-cjit-validate", "--enableConcurrentJIT=false", "--validateGraph=true", *EAGER_OPTIONS)
548 end
549
550 def runFTLEager
551     run("ftl-eager", "--useExperimentalFTL=true", *EAGER_OPTIONS)
552 end
553
554 def runFTLEagerNoCJITValidate
555     run("ftl-eager-no-cjit", "--useExperimentalFTL=true", "--enableConcurrentJIT=false", "--validateGraph=true", *EAGER_OPTIONS)
556 end
557
558 def runFTLEagerNoCJITOSRValidation
559     run("ftl-eager-no-cjit-osr-validation", "--useExperimentalFTL=true", "--enableConcurrentJIT=false", "--validateFTLOSRExitLiveness=true", *EAGER_OPTIONS)
560 end
561
562 def runAlwaysTriggerCopyPhase
563     run("always-trigger-copy-phase", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
564 end
565
566 def runNoCJITNoASO
567     run("no-cjit-no-aso", "--enableConcurrentJIT=false", "--enableArchitectureSpecificOptimizations=false")
568 end
569
570 def defaultRun
571     runDefault
572     runNoLLInt
573     runAlwaysTriggerCopyPhase
574     runNoCJITValidatePhases
575     runDFGEager
576     runDFGEagerNoCJITValidate
577     if $enableFTL
578         runDefaultFTL
579         runFTLNoCJITValidate
580         runFTLNoCJITOSRValidation
581         runFTLEager
582         runFTLEagerNoCJITValidate
583         runFTLEagerNoCJITOSRValidation
584     end
585 end
586
587 def defaultQuickRun
588     runDefault
589     runNoCJITValidate
590     if $enableFTL
591         runDefaultFTL
592         runFTLNoCJIT
593     end
594 end
595
596 def runProfiler
597     profilerOutput = uniqueFilename(".json")
598     if $canRunDisplayProfilerOutput
599         addRunCommand("profiler", ["ruby", (HELPERS_PATH + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
600     else
601         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
602         run("profiler-simple", "-p", profilerOutput.to_s)
603     end
604 end
605
606 def runLayoutTest(kind, *options)
607     raise unless $benchmark.to_s =~ /\.js$/
608     testName = $~.pre_match
609     if kind
610         kind = "layout-" + kind
611     else
612         kind = "layout"
613     end
614
615     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
616     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
617
618     args = [pathToVM.to_s] + options +
619         [(Pathname.new("resources") + "standalone-pre.js").to_s,
620          $benchmark.to_s,
621          (Pathname.new("resources") + "standalone-post.js").to_s]
622     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
623 end
624
625 def runLayoutTestDefault
626     runLayoutTest(nil)
627 end
628
629 def runLayoutTestNoLLInt
630     runLayoutTest("no-llint", "--useLLInt=false")
631 end
632
633 def runLayoutTestNoCJIT
634     runLayoutTest("no-cjit", "--enableConcurrentJIT=false")
635 end
636
637 def runLayoutTestDFGEagerNoCJIT
638     runLayoutTest("dfg-eager-no-cjit", "--enableConcurrentJIT=false", *EAGER_OPTIONS)
639 end
640
641 def defaultRunLayoutTest
642     runLayoutTestDefault
643     runLayoutTestNoLLInt
644     runLayoutTestNoCJIT
645     runLayoutTestDFGEagerNoCJIT
646 end
647
648 def prepareExtraRelativeFiles(extraFiles, destination)
649     Dir.chdir($outputDir) {
650         extraFiles.each {
651             | file |
652             FileUtils.cp $extraFilesBaseDir + file, destination + file
653         }
654     }
655 end
656
657 def baseDirForCollection(collectionName)
658     Pathname(".tests") + collectionName
659 end
660
661 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
662     raise unless absoluteBase.absolute?
663     Dir.chdir($outputDir) {
664         collectionBaseDir = baseDirForCollection($collectionName)
665         extraFiles.each {
666             | file |
667             destination = collectionBaseDir + file
668             FileUtils.mkdir_p destination.dirname unless destination.directory?
669             FileUtils.cp absoluteBase + file, destination
670         }
671     }
672 end
673
674 def runMozillaTest(kind, mode, extraFiles, *options)
675     if kind
676         kind = "mozilla-" + kind
677     else
678         kind = "mozilla"
679     end
680     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
681     args = [pathToVM.to_s] + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
682     case mode
683     when :normal
684         errorHandler = mozillaErrorHandler
685     when :negative
686         errorHandler = mozillaExit3ErrorHandler
687     when :fail
688         errorHandler = mozillaFailErrorHandler
689     when :skip
690         return
691     else
692         raise "Invalid mode: #{mode}"
693     end
694     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
695 end
696
697 def runMozillaTestDefault(mode, *extraFiles)
698     runMozillaTest(nil, mode, extraFiles)
699 end
700
701 def runMozillaTestLLInt(mode, *extraFiles)
702     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
703 end
704
705 def runMozillaTestBaselineJIT(mode, *extraFiles)
706     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
707 end
708
709 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
710     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--enableConcurrentJIT=false", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *EAGER_OPTIONS)
711 end
712
713 def defaultRunMozillaTest(mode, *extraFiles)
714     runMozillaTestDefault(mode, *extraFiles)
715     runMozillaTestLLInt(mode, *extraFiles)
716     runMozillaTestBaselineJIT(mode, *extraFiles)
717     runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
718 end
719
720 def skip
721     puts "Skipping #{$collectionName}/#{$benchmark}"
722 end
723
724 def allJSFiles(path)
725     if path.file?
726         [path]
727     else
728         result = []
729         Dir.foreach(path) {
730             | filename |
731             next unless filename =~ /\.js$/
732             next unless (path + filename).file?
733             result << path + filename
734         }
735         result
736     end
737 end
738
739 def uniqueifyName(names, name)
740     result = name.to_s
741     toAdd = 1
742     while names[result]
743         result = "#{name}-#{toAdd}"
744         toAdd += 1
745     end
746     names[result] = true
747     result
748 end
749
750 def simplifyCollectionName(collectionPath)
751     outerDir = collectionPath.dirname
752     name = collectionPath.basename
753     lastName = name
754     if collectionPath.directory?
755         while lastName.to_s =~ /test/
756             lastName = outerDir.basename
757             name = lastName + name
758             outerDir = outerDir.dirname
759         end
760     end
761     uniqueifyName($collectionNames, name)
762 end
763
764 def prepareCollection(name)
765     FileUtils.mkdir_p $outputDir + name
766
767     absoluteCollection = $collection.realpath
768
769     Dir.chdir($outputDir) {
770         bundleDir = baseDirForCollection(name)
771
772         # Create the proper directory structures.
773         FileUtils.mkdir_p bundleDir
774         if bundleDir.basename == $collection.basename
775             FileUtils.cp_r absoluteCollection, bundleDir.dirname
776         else
777             FileUtils.cp_r absoluteCollection, bundleDir
778         end
779
780         $extraFilesBaseDir = absoluteCollection
781
782         # Redirect the collection's location to the newly constructed bundle.
783         if absoluteCollection.directory?
784             $collection = bundleDir
785         else
786             $collection = bundleDir + $collection.basename
787         end
788     }
789 end
790
791 $collectionNames = {}
792
793 def handleCollectionFile(collection)
794     collectionName = simplifyCollectionName(collection)
795    
796     paths = {}
797     subCollections = []
798     YAML::load(IO::read(collection)).each {
799         | entry |
800         if entry["collection"]
801             subCollections << entry["collection"]
802             next
803         end
804         
805         if Pathname.new(entry["path"]).absolute?
806             raise "Absolute path: " + entry["path"] + " in #{collection}"
807         end
808         
809         if paths[entry["path"]]
810             raise "Duplicate path: " + entry["path"] + " in #{collection}"
811         end
812         
813         subCollection = collection.dirname + entry["path"]
814         
815         if subCollection.file?
816             subCollectionName = Pathname.new(entry["path"]).dirname
817         else
818             subCollectionName = entry["path"]
819         end
820         
821         $collection = subCollection
822         $collectionName = Pathname.new(collectionName)
823         Pathname.new(subCollectionName).each_filename {
824             | filename |
825             next if filename =~ /^\./
826             $collectionName += filename
827         }
828         $collectionName = $collectionName.to_s
829         
830         prepareCollection($collectionName)
831       
832         Dir.chdir($outputDir) {
833             directoryToSearch = $collection
834             if entry["tests"]
835                 directoryToSearch += entry["tests"]
836             end
837             allJSFiles(directoryToSearch).each {
838                 | path |
839                
840                 $benchmark = path.basename
841                 $benchmarkDirectory = path.dirname
842                 
843                 $runCommandOptions = {}
844                 eval entry["cmd"]
845             }
846         }
847     }
848     
849     subCollections.each {
850         | subCollection |
851         handleCollection(collection.dirname + subCollection)
852     }
853 end
854
855 def handleCollectionDirectory(collection)
856     collectionName = simplifyCollectionName(collection)
857     
858     $collection = collection
859     $collectionName = collectionName
860     prepareCollection(collectionName)
861    
862     Dir.chdir($outputDir) {
863         $benchmarkDirectory = $collection
864         allJSFiles($collection).each {
865             | path |
866             
867             $benchmark = path.basename
868             
869             $runCommandOptions = {}
870             defaultRun unless parseRunCommands
871         }
872     }
873 end
874
875 def handleCollection(collection)
876     collection = Pathname.new(collection)
877     
878     if collection.file?
879         handleCollectionFile(collection)
880     else
881         handleCollectionDirectory(collection)
882     end
883 end
884
885 def appendFailure(plan)
886     File.open($outputDir + "failed", "a") {
887         | outp |
888         outp.puts plan.name
889     }
890     $numFailures += 1
891 end
892
893 def prepareBundle
894     raise if $bundle
895
896     copyVMToBundle
897
898     ARGV.each {
899         | collection |
900         handleCollection(collection)
901     }
902
903     puts
904 end
905
906 def cleanOldResults
907     raise unless $bundle
908
909     eachResultFile($outputDir) {
910         | path |
911         FileUtils.rm_f path
912     }
913 end
914
915 def cleanEmptyResultFiles
916     eachResultFile($outputDir) {
917         | path |
918         next unless path.basename.to_s =~ /\.out$/
919         next unless FileTest.size(path) == 0
920         FileUtils.rm_f path
921     }
922 end
923
924 def eachResultFile(startingDir, &block)
925     dirsToClean = [startingDir]
926     until dirsToClean.empty?
927         nextDir = dirsToClean.pop
928         Dir.foreach(nextDir) {
929             | entry |
930             next if entry =~ /^\./
931             path = nextDir + entry
932             if path.directory?
933                 dirsToClean.push(path)
934             else
935                 block.call(path)
936             end
937         }
938     end
939 end
940
941 def prepareTestRunner
942     raise if $bundle
943
944     $runlist.each_with_index {
945         | plan, index |
946         plan.index = index
947     }
948
949     Dir.mkdir($runnerDir) unless $runnerDir.directory?
950     toDelete = []
951     Dir.foreach($runnerDir) {
952         | filename |
953         if filename =~ /^test_/
954             toDelete << filename
955         end
956     }
957     
958     toDelete.each {
959         | filename |
960         File.unlink($runnerDir + filename)
961     }
962
963     $runlist.each {
964         | plan |
965         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
966     }
967
968     case $testRunnerType
969     when :make
970         prepareMakeTestRunner
971     when :shell
972         prepareShellTestRunner
973     else
974         raise "Unknown test runner type: #{$testRunnerType.to_s}"
975     end
976 end
977
978 def prepareShellTestRunner
979     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
980 end
981
982 def prepareMakeTestRunner
983     # The goals of our parallel test runner are scalability and simplicity. The
984     # simplicity part is particularly important. We don't want to have to have
985     # a full-time contributor just philosophising about parallel testing.
986     #
987     # As such, we just pass off all of the hard work to 'make'. This creates a
988     # dummy directory ("$outputDir/.runner") in which we create a dummy
989     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
990     # That is, for each test we know we will run, there is a rule in the
991     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
992     # Makefile results in 'make' doing all of the hard work:
993     #
994     # - Load balancing just works. Most systems have a great load balancer in
995     #   'make'. If your system doesn't then just install a real 'make'.
996     #
997     # - Interruptions just work. For example Ctrl-C handling in 'make' is
998     #   exactly right. You don't have to worry about zombie processes.
999     #
1000     # We then do some tricks to make failure detection work and to make this
1001     # totally sound. If a test fails, we don't want the whole 'make' job to
1002     # stop. We also don't have any facility for makefile-escaping of path names.
1003     # We do have such a thing for shell-escaping, though. We fix both problems
1004     # by having the actual work for each of the test rules be done in a shell
1005     # script on the side. There is one such script per test. The script responds
1006     # to failure by printing something on the console and then touching a
1007     # failure file for that test, but then still returns 0. This makes 'make'
1008     # continue past that failure and complete all the tests anyway.
1009     #
1010     # In the end, this script collects all of the failures by searching for
1011     # files in the .runner directory whose name matches /^test_fail_/, where
1012     # the thing after the 'fail_' is the test index. Those are the files that
1013     # would be created by the test scripts if they detect failure. We're
1014     # basically using the filesystem as a concurrent database of test failures.
1015     # Even if two tests fail at the same time, since they're touching different
1016     # files we won't miss any failures.
1017     runIndices = []
1018     $runlist.each {
1019         | plan |
1020         runIndices << plan.index
1021     }
1022     
1023     File.open($runnerDir + "Makefile", "w") {
1024         | outp |
1025         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1026         runIndices.each {
1027             | index |
1028             plan = $runlist[index]
1029             outp.puts "test_done_#{index}:"
1030             outp.puts "\tsh test_script_#{plan.index}"
1031         }
1032     }
1033 end
1034
1035 if $enableFTL and ENV["JSC_timeout"]
1036     # Currently, using the FTL is a performance regression particularly in real
1037     # (i.e. non-loopy) benchmarks. Account for this in the timeout.
1038     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i * 2).to_s
1039 end
1040
1041 if ENV["JSC_timeout"]
1042     # In the worst case, the processors just interfere with each other.
1043     # Increase the timeout proportionally to the number of processors.
1044     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i.to_f * Math.sqrt($numProcessors)).to_i.to_s
1045 end
1046     
1047 puts
1048
1049 def cleanRunnerDirectory
1050     raise unless $bundle
1051     Dir.foreach($runnerDir) {
1052         | filename |
1053         next unless filename =~ /^test_fail/
1054         FileUtils.rm_f $runnerDir + filename
1055     }
1056 end
1057
1058 def runTestRunner
1059     case $testRunnerType
1060     when :shell
1061         runShellTestRunner
1062     when :make
1063         runMakeTestRunner
1064     else
1065         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1066     end
1067 end
1068
1069 def sshRead(cmd)
1070     raise unless $remote
1071
1072     result = ""
1073     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1074       | inp |
1075       inp.each_line {
1076         | line |
1077         result += line
1078       }
1079     }
1080     raise "#{$?}" unless $?.success?
1081     result
1082 end
1083
1084 def runShellTestRunner
1085     if $remote
1086         $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1087         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + "payload.tar.gz").to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1088         remoteScript = ""
1089         remoteScript += "cd #{$remoteDirectory} && "
1090         remoteScript += "rm -rf #{$outputDir.basename} && "
1091         remoteScript += "tar xzf payload.tar.gz && "
1092         remoteScript += "cd #{$outputDir.basename}/.runner && "
1093         remoteScript += "DYLD_FRAMEWORK_PATH=$(cd ../#{$frameworkPath.dirname}; pwd) sh runscript" 
1094         system("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1095     else
1096         Dir.chdir($runnerDir) {
1097             mysys("sh", "runscript")
1098         }
1099     end
1100 end
1101
1102 def runMakeTestRunner
1103     raise if $remote
1104     Dir.chdir($runnerDir) {
1105         # -1 for the Makefile, and -2 for '..' and '.'
1106         numberOfTests = Dir.entries(".").count - 3
1107         unless $progressMeter
1108             mysys("make", "-j", $numProcessors.to_s, "-s", "-f", "Makefile")
1109         else
1110             cmd = "make -j #{$numProcessors} -s -f Makefile"
1111             running = {}
1112             didRun = {}
1113             didFail = {}
1114             blankLine = true
1115             prevStringLength = 0
1116             IO.popen(cmd, "r") {
1117                 | inp |
1118                 inp.each_line {
1119                     | line |
1120                     line.chomp!
1121                     if line =~ /^Running /
1122                         running[$~.post_match] = true
1123                     elsif line =~ /^PASS: /
1124                         didRun[$~.post_match] = true
1125                     elsif line =~ /^FAIL: /
1126                         didRun[$~.post_match] = true
1127                         didFail[$~.post_match] = true
1128                     else
1129                         unless blankLine
1130                             print("\r" + " " * prevStringLength + "\r")
1131                         end
1132                         puts line
1133                         blankLine = true
1134                     end
1135                     
1136                     def lpad(str, chars)
1137                         str = str.to_s
1138                         if str.length > chars
1139                             str
1140                         else
1141                             "%#{chars}s"%(str)
1142                         end
1143                     end
1144     
1145                     string  = ""
1146                     string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1147                     unless didFail.empty?
1148                         string += " (failed #{didFail.size})"
1149                     end
1150                     string += " "
1151                     (running.size - didRun.size).times {
1152                         string += "."
1153                     }
1154                     if string.length < prevStringLength
1155                         print string
1156                         print(" " * (prevStringLength - string.length))
1157                     end
1158                     print string
1159                     prevStringLength = string.length
1160                     blankLine = false
1161                     $stdout.flush
1162                 }
1163             }
1164             puts
1165             raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1166         end
1167     }
1168 end
1169
1170 def detectFailures
1171     raise if $bundle
1172
1173     if $remote
1174         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && (ls test_fail_* 2> /dev/null || true)")
1175         output.split(/\n/).each {
1176             | line |
1177             next unless line =~ /test_fail_/
1178             appendFailure($runlist[$~.post_match.to_i])
1179         }
1180     else
1181         Dir.foreach($runnerDir) {
1182             | filename |
1183             next unless filename =~ /test_fail_/
1184             appendFailure($runlist[$~.post_match.to_i])
1185         }
1186     end
1187 end
1188
1189 def compressBundle
1190     cmd = "cd #{$outputDir}/.. && tar -czf payload.tar.gz #{$outputDir.basename}"
1191     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1192     raise unless system(cmd)
1193 end
1194
1195 def clean(file)
1196     FileUtils.rm_rf file unless $bundle
1197 end
1198
1199 clean($outputDir + "failed")
1200 clean($outputDir + ".vm")
1201 clean($outputDir + ".runner")
1202 clean($outputDir + ".tests")
1203 clean($outputDir + "_payload")
1204
1205 Dir.mkdir($outputDir) unless $outputDir.directory?
1206
1207 $outputDir = $outputDir.realpath
1208 $runnerDir = $outputDir + ".runner"
1209
1210 def runBundle
1211     raise unless $bundle
1212
1213     cleanRunnerDirectory
1214     cleanOldResults
1215     runTestRunner
1216     cleanEmptyResultFiles
1217 end
1218
1219 def runNormal
1220     raise if $bundle or $tarball
1221
1222     prepareBundle
1223     prepareTestRunner
1224     runTestRunner
1225     cleanEmptyResultFiles
1226     detectFailures
1227 end
1228
1229 def runTarball
1230     raise unless $tarball
1231
1232     prepareBundle 
1233     prepareTestRunner
1234     compressBundle
1235 end
1236
1237 def runRemote
1238     raise unless $remote
1239
1240     prepareBundle
1241     prepareTestRunner
1242     compressBundle
1243     runTestRunner
1244     detectFailures
1245 end
1246
1247 if $bundle
1248     runBundle
1249 elsif $remote
1250     runRemote
1251 elsif $tarball
1252     runTarball
1253 else
1254     runNormal
1255 end