Move cursor to corner and fix safari window size before running benchmark
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2015 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
41 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
42 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
43 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
44 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
45 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
46
47 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
48
49 begin
50     require 'shellwords'
51 rescue Exception => e
52     $stderr.puts "Warning: did not find shellwords, not running any tests."
53     exit 0
54 end
55
56 $canRunDisplayProfilerOutput = false
57
58 begin
59     require 'rubygems'
60     require 'json'
61     require 'highline'
62     $canRunDisplayProfilerOutput = true
63 rescue Exception => e
64     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
65     $stderr.puts "Error: #{e.inspect}"
66 end
67
68 def printCommandArray(*cmd)
69     begin
70         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
71     rescue
72         commandArray = cmd.join(' ')
73     end
74     $stderr.puts ">> #{commandArray}"
75 end
76
77 def mysys(*cmd)
78     printCommandArray(*cmd) if $verbosity >= 1
79     raise "Command failed: #{$?.inspect}" unless system(*cmd)
80 end
81
82 def escapeAll(array)
83     array.map {
84         | v |
85         raise "Detected a non-string in #{inspect}" unless v.is_a? String
86         Shellwords.shellescape(v)
87     }.join(' ')
88 end
89
90
91 $jscPath = nil
92 $doNotMessWithVMPath = false
93 $enableFTL = false
94 $jitTests = true
95 $memoryLimited = false
96 $outputDir = Pathname.new("results")
97 $verbosity = 0
98 $bundle = nil
99 $tarball = false
100 $tarFileName = "payload.tar.gz"
101 $copyVM = false
102 $testRunnerType = nil
103 $remoteUser = nil
104 $remoteHost = nil
105 $remotePort = nil
106 $remoteDirectory = nil
107 $architecture = nil
108 $hostOS = nil
109 $filter = nil
110
111
112 def usage
113     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
114     puts
115     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
116     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
117     puts "                            --jsc specifies an already present JavaScriptCore to test."
118     puts "--ftl-jit                   Indicate that we have the FTL JIT."
119     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
120     puts "                            Skip tests tagged with //@large-heap"
121     puts "--no-jit                    Do not run JIT specific tests."
122     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
123     puts "--verbose            (-v)   Print more things while running."
124     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
125     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
126     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
127     puts "                            e.g. x86, x86_64, arm."
128     puts "--os                        Specify os instead of determining from JavaScriptCore build."
129     puts "                            e.g. darwin, linux & windows."
130     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
131     puts "                            In general the shell runner is slower than the make runner."
132     puts "--make-runner               Uses the faster make-based runner."
133     puts "--remote                    Specify a remote host on which to run tests from command line argument."
134     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
135     puts "--child-processes    (-c)   Specify the number of child processes."
136     puts "--filter                    Only run tests whose name matches the given regular expression."
137     puts "--help               (-h)   Print this message."
138     exit 1
139 end
140
141 jscArg = nil
142
143 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
144                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
145                ['--no-copy', GetoptLong::NO_ARGUMENT],
146                ['--ftl-jit', GetoptLong::NO_ARGUMENT],
147                ['--memory-limited', GetoptLong::NO_ARGUMENT],
148                ['--no-jit', GetoptLong::NO_ARGUMENT],
149                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
150                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
151                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
152                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
153                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
154                ['--os', GetoptLong::REQUIRED_ARGUMENT],
155                ['--shell-runner', GetoptLong::NO_ARGUMENT],
156                ['--make-runner', GetoptLong::NO_ARGUMENT],
157                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
158                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
159                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
160                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
161                ['--verbose', '-v', GetoptLong::NO_ARGUMENT]).each {
162     | opt, arg |
163     case opt
164     when '--help'
165         usage
166     when '--jsc'
167         jscArg = arg
168     when '--no-copy'
169         $doNotMessWithVMPath = true
170     when '--output-dir'
171         $outputDir = Pathname.new(arg)
172     when '--ftl-jit'
173         $enableFTL = true
174     when '--memory-limited'
175         $memoryLimited = true
176     when '--no-jit'
177         $jitTests = false
178     when '--verbose'
179         $verbosity += 1
180     when '--run-bundle'
181         $bundle = Pathname.new(arg)
182     when '--tarball'
183         $tarball = true
184         $copyVM = true
185         $tarFileName = arg unless arg == ''
186     when '--force-vm-copy'
187         $copyVM = true
188     when '--shell-runner'
189         $testRunnerType = :shell
190     when '--make-runner'
191         $testRunnerType = :make
192     when '--remote'
193         $copyVM = true
194         $tarball = true
195         $remote = true
196         uri = URI("ssh://" + arg)
197         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
198     when '--remote-config-file'
199         $remoteConfigFile = arg
200     when '--child-processes'
201         $numChildProcesses = arg.to_i
202     when '--filter'
203         $filter = Regexp.new(arg)
204     when '--arch'
205         $architecture = arg
206     when '--os'
207         $hostOS = arg
208     end
209 }
210
211 if $remoteConfigFile
212     file = File.read($remoteConfigFile)
213     config = JSON.parse(file)
214
215     if !$remote and config['remote']
216         $copyVM = true
217         $tarball = true
218         $remote = true
219         uri = URI("ssh://" + config['remote'])
220         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
221     end
222
223     if config['remoteDirectory']
224         $remoteDirectory = config['remoteDirectory']
225     end
226 end
227
228 unless jscArg
229     $stderr.puts "Error: must specify -jsc <path>"
230     exit 1
231 end
232
233 if $enableFTL and !$jitTests
234     $stderr.puts "Error: can only specify one of --no-jit and --ftl-jit"
235     exit 1
236 end
237
238 if $doNotMessWithVMPath
239     $jscPath = Pathname.new(jscArg)
240 else
241     $jscPath = Pathname.new(jscArg).realpath
242 end
243
244 $progressMeter = ($verbosity == 0 and $stdout.tty?)
245
246 if $bundle
247     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
248     $outputDir = $bundle
249 end
250
251 # Try to determine architecture. Return nil on failure.
252 def machOArchitectureCode
253     begin 
254         otoolLines = `otool -aSfh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
255         otoolLines.each_with_index {
256             | value, index |
257             if value =~ /magic/ and value =~ /cputype/
258                 return otoolLines[index + 1].split[1].to_i
259             end
260         }
261     rescue
262         $stderr.puts "Warning: unable to execute otool."
263     end
264     $stderr.puts "Warning: unable to determine architecture."
265     nil
266 end
267
268 def determineArchitectureFromMachOBinary
269     code = machOArchitectureCode
270     return nil unless code
271     is64BitFlag = 0x01000000
272     case code
273     when 7
274         "x86"
275     when 7 | is64BitFlag
276         "x86-64"
277     when 12
278         "arm"
279     when 12 | is64BitFlag
280         "arm64"
281     else
282         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
283         nil
284     end
285 end
286
287 def determineArchitectureFromELFBinary
288     f = File.open($jscPath.to_s)
289     data = f.read(19)
290
291     if !(data[0,4] == "\x7F\x45\x4C\x46")
292         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
293         return nil
294     end
295
296     code = data[18].ord
297     case code
298     when 3
299         "x86"
300     when 62
301         "x86-64"
302     when 40
303         "arm"
304     when 183
305         "arm64"
306     else
307         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
308         nil
309     end
310 end
311
312 def determineArchitecture
313     case $hostOS
314     when "darwin"
315         determineArchitectureFromMachOBinary
316     when "linux"
317         determineArchitectureFromELFBinary
318     else
319         $stderr.puts "Warning: unable to determine architecture on this platform."
320         nil
321     end
322 end
323
324 def determineOS
325     case RbConfig::CONFIG["host_os"]
326     when /darwin/i
327         "darwin"
328     when /linux/i
329         "linux"
330     when /mswin|mingw|cygwin/
331         "windows"
332     else
333         $stderr.puts "Warning: unable to determine host operating system"
334         nil
335     end
336 end
337
338 $hostOS = determineOS unless $hostOS
339 $architecture = determineArchitecture unless $architecture
340
341 if !$testRunnerType
342     if $remote and $hostOS == "darwin"
343         $testRunnerType = :shell
344     else
345         $testRunnerType = :make
346     end
347 end
348
349 $numFailures = 0
350
351 BASE_OPTIONS = ["--useFTLJIT=false", "--enableFunctionDotArguments=true"]
352 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000"]
353 NO_CJIT_OPTIONS = ["--enableConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100"]
354 FTL_OPTIONS = ["--useFTLJIT=true", "--ftlCrashesIfCantInitializeLLVM=true"]
355
356 $runlist = []
357
358 def frameworkFromJSCPath(jscPath)
359     parentDirectory = jscPath.dirname
360     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
361         parentDirectory.dirname
362     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
363         jscPath.dirname + "JavaScriptCore.framework"
364     else
365         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
366         nil
367     end
368 end
369
370 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
371     dir = Pathname.new(".")
372     $benchmarkDirectory.each_filename {
373         | pathComponent |
374         dir += ".."
375     }
376     dir + resourcePath
377 end
378
379 def pathToVM
380     pathToBundleResourceFromBenchmarkDirectory($jscPath)
381 end
382
383 def pathToHelpers
384     pathToBundleResourceFromBenchmarkDirectory(".helpers")
385 end
386
387 def prefixCommand(prefix)
388     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
389 end
390
391 def redirectAndPrefixCommand(prefix)
392     prefixCommand(prefix) + " 2>&1"
393 end
394
395 def pipeAndPrefixCommand(outputFilename, prefix)
396     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
397 end
398
399 # Output handler for tests that are expected to be silent.
400 def silentOutputHandler
401     Proc.new {
402         | name |
403         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
404     }
405 end
406
407 # Output handler for tests that are expected to produce meaningful output.
408 def noisyOutputHandler
409     Proc.new {
410         | name |
411         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
412     }
413 end
414
415 # Error handler for tests that fail exactly when they return non-zero exit status.
416 def simpleErrorHandler
417     Proc.new {
418         | outp, plan |
419         outp.puts "if test -e #{plan.failFile}"
420         outp.puts "then"
421         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
422         outp.puts "    " + plan.failCommand
423         outp.puts "else"
424         outp.puts "    " + plan.successCommand
425         outp.puts "fi"
426     }
427 end
428
429 # Error handler for tests that diff their output with some expectation.
430 def diffErrorHandler(expectedFilename)
431     Proc.new {
432         | outp, plan |
433         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
434         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
435         
436         outp.puts "if test -e #{plan.failFile}"
437         outp.puts "then"
438         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
439         outp.puts "    " + plan.failCommand
440         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
441         outp.puts "then"
442         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
443         outp.puts "    if [ $? -eq 0 ]"
444         outp.puts "    then"
445         outp.puts "    " + plan.successCommand
446         outp.puts "    else"
447         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
448         outp.puts "        " + plan.failCommand
449         outp.puts "    fi"
450         outp.puts "else"
451         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
452         outp.puts "    " + plan.failCommand
453         outp.puts "fi"
454     }
455 end
456
457 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
458 # tests.
459 def mozillaErrorHandler
460     Proc.new {
461         | outp, plan |
462         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
463
464         outp.puts "if test -e #{plan.failFile}"
465         outp.puts "then"
466         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
467         outp.puts "    " + plan.failCommand
468         outp.puts "elif grep -i -q failed! #{outputFilename}"
469         outp.puts "then"
470         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
471         outp.puts "    " + plan.failCommand
472         outp.puts "else"
473         outp.puts "    " + plan.successCommand
474         outp.puts "fi"
475     }
476 end
477
478 # Error handler for tests that report error by saying "failed!", and are expected to
479 # fail. This is used by Mozilla tests.
480 def mozillaFailErrorHandler
481     Proc.new {
482         | outp, plan |
483         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
484
485         outp.puts "if test -e #{plan.failFile}"
486         outp.puts "then"
487         outp.puts "    " + plan.successCommand
488         outp.puts "elif grep -i -q failed! #{outputFilename}"
489         outp.puts "then"
490         outp.puts "    " + plan.successCommand
491         outp.puts "else"
492         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
493         outp.puts "    " + plan.failCommand
494         outp.puts "fi"
495     }
496 end
497
498 # Error handler for tests that report error by saying "failed!", and are expected to have
499 # an exit code of 3.
500 def mozillaExit3ErrorHandler
501     Proc.new {
502         | outp, plan |
503         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
504
505         outp.puts "if test -e #{plan.failFile}"
506         outp.puts "then"
507         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
508         outp.puts "    then"
509         outp.puts "        if grep -i -q failed! #{outputFilename}"
510         outp.puts "        then"
511         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
512         outp.puts "            " + plan.failCommand
513         outp.puts "        else"
514         outp.puts "            " + plan.successCommand
515         outp.puts "        fi"
516         outp.puts "    else"
517         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
518         outp.puts "        " + plan.failCommand
519         outp.puts "    fi"
520         outp.puts "else"
521         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
522         outp.puts "    " + plan.failCommand
523         outp.puts "fi"
524     }
525 end
526
527 $runCommandOptions = {}
528
529 class Plan
530     attr_reader :directory, :arguments, :name, :outputHandler, :errorHandler
531     attr_accessor :index
532     
533     def initialize(directory, arguments, name, outputHandler, errorHandler)
534         @directory = directory
535         @arguments = arguments
536         @name = name
537         @outputHandler = outputHandler
538         @errorHandler = errorHandler
539         @isSlow = !!$runCommandOptions[:isSlow]
540     end
541     
542     def shellCommand
543         # It's important to remember that the test is actually run in a subshell, so if we change directory
544         # in the subshell when we return we will be in our original directory. This is nice because we don't
545         # have to bend over backwards to do things relative to the root.
546         "(cd ../#{Shellwords.shellescape(@directory.to_s)} && \"$@\" " + escapeAll(@arguments) + ")"
547     end
548     
549     def reproScriptCommand
550         # We have to find our way back to the .runner directory since that's where all of the relative
551         # paths assume they start out from.
552         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
553         script += "cd $CURRENT_DIR\n"
554         Pathname.new(@name).dirname.each_filename {
555             | pathComponent |
556             script += "cd ..\n"
557         }
558         script += "cd .runner\n"
559
560         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
561         script += "export JSC_timeout=#{Shellwords.shellescape(ENV['JSC_timeout'])}\n"
562         script += "#{shellCommand} || exit 1"
563         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
564     end
565     
566     def failCommand
567         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
568     end
569     
570     def successCommand
571         if $progressMeter or $verbosity >= 2
572             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
573         else
574             "rm -f #{failFile}"
575         end
576     end
577     
578     def failFile
579         "test_fail_#{@index}"
580     end
581     
582     def writeRunScript(filename)
583         File.open(filename, "w") {
584             | outp |
585             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
586             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
587             cmd += @outputHandler.call(@name)
588             if $verbosity >= 3
589                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
590             end
591             outp.puts cmd
592             @errorHandler.call(outp, self)
593         }
594     end
595 end
596
597 $uniqueFilenameCounter = 0
598 def uniqueFilename(extension)
599     payloadDir = $outputDir + "_payload"
600     Dir.mkdir payloadDir unless payloadDir.directory?
601     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
602     $uniqueFilenameCounter += 1
603     result
604 end
605
606 def baseOutputName(kind)
607     "#{$collectionName}/#{$benchmark}.#{kind}"
608 end
609
610 def addRunCommand(kind, command, outputHandler, errorHandler)
611     $didAddRunCommand = true
612     name = baseOutputName(kind)
613     if $filter and name !~ $filter
614         return
615     end
616     plan = Plan.new($benchmarkDirectory, command, name, outputHandler, errorHandler)
617     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
618         $runlist.unshift plan
619     else
620         $runlist << plan
621     end
622 end
623
624 # Returns true if there were run commands found in the file ($benchmarkDirectory +
625 # $benchmark), in which case those run commands have already been executed. Otherwise
626 # returns false, in which case you're supposed to add your own run commands.
627 def parseRunCommands
628     oldDidAddRunCommand = $didAddRunCommand
629     $didAddRunCommand = false
630
631     Dir.chdir($outputDir) {
632         File.open($benchmarkDirectory + $benchmark) {
633             | inp |
634             inp.each_line {
635                 | line |
636                 begin
637                     doesMatch = line =~ /^\/\/@/
638                 rescue Exception => e
639                     # Apparently this happens in the case of some UTF8 stuff in some files, where
640                     # Ruby tries to be strict and throw exceptions.
641                     next
642                 end
643                 next unless doesMatch
644                 eval $~.post_match
645             }
646         }
647     }
648
649     result = $didAddRunCommand
650     $didAddRunCommand = result or oldDidAddRunCommand
651     result
652 end
653
654 def slow!
655     $runCommandOptions[:isSlow] = true
656 end
657
658 def run(kind, *options)
659     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
660 end
661
662 def runDefault
663     run("default")
664 end
665
666 def runWithRAMSize(size)
667     run("ram-size-#{size}", "--forceRAMSize=#{size}")
668 end
669
670 def runNoLLInt
671     if $jitTests
672         run("no-llint", "--useLLInt=false")
673     end
674 end
675
676 def runNoCJITValidate
677     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *NO_CJIT_OPTIONS)
678 end
679
680 def runNoCJITValidatePhases
681     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
682 end
683
684 def runDefaultFTL
685     run("default-ftl", *FTL_OPTIONS) if $enableFTL
686 end
687
688 def runFTLNoCJIT
689     run("ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
690 end
691
692 def runFTLNoCJITValidate
693     run("ftl-no-cjit-validate", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
694 end
695
696 def runFTLNoCJITNoInlineValidate
697     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
698 end
699
700 def runFTLNoCJITOSRValidation
701     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
702 end
703
704 def runDFGEager
705     run("dfg-eager", *EAGER_OPTIONS)
706 end
707
708 def runDFGEagerNoCJITValidate
709     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
710 end
711
712 def runFTLEager
713     run("ftl-eager", *(FTL_OPTIONS + EAGER_OPTIONS)) if $enableFTL
714 end
715
716 def runFTLEagerNoCJITValidate
717     run("ftl-eager-no-cjit", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
718 end
719
720 def runFTLEagerNoCJITOSRValidation
721     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
722 end
723
724 def runAlwaysTriggerCopyPhase
725     run("always-trigger-copy-phase", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
726 end
727
728 def runNoCJITNoASO
729     run("no-cjit-no-aso", "--enableArchitectureSpecificOptimizations=false", *NO_CJIT_OPTIONS)
730 end
731
732 def runFTLNoCJITNoSimpleOpt
733     run("ftl-no-cjit-no-simple-opt", "--llvmSimpleOpt=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
734 end
735
736 def runNoCJITNoAccessInlining
737     run("no-cjit-no-access-inlining", "--enableAccessInlining=false", *NO_CJIT_OPTIONS)
738 end
739
740 def runFTLNoCJITNoAccessInlining
741     run("ftl-no-cjit-no-access-inlining", "--enableAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
742 end
743
744 def defaultRun
745     runDefault
746     runAlwaysTriggerCopyPhase
747     if $jitTests
748         runNoLLInt
749         runNoCJITValidatePhases
750         runDFGEager
751         runDFGEagerNoCJITValidate
752         runDefaultFTL
753         runFTLNoCJITValidate
754         runFTLNoCJITNoInlineValidate
755         runFTLEager
756         runFTLEagerNoCJITValidate
757     end
758 end
759
760 def defaultQuickRun
761     if $enableFTL
762         runDefaultFTL
763         runFTLNoCJITValidate
764     else
765         runDefault
766         if $jitTests
767             runNoCJITValidate
768         end
769     end
770 end
771
772 def defaultSpotCheck
773     defaultQuickRun
774     runFTLNoCJITNoSimpleOpt
775     runFTLNoCJITOSRValidation
776     runNoCJITNoAccessInlining
777     runFTLNoCJITNoAccessInlining
778 end
779
780 # This is expected to not do eager runs because eager runs can have a lot of recompilations
781 # for reasons that don't arise in the real world. It's used for tests that assert convergence
782 # by counting recompilations.
783 def defaultNoEagerRun
784     runDefault
785     runAlwaysTriggerCopyPhase
786     if $jitTests
787         runNoLLInt
788         runNoCJITValidatePhases
789         runDefaultFTL
790         runFTLNoCJITValidate
791         runFTLNoCJITNoInlineValidate
792     end
793 end
794
795 def runProfiler
796     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
797         skip
798         return
799     end
800
801     profilerOutput = uniqueFilename(".json")
802     if $canRunDisplayProfilerOutput
803         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
804     else
805         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
806         run("profiler-simple", "-p", profilerOutput.to_s)
807     end
808 end
809
810 def runExceptionFuzz
811     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
812     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
813 end
814
815 def runExecutableAllocationFuzz(name, *options)
816     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
817     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
818 end
819
820 def runTypeProfiler
821     if !$jitTests
822         return
823     end
824
825     if $enableFTL
826         run("ftl-no-cjit-type-profiler", "--enableTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
827     else
828         run("no-cjit-type-profiler", "--enableTypeProfiler=true", *NO_CJIT_OPTIONS)
829     end
830 end
831
832 def runControlFlowProfiler
833     if !$jitTests
834         return
835     end
836
837     if $enableFTL
838         run("ftl-no-cjit-type-profiler", "--enableControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
839     else
840         run("no-cjit-type-profiler", "--enableControlFlowProfiler=true", *NO_CJIT_OPTIONS)
841     end
842 end
843
844 def runLayoutTest(kind, *options)
845     raise unless $benchmark.to_s =~ /\.js$/
846     testName = $~.pre_match
847     if kind
848         kind = "layout-" + kind
849     else
850         kind = "layout"
851     end
852
853     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
854     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
855
856     args = [pathToVM.to_s] + BASE_OPTIONS + options +
857         [(Pathname.new("resources") + "standalone-pre.js").to_s,
858          $benchmark.to_s,
859          (Pathname.new("resources") + "standalone-post.js").to_s]
860     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
861 end
862
863 def runLayoutTestDefault
864     runLayoutTest(nil)
865 end
866
867 def runLayoutTestNoLLInt
868     runLayoutTest("no-llint", "--useLLInt=false")
869 end
870
871 def runLayoutTestNoCJIT
872     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
873 end
874
875 def runLayoutTestDFGEagerNoCJIT
876     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
877 end
878
879 def runLayoutTestDefaultFTL
880     runLayoutTest("ftl", "--testTheFTL=true", *FTL_OPTIONS) if $enableFTL
881 end
882
883 def runLayoutTestFTLNoCJIT
884     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
885 end
886
887 def runLayoutTestFTLEagerNoCJIT
888     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
889 end
890
891 def noFTLRunLayoutTest
892     if !$jitTests
893         return
894     end
895
896     runLayoutTestNoLLInt
897     runLayoutTestNoCJIT
898     runLayoutTestDFGEagerNoCJIT
899 end
900
901 def defaultRunLayoutTest
902     runLayoutTestDefault
903     if $jitTests
904         noFTLRunLayoutTest
905         runLayoutTestDefaultFTL
906         runLayoutTestFTLNoCJIT
907         runLayoutTestFTLEagerNoCJIT
908     end
909 end
910
911 def prepareExtraRelativeFiles(extraFiles, destination)
912     Dir.chdir($outputDir) {
913         extraFiles.each {
914             | file |
915             FileUtils.cp $extraFilesBaseDir + file, destination + file
916         }
917     }
918 end
919
920 def baseDirForCollection(collectionName)
921     Pathname(".tests") + collectionName
922 end
923
924 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
925     raise unless absoluteBase.absolute?
926     Dir.chdir($outputDir) {
927         collectionBaseDir = baseDirForCollection($collectionName)
928         extraFiles.each {
929             | file |
930             destination = collectionBaseDir + file
931             FileUtils.mkdir_p destination.dirname unless destination.directory?
932             FileUtils.cp absoluteBase + file, destination
933         }
934     }
935 end
936
937 def runMozillaTest(kind, mode, extraFiles, *options)
938     if kind
939         kind = "mozilla-" + kind
940     else
941         kind = "mozilla"
942     end
943     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
944     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
945     case mode
946     when :normal
947         errorHandler = mozillaErrorHandler
948     when :negative
949         errorHandler = mozillaExit3ErrorHandler
950     when :fail
951         errorHandler = mozillaFailErrorHandler
952     when :skip
953         return
954     else
955         raise "Invalid mode: #{mode}"
956     end
957     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
958 end
959
960 def runMozillaTestDefault(mode, *extraFiles)
961     runMozillaTest(nil, mode, extraFiles)
962 end
963
964 def runMozillaTestDefaultFTL(mode, *extraFiles)
965     runMozillaTest("ftl", mode, extraFiles, *FTL_OPTIONS) if $enableFTL
966 end
967
968 def runMozillaTestLLInt(mode, *extraFiles)
969     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
970 end
971
972 def runMozillaTestBaselineJIT(mode, *extraFiles)
973     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
974 end
975
976 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
977     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
978 end
979
980 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
981     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
982 end
983
984 def defaultRunMozillaTest(mode, *extraFiles)
985     runMozillaTestDefault(mode, *extraFiles)
986     if $jitTests
987         runMozillaTestLLInt(mode, *extraFiles)
988         runMozillaTestBaselineJIT(mode, *extraFiles)
989         runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
990         runMozillaTestDefaultFTL(mode, *extraFiles)
991         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
992     end
993 end
994
995 def skip
996     $didAddRunCommand = true
997     puts "Skipping #{$collectionName}/#{$benchmark}"
998 end
999
1000 def largeHeap
1001     if $memoryLimited
1002         $didAddRunCommand = true
1003         puts "Skipping #{$collectionName}/#{$benchmark}"
1004     end
1005 end
1006
1007 def allJSFiles(path)
1008     if path.file?
1009         [path]
1010     else
1011         result = []
1012         Dir.foreach(path) {
1013             | filename |
1014             next unless filename =~ /\.js$/
1015             next unless (path + filename).file?
1016             result << path + filename
1017         }
1018         result
1019     end
1020 end
1021
1022 def uniqueifyName(names, name)
1023     result = name.to_s
1024     toAdd = 1
1025     while names[result]
1026         result = "#{name}-#{toAdd}"
1027         toAdd += 1
1028     end
1029     names[result] = true
1030     result
1031 end
1032
1033 def simplifyCollectionName(collectionPath)
1034     outerDir = collectionPath.dirname
1035     name = collectionPath.basename
1036     lastName = name
1037     if collectionPath.directory?
1038         while lastName.to_s =~ /test/
1039             lastName = outerDir.basename
1040             name = lastName + name
1041             outerDir = outerDir.dirname
1042         end
1043     end
1044     uniqueifyName($collectionNames, name)
1045 end
1046
1047 def prepareCollection(name)
1048     FileUtils.mkdir_p $outputDir + name
1049
1050     absoluteCollection = $collection.realpath
1051
1052     Dir.chdir($outputDir) {
1053         bundleDir = baseDirForCollection(name)
1054
1055         # Create the proper directory structures.
1056         FileUtils.mkdir_p bundleDir
1057         if bundleDir.basename == $collection.basename
1058             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1059         else
1060             FileUtils.cp_r absoluteCollection, bundleDir
1061         end
1062
1063         $extraFilesBaseDir = absoluteCollection
1064
1065         # Redirect the collection's location to the newly constructed bundle.
1066         if absoluteCollection.directory?
1067             $collection = bundleDir
1068         else
1069             $collection = bundleDir + $collection.basename
1070         end
1071     }
1072 end
1073
1074 $collectionNames = {}
1075
1076 def handleCollectionFile(collection)
1077     collectionName = simplifyCollectionName(collection)
1078    
1079     paths = {}
1080     subCollections = []
1081     YAML::load(IO::read(collection)).each {
1082         | entry |
1083         if entry["collection"]
1084             subCollections << entry["collection"]
1085             next
1086         end
1087         
1088         if Pathname.new(entry["path"]).absolute?
1089             raise "Absolute path: " + entry["path"] + " in #{collection}"
1090         end
1091         
1092         if paths[entry["path"]]
1093             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1094         end
1095         
1096         subCollection = collection.dirname + entry["path"]
1097         
1098         if subCollection.file?
1099             subCollectionName = Pathname.new(entry["path"]).dirname
1100         else
1101             subCollectionName = entry["path"]
1102         end
1103         
1104         $collection = subCollection
1105         $collectionName = Pathname.new(collectionName)
1106         Pathname.new(subCollectionName).each_filename {
1107             | filename |
1108             next if filename =~ /^\./
1109             $collectionName += filename
1110         }
1111         $collectionName = $collectionName.to_s
1112         
1113         prepareCollection($collectionName)
1114       
1115         Dir.chdir($outputDir) {
1116             directoryToSearch = $collection
1117             if entry["tests"]
1118                 directoryToSearch += entry["tests"]
1119             end
1120             allJSFiles(directoryToSearch).each {
1121                 | path |
1122                
1123                 $benchmark = path.basename
1124                 $benchmarkDirectory = path.dirname
1125                 
1126                 $runCommandOptions = {}
1127                 eval entry["cmd"]
1128             }
1129         }
1130     }
1131     
1132     subCollections.each {
1133         | subCollection |
1134         handleCollection(collection.dirname + subCollection)
1135     }
1136 end
1137
1138 def handleCollectionDirectory(collection)
1139     collectionName = simplifyCollectionName(collection)
1140     
1141     $collection = collection
1142     $collectionName = collectionName
1143     prepareCollection(collectionName)
1144    
1145     Dir.chdir($outputDir) {
1146         $benchmarkDirectory = $collection
1147         allJSFiles($collection).each {
1148             | path |
1149             
1150             $benchmark = path.basename
1151             
1152             $runCommandOptions = {}
1153             defaultRun unless parseRunCommands
1154         }
1155     }
1156 end
1157
1158 def handleCollection(collection)
1159     collection = Pathname.new(collection)
1160     
1161     if collection.file?
1162         handleCollectionFile(collection)
1163     else
1164         handleCollectionDirectory(collection)
1165     end
1166 end
1167
1168 def appendFailure(plan)
1169     File.open($outputDir + "failed", "a") {
1170         | outp |
1171         outp.puts plan.name
1172     }
1173     $numFailures += 1
1174 end
1175
1176 def prepareBundle
1177     raise if $bundle
1178
1179     if $doNotMessWithVMPath
1180         if !$remote and !$tarball
1181             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1182             $jscPath = Pathname.new($jscPath).realpath
1183         else
1184             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1185         end
1186     else
1187         originalJSCPath = $jscPath
1188         vmDir = $outputDir + ".vm"
1189         FileUtils.mkdir_p vmDir
1190         
1191         frameworkPath = frameworkFromJSCPath($jscPath)
1192         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1193         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1194         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1195
1196         if frameworkPath
1197             source = frameworkPath
1198             destination = Pathname.new(".vm")
1199         else
1200             source = originalJSCPath
1201             destination = $jscPath
1202
1203             Dir.chdir($outputDir) {
1204                 FileUtils.mkdir_p $jscPath.dirname
1205             }
1206         end
1207
1208         Dir.chdir($outputDir) {
1209             if $copyVM
1210                 FileUtils.cp_r source, destination
1211             else
1212                 begin 
1213                     FileUtils.ln_s source, destination
1214                 rescue Exception
1215                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1216                     FileUtils.cp_r source, destination
1217                 end
1218             end
1219
1220             if $remote and $hostOS == "linux"
1221                 if $enableFTL
1222                     begin
1223                         FileUtils.cp_r originalJSCPath + "../../lib/libllvmForJSC.so" , $jscPath.dirname
1224                     rescue
1225                         $stderr.puts "Warning: unable to copy libllvmForJSC.so to the bundle."
1226                     end
1227                 end
1228                 begin
1229                     dependencies = `ldd #{source}`
1230                     dependencies.split(/\n/).each {
1231                         | dependency |
1232                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1233                     }
1234                 rescue
1235                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1236                 end
1237             end
1238         }
1239     end
1240     
1241     Dir.chdir($outputDir) {
1242         FileUtils.cp_r HELPERS_PATH, ".helpers"
1243     }
1244
1245     ARGV.each {
1246         | collection |
1247         handleCollection(collection)
1248     }
1249
1250     puts
1251 end
1252
1253 def cleanOldResults
1254     raise unless $bundle
1255
1256     eachResultFile($outputDir) {
1257         | path |
1258         FileUtils.rm_f path
1259     }
1260 end
1261
1262 def cleanEmptyResultFiles
1263     eachResultFile($outputDir) {
1264         | path |
1265         next unless path.basename.to_s =~ /\.out$/
1266         next unless FileTest.size(path) == 0
1267         FileUtils.rm_f path
1268     }
1269 end
1270
1271 def eachResultFile(startingDir, &block)
1272     dirsToClean = [startingDir]
1273     until dirsToClean.empty?
1274         nextDir = dirsToClean.pop
1275         Dir.foreach(nextDir) {
1276             | entry |
1277             next if entry =~ /^\./
1278             path = nextDir + entry
1279             if path.directory?
1280                 dirsToClean.push(path)
1281             else
1282                 block.call(path)
1283             end
1284         }
1285     end
1286 end
1287
1288 def prepareTestRunner
1289     raise if $bundle
1290
1291     $runlist.each_with_index {
1292         | plan, index |
1293         plan.index = index
1294     }
1295
1296     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1297     toDelete = []
1298     Dir.foreach($runnerDir) {
1299         | filename |
1300         if filename =~ /^test_/
1301             toDelete << filename
1302         end
1303     }
1304     
1305     toDelete.each {
1306         | filename |
1307         File.unlink($runnerDir + filename)
1308     }
1309
1310     $runlist.each {
1311         | plan |
1312         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1313     }
1314
1315     case $testRunnerType
1316     when :make
1317         prepareMakeTestRunner
1318     when :shell
1319         prepareShellTestRunner
1320     else
1321         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1322     end
1323 end
1324
1325 def prepareShellTestRunner
1326     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
1327 end
1328
1329 def prepareMakeTestRunner
1330     # The goals of our parallel test runner are scalability and simplicity. The
1331     # simplicity part is particularly important. We don't want to have to have
1332     # a full-time contributor just philosophising about parallel testing.
1333     #
1334     # As such, we just pass off all of the hard work to 'make'. This creates a
1335     # dummy directory ("$outputDir/.runner") in which we create a dummy
1336     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
1337     # That is, for each test we know we will run, there is a rule in the
1338     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
1339     # Makefile results in 'make' doing all of the hard work:
1340     #
1341     # - Load balancing just works. Most systems have a great load balancer in
1342     #   'make'. If your system doesn't then just install a real 'make'.
1343     #
1344     # - Interruptions just work. For example Ctrl-C handling in 'make' is
1345     #   exactly right. You don't have to worry about zombie processes.
1346     #
1347     # We then do some tricks to make failure detection work and to make this
1348     # totally sound. If a test fails, we don't want the whole 'make' job to
1349     # stop. We also don't have any facility for makefile-escaping of path names.
1350     # We do have such a thing for shell-escaping, though. We fix both problems
1351     # by having the actual work for each of the test rules be done in a shell
1352     # script on the side. There is one such script per test. The script responds
1353     # to failure by printing something on the console and then touching a
1354     # failure file for that test, but then still returns 0. This makes 'make'
1355     # continue past that failure and complete all the tests anyway.
1356     #
1357     # In the end, this script collects all of the failures by searching for
1358     # files in the .runner directory whose name matches /^test_fail_/, where
1359     # the thing after the 'fail_' is the test index. Those are the files that
1360     # would be created by the test scripts if they detect failure. We're
1361     # basically using the filesystem as a concurrent database of test failures.
1362     # Even if two tests fail at the same time, since they're touching different
1363     # files we won't miss any failures.
1364     runIndices = []
1365     $runlist.each {
1366         | plan |
1367         runIndices << plan.index
1368     }
1369     
1370     File.open($runnerDir + "Makefile", "w") {
1371         | outp |
1372         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1373         runIndices.each {
1374             | index |
1375             plan = $runlist[index]
1376             outp.puts "test_done_#{index}:"
1377             outp.puts "\tsh test_script_#{plan.index}"
1378         }
1379     }
1380 end
1381
1382     
1383 puts
1384
1385 def cleanRunnerDirectory
1386     raise unless $bundle
1387     Dir.foreach($runnerDir) {
1388         | filename |
1389         next unless filename =~ /^test_fail/
1390         FileUtils.rm_f $runnerDir + filename
1391     }
1392 end
1393
1394 def sshRead(cmd)
1395     raise unless $remote
1396
1397     result = ""
1398     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1399       | inp |
1400       inp.each_line {
1401         | line |
1402         result += line
1403       }
1404     }
1405     raise "#{$?}" unless $?.success?
1406     result
1407 end
1408
1409 def runCommandOnTester(cmd)
1410     if $remote
1411         result = sshRead(cmd)
1412     else
1413         result = `#{cmd}`
1414     end
1415 end
1416
1417 def numberOfProcessors
1418     begin
1419         numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1420     rescue
1421         numProcessors = 0
1422     end
1423
1424     if numProcessors == 0
1425         begin
1426             numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1427         rescue
1428             numProcessors == 0
1429         end
1430     end
1431
1432     if numProcessors == 0
1433         numProcessors = 1
1434     end
1435     return numProcessors
1436 end
1437
1438 def runAndMonitorTestRunnerCommand(*cmd)
1439     numberOfTests = 0
1440     Dir.chdir($runnerDir) {
1441         # -1 for the runscript, and -2 for '..' and '.'
1442         numberOfTests = Dir.entries(".").count - 3
1443     }
1444     unless $progressMeter
1445         mysys(cmd.join(' '))
1446     else
1447        running = {}
1448        didRun = {}
1449        didFail = {}
1450        blankLine = true
1451        prevStringLength = 0
1452        IO.popen(cmd.join(' '), mode="r") {
1453            | inp |
1454            inp.each_line {
1455                | line |
1456                line.chomp!
1457                if line =~ /^Running /
1458                    running[$~.post_match] = true
1459                elsif line =~ /^PASS: /
1460                    didRun[$~.post_match] = true
1461                elsif line =~ /^FAIL: /
1462                    didRun[$~.post_match] = true
1463                    didFail[$~.post_match] = true
1464                else
1465                    unless blankLine
1466                        print("\r" + " " * prevStringLength + "\r")
1467                    end
1468                    puts line
1469                    blankLine = true
1470                end
1471
1472                def lpad(str, chars)
1473                    str = str.to_s
1474                    if str.length > chars
1475                        str
1476                    else
1477                       "%#{chars}s"%(str)
1478                    end
1479                end
1480
1481                string  = ""
1482                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1483                unless didFail.empty?
1484                    string += " (failed #{didFail.size})"
1485                end
1486                string += " "
1487                (running.size - didRun.size).times {
1488                    string += "."
1489                }
1490                if string.length < prevStringLength
1491                    print string
1492                    print(" " * (prevStringLength - string.length))
1493                end
1494                print string
1495                prevStringLength = string.length
1496                blankLine = false
1497                $stdout.flush
1498            }
1499        }
1500        puts
1501        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1502     end
1503 end
1504
1505 def runTestRunner
1506     case $testRunnerType
1507     when :shell
1508         testRunnerCommand = "sh runscript"
1509     when :make
1510         testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
1511     else
1512         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1513     end
1514
1515     if $remote
1516         if !$remoteDirectory
1517             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1518         end
1519         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1520         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1521         remoteScript = "\""
1522         remoteScript += "cd #{$remoteDirectory} && "
1523         remoteScript += "rm -rf #{$outputDir.basename} && "
1524         remoteScript += "tar xzf #{$tarFileName} && "
1525         remoteScript += "cd #{$outputDir.basename}/.runner && "
1526         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1527         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1528         remoteScript += "export JSC_timeout=#{Shellwords.shellescape(ENV['JSC_timeout'])} && "
1529         remoteScript += "#{testRunnerCommand}\""
1530         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1531     else
1532         Dir.chdir($runnerDir) {
1533             runAndMonitorTestRunnerCommand(testRunnerCommand)
1534         }
1535     end
1536 end
1537
1538 def detectFailures
1539     raise if $bundle
1540
1541     if $remote
1542         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1543         output.split(/\n/).each {
1544             | line |
1545             next unless line =~ /test_fail_/
1546             appendFailure($runlist[$~.post_match.to_i])
1547         }
1548     else
1549         Dir.foreach($runnerDir) {
1550             | filename |
1551             next unless filename =~ /test_fail_/
1552             appendFailure($runlist[$~.post_match.to_i])
1553         }
1554     end
1555 end
1556
1557 def compressBundle
1558     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
1559     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1560     raise unless system(cmd)
1561 end
1562
1563 def clean(file)
1564     FileUtils.rm_rf file unless $bundle
1565 end
1566
1567 clean($outputDir + "failed")
1568 clean($outputDir + ".vm")
1569 clean($outputDir + ".helpers")
1570 clean($outputDir + ".runner")
1571 clean($outputDir + ".tests")
1572 clean($outputDir + "_payload")
1573
1574 Dir.mkdir($outputDir) unless $outputDir.directory?
1575
1576 $outputDir = $outputDir.realpath
1577 $runnerDir = $outputDir + ".runner"
1578
1579 if !$numChildProcesses
1580     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
1581         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
1582     else
1583         $numChildProcesses = numberOfProcessors
1584     end
1585 end
1586
1587 if $enableFTL and ENV["JSC_timeout"]
1588     # Currently, using the FTL is a performance regression particularly in real
1589     # (i.e. non-loopy) benchmarks. Account for this in the timeout.
1590     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i * 2).to_s
1591 end
1592
1593 if ENV["JSC_timeout"]
1594     # In the worst case, the processors just interfere with each other.
1595     # Increase the timeout proportionally to the number of processors.
1596     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
1597 end
1598
1599 def runBundle
1600     raise unless $bundle
1601
1602     cleanRunnerDirectory
1603     cleanOldResults
1604     runTestRunner
1605     cleanEmptyResultFiles
1606 end
1607
1608 def runNormal
1609     raise if $bundle or $tarball
1610
1611     prepareBundle
1612     prepareTestRunner
1613     runTestRunner
1614     cleanEmptyResultFiles
1615     detectFailures
1616 end
1617
1618 def runTarball
1619     raise unless $tarball
1620
1621     prepareBundle 
1622     prepareTestRunner
1623     compressBundle
1624 end
1625
1626 def runRemote
1627     raise unless $remote
1628
1629     prepareBundle
1630     prepareTestRunner
1631     compressBundle
1632     runTestRunner
1633     detectFailures
1634 end
1635
1636 if $bundle
1637     runBundle
1638 elsif $remote
1639     runRemote
1640 elsif $tarball
1641     runTarball
1642 else
1643     runNormal
1644 end