95f3f97b75a122e8d5413a607f0802b226497d61
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013, 2014 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
41 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
42 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
43 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
44 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
45 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
46
47 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
48
49 begin
50     require 'shellwords'
51 rescue Exception => e
52     $stderr.puts "Warning: did not find shellwords, not running any tests."
53     exit 0
54 end
55
56 $canRunDisplayProfilerOutput = false
57
58 begin
59     require 'rubygems'
60     require 'json'
61     require 'highline'
62     $canRunDisplayProfilerOutput = true
63 rescue Exception => e
64     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
65     $stderr.puts "Error: #{e.inspect}"
66 end
67
68 def printCommandArray(*cmd)
69     begin
70         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
71     rescue
72         commandArray = cmd.join(' ')
73     end
74     $stderr.puts ">> #{commandArray}"
75 end
76
77 def mysys(*cmd)
78     printCommandArray(*cmd) if $verbosity >= 1
79     raise "Command failed: #{$?.inspect}" unless system(*cmd)
80 end
81
82 def escapeAll(array)
83     array.map {
84         | v |
85         raise "Detected a non-string in #{inspect}" unless v.is_a? String
86         Shellwords.shellescape(v)
87     }.join(' ')
88 end
89
90
91 $jscPath = nil
92 $doNotMessWithVMPath = false
93 $enableFTL = false
94 $memoryLimited = false
95 $outputDir = Pathname.new("results")
96 $verbosity = 0
97 $bundle = nil
98 $tarball = false
99 $tarFileName = "payload.tar.gz"
100 $copyVM = false
101 $testRunnerType = nil
102 $remoteUser = nil
103 $remoteHost = nil
104 $remotePort = nil
105 $remoteDirectory = nil
106 $architecture = nil
107 $hostOS = nil
108
109
110 def usage
111     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
112     puts
113     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
114     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
115     puts "                            --jsc specifies an already present JavaScriptCore to test."
116     puts "--ftl-jit                   Indicate that we have the FTL JIT."
117     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
118     puts "                            Skip tests tagged with //@large-heap"
119     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
120     puts "--verbose            (-v)   Print more things while running."
121     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
122     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
123     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
124     puts "                            e.g. x86, x86_64, arm."
125     puts "--os                        Specify os instead of determining from JavaScriptCore build."
126     puts "                            e.g. darwin, linux & windows."
127     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
128     puts "                            In general the shell runner is slower than the make runner."
129     puts "--make-runner               Uses the faster make-based runner."
130     puts "--remote                    Specify a remote host on which to run tests from command line argument."
131     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
132     puts "--child-processes    (-c)   Specify the number of child processes."
133     puts "--help               (-h)   Print this message."
134     exit 1
135 end
136
137 jscArg = nil
138
139 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
140                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
141                ['--no-copy', GetoptLong::NO_ARGUMENT],
142                ['--ftl-jit', GetoptLong::NO_ARGUMENT],
143                ['--memory-limited', GetoptLong::NO_ARGUMENT],
144                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
145                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
146                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
147                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
148                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
149                ['--os', GetoptLong::REQUIRED_ARGUMENT],
150                ['--shell-runner', GetoptLong::NO_ARGUMENT],
151                ['--make-runner', GetoptLong::NO_ARGUMENT],
152                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
153                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
154                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
155                ['--verbose', '-v', GetoptLong::NO_ARGUMENT]).each {
156     | opt, arg |
157     case opt
158     when '--help'
159         usage
160     when '--jsc'
161         jscArg = arg
162     when '--no-copy'
163         $doNotMessWithVMPath = true
164     when '--output-dir'
165         $outputDir = Pathname.new(arg)
166     when '--ftl-jit'
167         $enableFTL = true
168     when '--memory-limited'
169         $memoryLimited = true
170     when '--verbose'
171         $verbosity += 1
172     when '--run-bundle'
173         $bundle = Pathname.new(arg)
174     when '--tarball'
175         $tarball = true
176         $copyVM = true
177         $tarFileName = arg unless arg == ''
178     when '--force-vm-copy'
179         $copyVM = true
180     when '--shell-runner'
181         $testRunnerType = :shell
182     when '--make-runner'
183         $testRunnerType = :make
184     when '--remote'
185         $copyVM = true
186         $tarball = true
187         $remote = true
188         uri = URI("ssh://" + arg)
189         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
190     when '--remote-config-file'
191         $remoteConfigFile = arg
192     when '--child-processes'
193         $numChildProcesses = arg.to_i
194     when '--arch'
195         $architecture = arg
196     when '--os'
197         $hostOS = arg
198     end
199 }
200
201 if $remoteConfigFile
202     file = File.read($remoteConfigFile)
203     config = JSON.parse(file)
204
205     if !$remote and config['remote']
206         $copyVM = true
207         $tarball = true
208         $remote = true
209         uri = URI("ssh://" + config['remote'])
210         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
211     end
212
213     if config['remoteDirectory']
214         $remoteDirectory = config['remoteDirectory']
215     end
216 end
217
218 unless jscArg
219     $stderr.puts "Error: must specify -jsc <path>"
220     exit 1
221 end
222
223 if $doNotMessWithVMPath
224     $jscPath = Pathname.new(jscArg)
225 else
226     $jscPath = Pathname.new(jscArg).realpath
227 end
228
229 $progressMeter = ($verbosity == 0 and $stdout.tty?)
230
231 if $bundle
232     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
233     $outputDir = $bundle
234 end
235
236 # Try to determine architecture. Return nil on failure.
237 def machOArchitectureCode
238     begin 
239         otoolLines = `otool -aSfh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
240         otoolLines.each_with_index {
241             | value, index |
242             if value =~ /magic/ and value =~ /cputype/
243                 return otoolLines[index + 1].split[1].to_i
244             end
245         }
246     rescue
247         $stderr.puts "Warning: unable to execute otool."
248     end
249     $stderr.puts "Warning: unable to determine architecture."
250     nil
251 end
252
253 def determineArchitecture
254     code = machOArchitectureCode
255     return nil unless code
256     is64BitFlag = 0x01000000
257     case code
258     when 7
259         "x86"
260     when 7 | is64BitFlag
261         "x86-64"
262     when 12
263         "arm"
264     when 12 | is64BitFlag
265         "arm64"
266     else
267         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
268         nil
269     end
270 end
271
272 def determineOS
273     case RbConfig::CONFIG["host_os"]
274     when /darwin/i
275         "darwin"
276     when /linux/i
277         "linux"
278     when /mswin|mingw|cygwin/
279         "windows"
280     else
281         $stderr.puts "Warning: unable to determine host operating system"
282         nil
283     end
284 end
285
286 $architecture = determineArchitecture unless $architecture
287 $hostOS = determineOS unless $hostOS
288
289 if !$testRunnerType
290     if $remote and $hostOS == "darwin"
291         $testRunnerType = :shell
292     else
293         $testRunnerType = :make
294     end
295 end
296
297 $numFailures = 0
298
299 BASE_OPTIONS = ["--useFTLJIT=false", "--enableFunctionDotArguments=true"]
300 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000"]
301 NO_CJIT_OPTIONS = ["--enableConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100"]
302 FTL_OPTIONS = ["--useFTLJIT=true"]
303
304 $runlist = []
305
306 def frameworkFromJSCPath(jscPath)
307     parentDirectory = jscPath.dirname
308     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
309         parentDirectory.dirname
310     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
311         jscPath.dirname + "JavaScriptCore.framework"
312     else
313         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
314         nil
315     end
316 end
317
318 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
319     dir = Pathname.new(".")
320     $benchmarkDirectory.each_filename {
321         | pathComponent |
322         dir += ".."
323     }
324     dir + resourcePath
325 end
326
327 def pathToVM
328     pathToBundleResourceFromBenchmarkDirectory($jscPath)
329 end
330
331 def pathToHelpers
332     pathToBundleResourceFromBenchmarkDirectory(".helpers")
333 end
334
335 def prefixCommand(prefix)
336     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
337 end
338
339 def redirectAndPrefixCommand(prefix)
340     prefixCommand(prefix) + " 2>&1"
341 end
342
343 def pipeAndPrefixCommand(outputFilename, prefix)
344     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
345 end
346
347 # Output handler for tests that are expected to be silent.
348 def silentOutputHandler
349     Proc.new {
350         | name |
351         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
352     }
353 end
354
355 # Output handler for tests that are expected to produce meaningful output.
356 def noisyOutputHandler
357     Proc.new {
358         | name |
359         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
360     }
361 end
362
363 # Error handler for tests that fail exactly when they return non-zero exit status.
364 def simpleErrorHandler
365     Proc.new {
366         | outp, plan |
367         outp.puts "if test -e #{plan.failFile}"
368         outp.puts "then"
369         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
370         outp.puts "    " + plan.failCommand
371         outp.puts "else"
372         outp.puts "    " + plan.successCommand
373         outp.puts "fi"
374     }
375 end
376
377 # Error handler for tests that diff their output with some expectation.
378 def diffErrorHandler(expectedFilename)
379     Proc.new {
380         | outp, plan |
381         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
382         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
383         
384         outp.puts "if test -e #{plan.failFile}"
385         outp.puts "then"
386         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
387         outp.puts "    " + plan.failCommand
388         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
389         outp.puts "then"
390         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
391         outp.puts "    if [ $? -eq 0 ]"
392         outp.puts "    then"
393         outp.puts "    " + plan.successCommand
394         outp.puts "    else"
395         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
396         outp.puts "        " + plan.failCommand
397         outp.puts "    fi"
398         outp.puts "else"
399         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
400         outp.puts "    " + plan.failCommand
401         outp.puts "fi"
402     }
403 end
404
405 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
406 # tests.
407 def mozillaErrorHandler
408     Proc.new {
409         | outp, plan |
410         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
411
412         outp.puts "if test -e #{plan.failFile}"
413         outp.puts "then"
414         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
415         outp.puts "    " + plan.failCommand
416         outp.puts "elif grep -i -q failed! #{outputFilename}"
417         outp.puts "then"
418         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
419         outp.puts "    " + plan.failCommand
420         outp.puts "else"
421         outp.puts "    " + plan.successCommand
422         outp.puts "fi"
423     }
424 end
425
426 # Error handler for tests that report error by saying "failed!", and are expected to
427 # fail. This is used by Mozilla tests.
428 def mozillaFailErrorHandler
429     Proc.new {
430         | outp, plan |
431         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
432
433         outp.puts "if test -e #{plan.failFile}"
434         outp.puts "then"
435         outp.puts "    " + plan.successCommand
436         outp.puts "elif grep -i -q failed! #{outputFilename}"
437         outp.puts "then"
438         outp.puts "    " + plan.successCommand
439         outp.puts "else"
440         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
441         outp.puts "    " + plan.failCommand
442         outp.puts "fi"
443     }
444 end
445
446 # Error handler for tests that report error by saying "failed!", and are expected to have
447 # an exit code of 3.
448 def mozillaExit3ErrorHandler
449     Proc.new {
450         | outp, plan |
451         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
452
453         outp.puts "if test -e #{plan.failFile}"
454         outp.puts "then"
455         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
456         outp.puts "    then"
457         outp.puts "        if grep -i -q failed! #{outputFilename}"
458         outp.puts "        then"
459         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
460         outp.puts "            " + plan.failCommand
461         outp.puts "        else"
462         outp.puts "            " + plan.successCommand
463         outp.puts "        fi"
464         outp.puts "    else"
465         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
466         outp.puts "        " + plan.failCommand
467         outp.puts "    fi"
468         outp.puts "else"
469         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
470         outp.puts "    " + plan.failCommand
471         outp.puts "fi"
472     }
473 end
474
475 $runCommandOptions = {}
476
477 class Plan
478     attr_reader :directory, :arguments, :name, :outputHandler, :errorHandler
479     attr_accessor :index
480     
481     def initialize(directory, arguments, name, outputHandler, errorHandler)
482         @directory = directory
483         @arguments = arguments
484         @name = name
485         @outputHandler = outputHandler
486         @errorHandler = errorHandler
487         @isSlow = !!$runCommandOptions[:isSlow]
488     end
489     
490     def shellCommand
491         # It's important to remember that the test is actually run in a subshell, so if we change directory
492         # in the subshell when we return we will be in our original directory. This is nice because we don't
493         # have to bend over backwards to do things relative to the root.
494         "(cd ../#{Shellwords.shellescape(@directory.to_s)} && \"$@\" " + escapeAll(@arguments) + ")"
495     end
496     
497     def reproScriptCommand
498         # We have to find our way back to the .runner directory since that's where all of the relative
499         # paths assume they start out from.
500         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
501         script += "cd $CURRENT_DIR\n"
502         Pathname.new(@name).dirname.each_filename {
503             | pathComponent |
504             script += "cd ..\n"
505         }
506         script += "cd .runner\n"
507
508         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
509         script += "export JSC_timeout=#{Shellwords.shellescape(ENV['JSC_timeout'])}\n"
510         script += "#{shellCommand} || exit 1"
511         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
512     end
513     
514     def failCommand
515         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
516     end
517     
518     def successCommand
519         if $progressMeter or $verbosity >= 2
520             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
521         else
522             "rm -f #{failFile}"
523         end
524     end
525     
526     def failFile
527         "test_fail_#{@index}"
528     end
529     
530     def writeRunScript(filename)
531         File.open(filename, "w") {
532             | outp |
533             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
534             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
535             cmd += @outputHandler.call(@name)
536             if $verbosity >= 3
537                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
538             end
539             outp.puts cmd
540             @errorHandler.call(outp, self)
541         }
542     end
543 end
544
545 $uniqueFilenameCounter = 0
546 def uniqueFilename(extension)
547     payloadDir = $outputDir + "_payload"
548     Dir.mkdir payloadDir unless payloadDir.directory?
549     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
550     $uniqueFilenameCounter += 1
551     result
552 end
553
554 def baseOutputName(kind)
555     "#{$collectionName}/#{$benchmark}.#{kind}"
556 end
557
558 def addRunCommand(kind, command, outputHandler, errorHandler)
559     $didAddRunCommand = true
560     plan = Plan.new($benchmarkDirectory, command, baseOutputName(kind), outputHandler, errorHandler)
561     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
562         $runlist.unshift plan
563     else
564         $runlist << plan
565     end
566 end
567
568 # Returns true if there were run commands found in the file ($benchmarkDirectory +
569 # $benchmark), in which case those run commands have already been executed. Otherwise
570 # returns false, in which case you're supposed to add your own run commands.
571 def parseRunCommands
572     oldDidAddRunCommand = $didAddRunCommand
573     $didAddRunCommand = false
574
575     Dir.chdir($outputDir) {
576         File.open($benchmarkDirectory + $benchmark) {
577             | inp |
578             inp.each_line {
579                 | line |
580                 begin
581                     doesMatch = line =~ /^\/\/@/
582                 rescue Exception => e
583                     # Apparently this happens in the case of some UTF8 stuff in some files, where
584                     # Ruby tries to be strict and throw exceptions.
585                     next
586                 end
587                 next unless doesMatch
588                 eval $~.post_match
589             }
590         }
591     }
592
593     result = $didAddRunCommand
594     $didAddRunCommand = result or oldDidAddRunCommand
595     result
596 end
597
598 def slow!
599     $runCommandOptions[:isSlow] = true
600 end
601
602 def run(kind, *options)
603     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
604 end
605
606 def runDefault
607     run("default")
608 end
609
610 def runNoLLInt
611     run("no-llint", "--useLLInt=false")
612 end
613
614 def runNoCJITValidate
615     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *NO_CJIT_OPTIONS)
616 end
617
618 def runNoCJITValidatePhases
619     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
620 end
621
622 def runDefaultFTL
623     run("default-ftl", *FTL_OPTIONS) if $enableFTL
624 end
625
626 def runFTLNoCJIT
627     run("ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
628 end
629
630 def runFTLNoCJITValidate
631     run("ftl-no-cjit-validate", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
632 end
633
634 def runFTLNoCJITNoInlineValidate
635     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
636 end
637
638 def runFTLNoCJITOSRValidation
639     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
640 end
641
642 def runDFGEager
643     run("dfg-eager", *EAGER_OPTIONS)
644 end
645
646 def runDFGEagerNoCJITValidate
647     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
648 end
649
650 def runFTLEager
651     run("ftl-eager", *(FTL_OPTIONS + EAGER_OPTIONS)) if $enableFTL
652 end
653
654 def runFTLEagerNoCJITValidate
655     run("ftl-eager-no-cjit", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
656 end
657
658 def runFTLEagerNoCJITOSRValidation
659     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
660 end
661
662 def runAlwaysTriggerCopyPhase
663     run("always-trigger-copy-phase", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
664 end
665
666 def runNoCJITNoASO
667     run("no-cjit-no-aso", "--enableArchitectureSpecificOptimizations=false", *NO_CJIT_OPTIONS)
668 end
669
670 def runFTLNoCJITNoSimpleOpt
671     run("ftl-no-cjit-no-simple-opt", "--llvmSimpleOpt=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
672 end
673
674 def runNoCJITNoAccessInlining
675     run("no-cjit-no-access-inlining", "--enableAccessInlining=false", *NO_CJIT_OPTIONS)
676 end
677
678 def runFTLNoCJITNoAccessInlining
679     run("ftl-no-cjit-no-access-inlining", "--enableAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
680 end
681
682 def defaultRun
683     runDefault
684     runNoLLInt
685     runAlwaysTriggerCopyPhase
686     runNoCJITValidatePhases
687     runDFGEager
688     runDFGEagerNoCJITValidate
689     runDefaultFTL
690     runFTLNoCJITValidate
691     runFTLNoCJITNoInlineValidate
692     runFTLEager
693     runFTLEagerNoCJITValidate
694 end
695
696 def defaultQuickRun
697     if $enableFTL
698         runDefaultFTL
699         runFTLNoCJITValidate
700     else
701         runDefault
702         runNoCJITValidate
703     end
704 end
705
706 def defaultSpotCheck
707     defaultQuickRun
708     runFTLNoCJITNoSimpleOpt
709     runFTLNoCJITOSRValidation
710     runNoCJITNoAccessInlining
711     runFTLNoCJITNoAccessInlining
712 end
713
714 # This is expected to not do eager runs because eager runs can have a lot of recompilations
715 # for reasons that don't arise in the real world. It's used for tests that assert convergence
716 # by counting recompilations.
717 def defaultNoEagerRun
718     runDefault
719     runNoLLInt
720     runAlwaysTriggerCopyPhase
721     runNoCJITValidatePhases
722     runDefaultFTL
723     runFTLNoCJITValidate
724     runFTLNoCJITNoInlineValidate
725 end
726
727 def runProfiler
728     if $remote
729         skip
730         return
731     end
732
733     profilerOutput = uniqueFilename(".json")
734     if $canRunDisplayProfilerOutput
735         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
736     else
737         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
738         run("profiler-simple", "-p", profilerOutput.to_s)
739     end
740 end
741
742 def runExceptionFuzz
743     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
744     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
745 end
746
747 def runTypeProfiler
748     if $enableFTL
749         run("ftl-no-cjit-type-profiler", "--enableTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
750     else
751         run("no-cjit-type-profiler", "--enableTypeProfiler=true", *NO_CJIT_OPTIONS)
752     end
753 end
754
755 def runControlFlowProfiler
756     if $enableFTL
757         run("ftl-no-cjit-type-profiler", "--enableControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
758     else
759         run("no-cjit-type-profiler", "--enableControlFlowProfiler=true", *NO_CJIT_OPTIONS)
760     end
761 end
762
763 def runLayoutTest(kind, *options)
764     raise unless $benchmark.to_s =~ /\.js$/
765     testName = $~.pre_match
766     if kind
767         kind = "layout-" + kind
768     else
769         kind = "layout"
770     end
771
772     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
773     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
774
775     args = [pathToVM.to_s] + BASE_OPTIONS + options +
776         [(Pathname.new("resources") + "standalone-pre.js").to_s,
777          $benchmark.to_s,
778          (Pathname.new("resources") + "standalone-post.js").to_s]
779     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
780 end
781
782 def runLayoutTestDefault
783     runLayoutTest(nil)
784 end
785
786 def runLayoutTestNoLLInt
787     runLayoutTest("no-llint", "--useLLInt=false")
788 end
789
790 def runLayoutTestNoCJIT
791     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
792 end
793
794 def runLayoutTestDFGEagerNoCJIT
795     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
796 end
797
798 def runLayoutTestDefaultFTL
799     runLayoutTest("ftl", "--testTheFTL=true", *FTL_OPTIONS) if $enableFTL
800 end
801
802 def runLayoutTestFTLNoCJIT
803     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
804 end
805
806 def runLayoutTestFTLEagerNoCJIT
807     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
808 end
809
810 def noFTLRunLayoutTest
811     runLayoutTestDefault
812     runLayoutTestNoLLInt
813     runLayoutTestNoCJIT
814     runLayoutTestDFGEagerNoCJIT
815 end
816
817 def defaultRunLayoutTest
818     noFTLRunLayoutTest
819     runLayoutTestDefaultFTL
820     runLayoutTestFTLNoCJIT
821     runLayoutTestFTLEagerNoCJIT
822 end
823
824 def prepareExtraRelativeFiles(extraFiles, destination)
825     Dir.chdir($outputDir) {
826         extraFiles.each {
827             | file |
828             FileUtils.cp $extraFilesBaseDir + file, destination + file
829         }
830     }
831 end
832
833 def baseDirForCollection(collectionName)
834     Pathname(".tests") + collectionName
835 end
836
837 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
838     raise unless absoluteBase.absolute?
839     Dir.chdir($outputDir) {
840         collectionBaseDir = baseDirForCollection($collectionName)
841         extraFiles.each {
842             | file |
843             destination = collectionBaseDir + file
844             FileUtils.mkdir_p destination.dirname unless destination.directory?
845             FileUtils.cp absoluteBase + file, destination
846         }
847     }
848 end
849
850 def runMozillaTest(kind, mode, extraFiles, *options)
851     if kind
852         kind = "mozilla-" + kind
853     else
854         kind = "mozilla"
855     end
856     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
857     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
858     case mode
859     when :normal
860         errorHandler = mozillaErrorHandler
861     when :negative
862         errorHandler = mozillaExit3ErrorHandler
863     when :fail
864         errorHandler = mozillaFailErrorHandler
865     when :skip
866         return
867     else
868         raise "Invalid mode: #{mode}"
869     end
870     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
871 end
872
873 def runMozillaTestDefault(mode, *extraFiles)
874     runMozillaTest(nil, mode, extraFiles)
875 end
876
877 def runMozillaTestDefaultFTL(mode, *extraFiles)
878     runMozillaTest("ftl", mode, extraFiles, *FTL_OPTIONS) if $enableFTL
879 end
880
881 def runMozillaTestLLInt(mode, *extraFiles)
882     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
883 end
884
885 def runMozillaTestBaselineJIT(mode, *extraFiles)
886     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
887 end
888
889 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
890     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
891 end
892
893 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
894     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
895 end
896
897 def defaultRunMozillaTest(mode, *extraFiles)
898     runMozillaTestDefault(mode, *extraFiles)
899     runMozillaTestLLInt(mode, *extraFiles)
900     runMozillaTestBaselineJIT(mode, *extraFiles)
901     runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
902     runMozillaTestDefaultFTL(mode, *extraFiles)
903     runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
904 end
905
906 def skip
907     $didAddRunCommand = true
908     puts "Skipping #{$collectionName}/#{$benchmark}"
909 end
910
911 def largeHeap
912     if $memoryLimited
913         $didAddRunCommand = true
914         puts "Skipping #{$collectionName}/#{$benchmark}"
915     end
916 end
917
918 def allJSFiles(path)
919     if path.file?
920         [path]
921     else
922         result = []
923         Dir.foreach(path) {
924             | filename |
925             next unless filename =~ /\.js$/
926             next unless (path + filename).file?
927             result << path + filename
928         }
929         result
930     end
931 end
932
933 def uniqueifyName(names, name)
934     result = name.to_s
935     toAdd = 1
936     while names[result]
937         result = "#{name}-#{toAdd}"
938         toAdd += 1
939     end
940     names[result] = true
941     result
942 end
943
944 def simplifyCollectionName(collectionPath)
945     outerDir = collectionPath.dirname
946     name = collectionPath.basename
947     lastName = name
948     if collectionPath.directory?
949         while lastName.to_s =~ /test/
950             lastName = outerDir.basename
951             name = lastName + name
952             outerDir = outerDir.dirname
953         end
954     end
955     uniqueifyName($collectionNames, name)
956 end
957
958 def prepareCollection(name)
959     FileUtils.mkdir_p $outputDir + name
960
961     absoluteCollection = $collection.realpath
962
963     Dir.chdir($outputDir) {
964         bundleDir = baseDirForCollection(name)
965
966         # Create the proper directory structures.
967         FileUtils.mkdir_p bundleDir
968         if bundleDir.basename == $collection.basename
969             FileUtils.cp_r absoluteCollection, bundleDir.dirname
970         else
971             FileUtils.cp_r absoluteCollection, bundleDir
972         end
973
974         $extraFilesBaseDir = absoluteCollection
975
976         # Redirect the collection's location to the newly constructed bundle.
977         if absoluteCollection.directory?
978             $collection = bundleDir
979         else
980             $collection = bundleDir + $collection.basename
981         end
982     }
983 end
984
985 $collectionNames = {}
986
987 def handleCollectionFile(collection)
988     collectionName = simplifyCollectionName(collection)
989    
990     paths = {}
991     subCollections = []
992     YAML::load(IO::read(collection)).each {
993         | entry |
994         if entry["collection"]
995             subCollections << entry["collection"]
996             next
997         end
998         
999         if Pathname.new(entry["path"]).absolute?
1000             raise "Absolute path: " + entry["path"] + " in #{collection}"
1001         end
1002         
1003         if paths[entry["path"]]
1004             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1005         end
1006         
1007         subCollection = collection.dirname + entry["path"]
1008         
1009         if subCollection.file?
1010             subCollectionName = Pathname.new(entry["path"]).dirname
1011         else
1012             subCollectionName = entry["path"]
1013         end
1014         
1015         $collection = subCollection
1016         $collectionName = Pathname.new(collectionName)
1017         Pathname.new(subCollectionName).each_filename {
1018             | filename |
1019             next if filename =~ /^\./
1020             $collectionName += filename
1021         }
1022         $collectionName = $collectionName.to_s
1023         
1024         prepareCollection($collectionName)
1025       
1026         Dir.chdir($outputDir) {
1027             directoryToSearch = $collection
1028             if entry["tests"]
1029                 directoryToSearch += entry["tests"]
1030             end
1031             allJSFiles(directoryToSearch).each {
1032                 | path |
1033                
1034                 $benchmark = path.basename
1035                 $benchmarkDirectory = path.dirname
1036                 
1037                 $runCommandOptions = {}
1038                 eval entry["cmd"]
1039             }
1040         }
1041     }
1042     
1043     subCollections.each {
1044         | subCollection |
1045         handleCollection(collection.dirname + subCollection)
1046     }
1047 end
1048
1049 def handleCollectionDirectory(collection)
1050     collectionName = simplifyCollectionName(collection)
1051     
1052     $collection = collection
1053     $collectionName = collectionName
1054     prepareCollection(collectionName)
1055    
1056     Dir.chdir($outputDir) {
1057         $benchmarkDirectory = $collection
1058         allJSFiles($collection).each {
1059             | path |
1060             
1061             $benchmark = path.basename
1062             
1063             $runCommandOptions = {}
1064             defaultRun unless parseRunCommands
1065         }
1066     }
1067 end
1068
1069 def handleCollection(collection)
1070     collection = Pathname.new(collection)
1071     
1072     if collection.file?
1073         handleCollectionFile(collection)
1074     else
1075         handleCollectionDirectory(collection)
1076     end
1077 end
1078
1079 def appendFailure(plan)
1080     File.open($outputDir + "failed", "a") {
1081         | outp |
1082         outp.puts plan.name
1083     }
1084     $numFailures += 1
1085 end
1086
1087 def prepareBundle
1088     raise if $bundle
1089
1090     if $doNotMessWithVMPath
1091         if !$remote and !$tarball
1092             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1093             $jscPath = Pathname.new($jscPath).realpath
1094         else
1095             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1096         end
1097     else
1098         originalJSCPath = $jscPath
1099         vmDir = $outputDir + ".vm"
1100         FileUtils.mkdir_p vmDir
1101         
1102         frameworkPath = frameworkFromJSCPath($jscPath)
1103         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1104         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1105         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1106
1107         if frameworkPath
1108             source = frameworkPath
1109             destination = Pathname.new(".vm")
1110         else
1111             source = originalJSCPath
1112             destination = $jscPath
1113
1114             Dir.chdir($outputDir) {
1115                 FileUtils.mkdir_p $jscPath.dirname
1116             }
1117         end
1118
1119         Dir.chdir($outputDir) {
1120             if $copyVM
1121                 FileUtils.cp_r source, destination
1122             else
1123                 begin 
1124                     FileUtils.ln_s source, destination
1125                 rescue
1126                     FileUtils.cp_r source, destination
1127                 end
1128             end
1129
1130             if $remote and $hostOS == "linux"
1131                 begin
1132                     dependencies = `ldd #{source}`
1133                     dependencies.split(/\n/).each {
1134                         | dependency |
1135                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1136                     }
1137                 rescue
1138                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1139                 end
1140             end
1141         }
1142     end
1143     
1144     Dir.chdir($outputDir) {
1145         FileUtils.cp_r HELPERS_PATH, ".helpers"
1146     }
1147
1148     ARGV.each {
1149         | collection |
1150         handleCollection(collection)
1151     }
1152
1153     puts
1154 end
1155
1156 def cleanOldResults
1157     raise unless $bundle
1158
1159     eachResultFile($outputDir) {
1160         | path |
1161         FileUtils.rm_f path
1162     }
1163 end
1164
1165 def cleanEmptyResultFiles
1166     eachResultFile($outputDir) {
1167         | path |
1168         next unless path.basename.to_s =~ /\.out$/
1169         next unless FileTest.size(path) == 0
1170         FileUtils.rm_f path
1171     }
1172 end
1173
1174 def eachResultFile(startingDir, &block)
1175     dirsToClean = [startingDir]
1176     until dirsToClean.empty?
1177         nextDir = dirsToClean.pop
1178         Dir.foreach(nextDir) {
1179             | entry |
1180             next if entry =~ /^\./
1181             path = nextDir + entry
1182             if path.directory?
1183                 dirsToClean.push(path)
1184             else
1185                 block.call(path)
1186             end
1187         }
1188     end
1189 end
1190
1191 def prepareTestRunner
1192     raise if $bundle
1193
1194     $runlist.each_with_index {
1195         | plan, index |
1196         plan.index = index
1197     }
1198
1199     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1200     toDelete = []
1201     Dir.foreach($runnerDir) {
1202         | filename |
1203         if filename =~ /^test_/
1204             toDelete << filename
1205         end
1206     }
1207     
1208     toDelete.each {
1209         | filename |
1210         File.unlink($runnerDir + filename)
1211     }
1212
1213     $runlist.each {
1214         | plan |
1215         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1216     }
1217
1218     case $testRunnerType
1219     when :make
1220         prepareMakeTestRunner
1221     when :shell
1222         prepareShellTestRunner
1223     else
1224         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1225     end
1226 end
1227
1228 def prepareShellTestRunner
1229     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
1230 end
1231
1232 def prepareMakeTestRunner
1233     # The goals of our parallel test runner are scalability and simplicity. The
1234     # simplicity part is particularly important. We don't want to have to have
1235     # a full-time contributor just philosophising about parallel testing.
1236     #
1237     # As such, we just pass off all of the hard work to 'make'. This creates a
1238     # dummy directory ("$outputDir/.runner") in which we create a dummy
1239     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
1240     # That is, for each test we know we will run, there is a rule in the
1241     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
1242     # Makefile results in 'make' doing all of the hard work:
1243     #
1244     # - Load balancing just works. Most systems have a great load balancer in
1245     #   'make'. If your system doesn't then just install a real 'make'.
1246     #
1247     # - Interruptions just work. For example Ctrl-C handling in 'make' is
1248     #   exactly right. You don't have to worry about zombie processes.
1249     #
1250     # We then do some tricks to make failure detection work and to make this
1251     # totally sound. If a test fails, we don't want the whole 'make' job to
1252     # stop. We also don't have any facility for makefile-escaping of path names.
1253     # We do have such a thing for shell-escaping, though. We fix both problems
1254     # by having the actual work for each of the test rules be done in a shell
1255     # script on the side. There is one such script per test. The script responds
1256     # to failure by printing something on the console and then touching a
1257     # failure file for that test, but then still returns 0. This makes 'make'
1258     # continue past that failure and complete all the tests anyway.
1259     #
1260     # In the end, this script collects all of the failures by searching for
1261     # files in the .runner directory whose name matches /^test_fail_/, where
1262     # the thing after the 'fail_' is the test index. Those are the files that
1263     # would be created by the test scripts if they detect failure. We're
1264     # basically using the filesystem as a concurrent database of test failures.
1265     # Even if two tests fail at the same time, since they're touching different
1266     # files we won't miss any failures.
1267     runIndices = []
1268     $runlist.each {
1269         | plan |
1270         runIndices << plan.index
1271     }
1272     
1273     File.open($runnerDir + "Makefile", "w") {
1274         | outp |
1275         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1276         runIndices.each {
1277             | index |
1278             plan = $runlist[index]
1279             outp.puts "test_done_#{index}:"
1280             outp.puts "\tsh test_script_#{plan.index}"
1281         }
1282     }
1283 end
1284
1285     
1286 puts
1287
1288 def cleanRunnerDirectory
1289     raise unless $bundle
1290     Dir.foreach($runnerDir) {
1291         | filename |
1292         next unless filename =~ /^test_fail/
1293         FileUtils.rm_f $runnerDir + filename
1294     }
1295 end
1296
1297 def sshRead(cmd)
1298     raise unless $remote
1299
1300     result = ""
1301     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1302       | inp |
1303       inp.each_line {
1304         | line |
1305         result += line
1306       }
1307     }
1308     raise "#{$?}" unless $?.success?
1309     result
1310 end
1311
1312 def runCommandOnTester(cmd)
1313     if $remote
1314         result = sshRead(cmd)
1315     else
1316         result = `#{cmd}`
1317     end
1318 end
1319
1320 def numberOfProcessors
1321     begin
1322         numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1323     rescue
1324         numProcessors = 0
1325     end
1326
1327     if numProcessors == 0
1328         begin
1329             numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1330         rescue
1331             numProcessors == 0
1332         end
1333     end
1334
1335     if numProcessors == 0
1336         numProcessors = 1
1337     end
1338     return numProcessors
1339 end
1340
1341 def runAndMonitorTestRunnerCommand(*cmd)
1342     numberOfTests = 0
1343     Dir.chdir($runnerDir) {
1344         # -1 for the runscript, and -2 for '..' and '.'
1345         numberOfTests = Dir.entries(".").count - 3
1346     }
1347     unless $progressMeter
1348         mysys(cmd.join(' '))
1349     else
1350        running = {}
1351        didRun = {}
1352        didFail = {}
1353        blankLine = true
1354        prevStringLength = 0
1355        IO.popen(cmd.join(' '), mode="r") {
1356            | inp |
1357            inp.each_line {
1358                | line |
1359                line.chomp!
1360                if line =~ /^Running /
1361                    running[$~.post_match] = true
1362                elsif line =~ /^PASS: /
1363                    didRun[$~.post_match] = true
1364                elsif line =~ /^FAIL: /
1365                    didRun[$~.post_match] = true
1366                    didFail[$~.post_match] = true
1367                else
1368                    unless blankLine
1369                        print("\r" + " " * prevStringLength + "\r")
1370                    end
1371                    puts line
1372                    blankLine = true
1373                end
1374
1375                def lpad(str, chars)
1376                    str = str.to_s
1377                    if str.length > chars
1378                        str
1379                    else
1380                       "%#{chars}s"%(str)
1381                    end
1382                end
1383
1384                string  = ""
1385                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1386                unless didFail.empty?
1387                    string += " (failed #{didFail.size})"
1388                end
1389                string += " "
1390                (running.size - didRun.size).times {
1391                    string += "."
1392                }
1393                if string.length < prevStringLength
1394                    print string
1395                    print(" " * (prevStringLength - string.length))
1396                end
1397                print string
1398                prevStringLength = string.length
1399                blankLine = false
1400                $stdout.flush
1401            }
1402        }
1403        puts
1404        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1405     end
1406 end
1407
1408 def runTestRunner
1409     case $testRunnerType
1410     when :shell
1411         testRunnerCommand = "sh runscript"
1412     when :make
1413         testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
1414     else
1415         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1416     end
1417
1418     if $remote
1419         if !$remoteDirectory
1420             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1421         end
1422         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1423         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1424         remoteScript = "\""
1425         remoteScript += "cd #{$remoteDirectory} && "
1426         remoteScript += "rm -rf #{$outputDir.basename} && "
1427         remoteScript += "tar xzf #{$tarFileName} && "
1428         remoteScript += "cd #{$outputDir.basename}/.runner && "
1429         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1430         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1431         remoteScript += "export JSC_timeout=#{Shellwords.shellescape(ENV['JSC_timeout'])} && "
1432         remoteScript += "#{testRunnerCommand}\""
1433         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1434     else
1435         Dir.chdir($runnerDir) {
1436             runAndMonitorTestRunnerCommand(testRunnerCommand)
1437         }
1438     end
1439 end
1440
1441 def detectFailures
1442     raise if $bundle
1443
1444     if $remote
1445         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1446         output.split(/\n/).each {
1447             | line |
1448             next unless line =~ /test_fail_/
1449             appendFailure($runlist[$~.post_match.to_i])
1450         }
1451     else
1452         Dir.foreach($runnerDir) {
1453             | filename |
1454             next unless filename =~ /test_fail_/
1455             appendFailure($runlist[$~.post_match.to_i])
1456         }
1457     end
1458 end
1459
1460 def compressBundle
1461     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
1462     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1463     raise unless system(cmd)
1464 end
1465
1466 def clean(file)
1467     FileUtils.rm_rf file unless $bundle
1468 end
1469
1470 clean($outputDir + "failed")
1471 clean($outputDir + ".vm")
1472 clean($outputDir + ".helpers")
1473 clean($outputDir + ".runner")
1474 clean($outputDir + ".tests")
1475 clean($outputDir + "_payload")
1476
1477 Dir.mkdir($outputDir) unless $outputDir.directory?
1478
1479 $outputDir = $outputDir.realpath
1480 $runnerDir = $outputDir + ".runner"
1481
1482 if !$numChildProcesses
1483     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
1484         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
1485     else
1486         $numChildProcesses = numberOfProcessors
1487     end
1488 end
1489
1490 if $enableFTL and ENV["JSC_timeout"]
1491     # Currently, using the FTL is a performance regression particularly in real
1492     # (i.e. non-loopy) benchmarks. Account for this in the timeout.
1493     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i * 2).to_s
1494 end
1495
1496 if ENV["JSC_timeout"]
1497     # In the worst case, the processors just interfere with each other.
1498     # Increase the timeout proportionally to the number of processors.
1499     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
1500 end
1501
1502 def runBundle
1503     raise unless $bundle
1504
1505     cleanRunnerDirectory
1506     cleanOldResults
1507     runTestRunner
1508     cleanEmptyResultFiles
1509 end
1510
1511 def runNormal
1512     raise if $bundle or $tarball
1513
1514     prepareBundle
1515     prepareTestRunner
1516     runTestRunner
1517     cleanEmptyResultFiles
1518     detectFailures
1519 end
1520
1521 def runTarball
1522     raise unless $tarball
1523
1524     prepareBundle 
1525     prepareTestRunner
1526     compressBundle
1527 end
1528
1529 def runRemote
1530     raise unless $remote
1531
1532     prepareBundle
1533     prepareTestRunner
1534     compressBundle
1535     runTestRunner
1536     detectFailures
1537 end
1538
1539 if $bundle
1540     runBundle
1541 elsif $remote
1542     runRemote
1543 elsif $tarball
1544     runTarball
1545 else
1546     runNormal
1547 end