There should be a defaultSpotCheck-like mode that doesn't run the MaximalFlushInserti...
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2015 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
41 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
42 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
43 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
44 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
45 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
46
47 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
48
49 begin
50     require 'shellwords'
51 rescue Exception => e
52     $stderr.puts "Warning: did not find shellwords, not running any tests."
53     exit 0
54 end
55
56 $canRunDisplayProfilerOutput = false
57
58 begin
59     require 'rubygems'
60     require 'json'
61     require 'highline'
62     $canRunDisplayProfilerOutput = true
63 rescue Exception => e
64     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
65     $stderr.puts "Error: #{e.inspect}"
66 end
67
68 def printCommandArray(*cmd)
69     begin
70         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
71     rescue
72         commandArray = cmd.join(' ')
73     end
74     $stderr.puts ">> #{commandArray}"
75 end
76
77 def mysys(*cmd)
78     printCommandArray(*cmd) if $verbosity >= 1
79     raise "Command failed: #{$?.inspect}" unless system(*cmd)
80 end
81
82 def escapeAll(array)
83     array.map {
84         | v |
85         raise "Detected a non-string in #{inspect}" unless v.is_a? String
86         Shellwords.shellescape(v)
87     }.join(' ')
88 end
89
90
91 $jscPath = nil
92 $doNotMessWithVMPath = false
93 $enableFTL = false
94 $jitTests = true
95 $memoryLimited = false
96 $outputDir = Pathname.new("results")
97 $verbosity = 0
98 $bundle = nil
99 $tarball = false
100 $tarFileName = "payload.tar.gz"
101 $copyVM = false
102 $testRunnerType = nil
103 $remoteUser = nil
104 $remoteHost = nil
105 $remotePort = nil
106 $remoteDirectory = nil
107 $architecture = nil
108 $hostOS = nil
109 $filter = nil
110
111
112 def usage
113     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
114     puts
115     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
116     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
117     puts "                            --jsc specifies an already present JavaScriptCore to test."
118     puts "--ftl-jit                   Indicate that we have the FTL JIT."
119     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
120     puts "                            Skip tests tagged with //@large-heap"
121     puts "--no-jit                    Do not run JIT specific tests."
122     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
123     puts "--verbose            (-v)   Print more things while running."
124     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
125     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
126     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
127     puts "                            e.g. x86, x86_64, arm."
128     puts "--os                        Specify os instead of determining from JavaScriptCore build."
129     puts "                            e.g. darwin, linux & windows."
130     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
131     puts "                            In general the shell runner is slower than the make runner."
132     puts "--make-runner               Uses the faster make-based runner."
133     puts "--remote                    Specify a remote host on which to run tests from command line argument."
134     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
135     puts "--child-processes    (-c)   Specify the number of child processes."
136     puts "--filter                    Only run tests whose name matches the given regular expression."
137     puts "--help               (-h)   Print this message."
138     exit 1
139 end
140
141 jscArg = nil
142
143 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
144                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
145                ['--no-copy', GetoptLong::NO_ARGUMENT],
146                ['--ftl-jit', GetoptLong::NO_ARGUMENT],
147                ['--memory-limited', GetoptLong::NO_ARGUMENT],
148                ['--no-jit', GetoptLong::NO_ARGUMENT],
149                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
150                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
151                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
152                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
153                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
154                ['--os', GetoptLong::REQUIRED_ARGUMENT],
155                ['--shell-runner', GetoptLong::NO_ARGUMENT],
156                ['--make-runner', GetoptLong::NO_ARGUMENT],
157                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
158                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
159                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
160                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
161                ['--verbose', '-v', GetoptLong::NO_ARGUMENT]).each {
162     | opt, arg |
163     case opt
164     when '--help'
165         usage
166     when '--jsc'
167         jscArg = arg
168     when '--no-copy'
169         $doNotMessWithVMPath = true
170     when '--output-dir'
171         $outputDir = Pathname.new(arg)
172     when '--ftl-jit'
173         $enableFTL = true
174     when '--memory-limited'
175         $memoryLimited = true
176     when '--no-jit'
177         $jitTests = false
178     when '--verbose'
179         $verbosity += 1
180     when '--run-bundle'
181         $bundle = Pathname.new(arg)
182     when '--tarball'
183         $tarball = true
184         $copyVM = true
185         $tarFileName = arg unless arg == ''
186     when '--force-vm-copy'
187         $copyVM = true
188     when '--shell-runner'
189         $testRunnerType = :shell
190     when '--make-runner'
191         $testRunnerType = :make
192     when '--remote'
193         $copyVM = true
194         $tarball = true
195         $remote = true
196         uri = URI("ssh://" + arg)
197         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
198     when '--remote-config-file'
199         $remoteConfigFile = arg
200     when '--child-processes'
201         $numChildProcesses = arg.to_i
202     when '--filter'
203         $filter = Regexp.new(arg)
204     when '--arch'
205         $architecture = arg
206     when '--os'
207         $hostOS = arg
208     end
209 }
210
211 if $remoteConfigFile
212     file = File.read($remoteConfigFile)
213     config = JSON.parse(file)
214
215     if !$remote and config['remote']
216         $copyVM = true
217         $tarball = true
218         $remote = true
219         uri = URI("ssh://" + config['remote'])
220         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
221     end
222
223     if config['remoteDirectory']
224         $remoteDirectory = config['remoteDirectory']
225     end
226 end
227
228 unless jscArg
229     $stderr.puts "Error: must specify --jsc <path>"
230     exit 1
231 end
232
233 if $enableFTL and !$jitTests
234     $stderr.puts "Error: can only specify one of --no-jit and --ftl-jit"
235     exit 1
236 end
237
238 if $doNotMessWithVMPath
239     $jscPath = Pathname.new(jscArg)
240 else
241     $jscPath = Pathname.new(jscArg).realpath
242 end
243
244 $progressMeter = ($verbosity == 0 and $stdout.tty?)
245
246 if $bundle
247     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
248     $outputDir = $bundle
249 end
250
251 # Try to determine architecture. Return nil on failure.
252 def machOArchitectureCode
253     begin 
254         otoolLines = `otool -aSfh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
255         otoolLines.each_with_index {
256             | value, index |
257             if value =~ /magic/ and value =~ /cputype/
258                 return otoolLines[index + 1].split[1].to_i
259             end
260         }
261     rescue
262         $stderr.puts "Warning: unable to execute otool."
263     end
264     $stderr.puts "Warning: unable to determine architecture."
265     nil
266 end
267
268 def determineArchitectureFromMachOBinary
269     code = machOArchitectureCode
270     return nil unless code
271     is64BitFlag = 0x01000000
272     case code
273     when 7
274         "x86"
275     when 7 | is64BitFlag
276         "x86-64"
277     when 12
278         "arm"
279     when 12 | is64BitFlag
280         "arm64"
281     else
282         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
283         nil
284     end
285 end
286
287 def determineArchitectureFromELFBinary
288     f = File.open($jscPath.to_s)
289     data = f.read(19)
290
291     if !(data[0,4] == "\x7F\x45\x4C\x46")
292         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
293         return nil
294     end
295
296     code = data[18].ord
297     case code
298     when 3
299         "x86"
300     when 62
301         "x86-64"
302     when 40
303         "arm"
304     when 183
305         "arm64"
306     else
307         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
308         nil
309     end
310 end
311
312 def determineArchitecture
313     case $hostOS
314     when "darwin"
315         determineArchitectureFromMachOBinary
316     when "linux"
317         determineArchitectureFromELFBinary
318     else
319         $stderr.puts "Warning: unable to determine architecture on this platform."
320         nil
321     end
322 end
323
324 def determineOS
325     case RbConfig::CONFIG["host_os"]
326     when /darwin/i
327         "darwin"
328     when /linux/i
329         "linux"
330     when /mswin|mingw|cygwin/
331         "windows"
332     else
333         $stderr.puts "Warning: unable to determine host operating system"
334         nil
335     end
336 end
337
338 $hostOS = determineOS unless $hostOS
339 $architecture = determineArchitecture unless $architecture
340
341 if !$testRunnerType
342     if $remote and $hostOS == "darwin"
343         $testRunnerType = :shell
344     else
345         $testRunnerType = :make
346     end
347 end
348
349 $numFailures = 0
350
351 BASE_OPTIONS = ["--useFTLJIT=false", "--enableFunctionDotArguments=true"]
352 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000"]
353 NO_CJIT_OPTIONS = ["--enableConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100"]
354 FTL_OPTIONS = ["--useFTLJIT=true", "--ftlCrashesIfCantInitializeLLVM=true"]
355
356 $runlist = []
357
358 def frameworkFromJSCPath(jscPath)
359     parentDirectory = jscPath.dirname
360     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
361         parentDirectory.dirname
362     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
363         jscPath.dirname + "JavaScriptCore.framework"
364     else
365         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
366         nil
367     end
368 end
369
370 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
371     dir = Pathname.new(".")
372     $benchmarkDirectory.each_filename {
373         | pathComponent |
374         dir += ".."
375     }
376     dir + resourcePath
377 end
378
379 def pathToVM
380     pathToBundleResourceFromBenchmarkDirectory($jscPath)
381 end
382
383 def pathToHelpers
384     pathToBundleResourceFromBenchmarkDirectory(".helpers")
385 end
386
387 def prefixCommand(prefix)
388     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
389 end
390
391 def redirectAndPrefixCommand(prefix)
392     prefixCommand(prefix) + " 2>&1"
393 end
394
395 def pipeAndPrefixCommand(outputFilename, prefix)
396     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
397 end
398
399 # Output handler for tests that are expected to be silent.
400 def silentOutputHandler
401     Proc.new {
402         | name |
403         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
404     }
405 end
406
407 # Output handler for tests that are expected to produce meaningful output.
408 def noisyOutputHandler
409     Proc.new {
410         | name |
411         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
412     }
413 end
414
415 # Error handler for tests that fail exactly when they return non-zero exit status.
416 # This is useful when a test is expected to fail.
417 def simpleErrorHandler
418     Proc.new {
419         | outp, plan |
420         outp.puts "if test -e #{plan.failFile}"
421         outp.puts "then"
422         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
423         outp.puts "    " + plan.failCommand
424         outp.puts "else"
425         outp.puts "    " + plan.successCommand
426         outp.puts "fi"
427     }
428 end
429
430 # Error handler for tests that fail exactly when they return zero exit status.
431 def expectedFailErrorHandler
432     Proc.new {
433         | outp, plan |
434         outp.puts "if test -e #{plan.failFile}"
435         outp.puts "then"
436         outp.puts "    " + plan.successCommand
437         outp.puts "else"
438         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
439         outp.puts "    " + plan.failCommand
440         outp.puts "fi"
441     }
442 end
443
444 # Error handler for tests that fail exactly when they return non-zero exit status and produce
445 # lots of spew. This will echo that spew when the test fails.
446 def noisyErrorHandler
447     Proc.new {
448         | outp, plan |
449         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
450     
451         outp.puts "if test -e #{plan.failFile}"
452         outp.puts "then"
453         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
454         outp.puts "    " + plan.failCommand
455         outp.puts "else"
456         outp.puts "    " + plan.successCommand
457         outp.puts "fi"
458     }
459 end
460
461 # Error handler for tests that diff their output with some expectation.
462 def diffErrorHandler(expectedFilename)
463     Proc.new {
464         | outp, plan |
465         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
466         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
467         
468         outp.puts "if test -e #{plan.failFile}"
469         outp.puts "then"
470         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
471         outp.puts "    " + plan.failCommand
472         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
473         outp.puts "then"
474         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
475         outp.puts "    if [ $? -eq 0 ]"
476         outp.puts "    then"
477         outp.puts "    " + plan.successCommand
478         outp.puts "    else"
479         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
480         outp.puts "        " + plan.failCommand
481         outp.puts "    fi"
482         outp.puts "else"
483         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
484         outp.puts "    " + plan.failCommand
485         outp.puts "fi"
486     }
487 end
488
489 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
490 # tests.
491 def mozillaErrorHandler
492     Proc.new {
493         | outp, plan |
494         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
495
496         outp.puts "if test -e #{plan.failFile}"
497         outp.puts "then"
498         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
499         outp.puts "    " + plan.failCommand
500         outp.puts "elif grep -i -q failed! #{outputFilename}"
501         outp.puts "then"
502         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
503         outp.puts "    " + plan.failCommand
504         outp.puts "else"
505         outp.puts "    " + plan.successCommand
506         outp.puts "fi"
507     }
508 end
509
510 # Error handler for tests that report error by saying "failed!", and are expected to
511 # fail. This is used by Mozilla tests.
512 def mozillaFailErrorHandler
513     Proc.new {
514         | outp, plan |
515         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
516
517         outp.puts "if test -e #{plan.failFile}"
518         outp.puts "then"
519         outp.puts "    " + plan.successCommand
520         outp.puts "elif grep -i -q failed! #{outputFilename}"
521         outp.puts "then"
522         outp.puts "    " + plan.successCommand
523         outp.puts "else"
524         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
525         outp.puts "    " + plan.failCommand
526         outp.puts "fi"
527     }
528 end
529
530 # Error handler for tests that report error by saying "failed!", and are expected to have
531 # an exit code of 3.
532 def mozillaExit3ErrorHandler
533     Proc.new {
534         | outp, plan |
535         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
536
537         outp.puts "if test -e #{plan.failFile}"
538         outp.puts "then"
539         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
540         outp.puts "    then"
541         outp.puts "        if grep -i -q failed! #{outputFilename}"
542         outp.puts "        then"
543         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
544         outp.puts "            " + plan.failCommand
545         outp.puts "        else"
546         outp.puts "            " + plan.successCommand
547         outp.puts "        fi"
548         outp.puts "    else"
549         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
550         outp.puts "        " + plan.failCommand
551         outp.puts "    fi"
552         outp.puts "else"
553         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
554         outp.puts "    " + plan.failCommand
555         outp.puts "fi"
556     }
557 end
558
559 $runCommandOptions = {}
560
561 class Plan
562     attr_reader :directory, :arguments, :name, :outputHandler, :errorHandler
563     attr_accessor :index
564     
565     def initialize(directory, arguments, name, outputHandler, errorHandler)
566         @directory = directory
567         @arguments = arguments
568         @name = name
569         @outputHandler = outputHandler
570         @errorHandler = errorHandler
571         @isSlow = !!$runCommandOptions[:isSlow]
572     end
573     
574     def shellCommand
575         # It's important to remember that the test is actually run in a subshell, so if we change directory
576         # in the subshell when we return we will be in our original directory. This is nice because we don't
577         # have to bend over backwards to do things relative to the root.
578         "(cd ../#{Shellwords.shellescape(@directory.to_s)} && \"$@\" " + escapeAll(@arguments) + ")"
579     end
580     
581     def reproScriptCommand
582         # We have to find our way back to the .runner directory since that's where all of the relative
583         # paths assume they start out from.
584         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
585         script += "cd $CURRENT_DIR\n"
586         Pathname.new(@name).dirname.each_filename {
587             | pathComponent |
588             script += "cd ..\n"
589         }
590         script += "cd .runner\n"
591
592         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
593         script += "export JSC_timeout=#{Shellwords.shellescape(ENV['JSC_timeout'])}\n"
594         script += "#{shellCommand} || exit 1"
595         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
596     end
597     
598     def failCommand
599         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
600     end
601     
602     def successCommand
603         if $progressMeter or $verbosity >= 2
604             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
605         else
606             "rm -f #{failFile}"
607         end
608     end
609     
610     def failFile
611         "test_fail_#{@index}"
612     end
613     
614     def writeRunScript(filename)
615         File.open(filename, "w") {
616             | outp |
617             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
618             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
619             cmd += @outputHandler.call(@name)
620             if $verbosity >= 3
621                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
622             end
623             outp.puts cmd
624             @errorHandler.call(outp, self)
625         }
626     end
627 end
628
629 $uniqueFilenameCounter = 0
630 def uniqueFilename(extension)
631     payloadDir = $outputDir + "_payload"
632     Dir.mkdir payloadDir unless payloadDir.directory?
633     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
634     $uniqueFilenameCounter += 1
635     result
636 end
637
638 def baseOutputName(kind)
639     "#{$collectionName}/#{$benchmark}.#{kind}"
640 end
641
642 def addRunCommand(kind, command, outputHandler, errorHandler)
643     $didAddRunCommand = true
644     name = baseOutputName(kind)
645     if $filter and name !~ $filter
646         return
647     end
648     plan = Plan.new($benchmarkDirectory, command, name, outputHandler, errorHandler)
649     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
650         $runlist.unshift plan
651     else
652         $runlist << plan
653     end
654 end
655
656 # Returns true if there were run commands found in the file ($benchmarkDirectory +
657 # $benchmark), in which case those run commands have already been executed. Otherwise
658 # returns false, in which case you're supposed to add your own run commands.
659 def parseRunCommands
660     oldDidAddRunCommand = $didAddRunCommand
661     $didAddRunCommand = false
662
663     Dir.chdir($outputDir) {
664         File.open($benchmarkDirectory + $benchmark) {
665             | inp |
666             inp.each_line {
667                 | line |
668                 begin
669                     doesMatch = line =~ /^\/\/@/
670                 rescue Exception => e
671                     # Apparently this happens in the case of some UTF8 stuff in some files, where
672                     # Ruby tries to be strict and throw exceptions.
673                     next
674                 end
675                 next unless doesMatch
676                 eval $~.post_match
677             }
678         }
679     }
680
681     result = $didAddRunCommand
682     $didAddRunCommand = result or oldDidAddRunCommand
683     result
684 end
685
686 def slow!
687     $runCommandOptions[:isSlow] = true
688 end
689
690 def run(kind, *options)
691     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
692 end
693
694 def runDefault
695     run("default")
696 end
697
698 def runWithRAMSize(size)
699     run("ram-size-#{size}", "--forceRAMSize=#{size}")
700 end
701
702 def runNoLLInt
703     if $jitTests
704         run("no-llint", "--useLLInt=false")
705     end
706 end
707
708 def runNoCJITValidate
709     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *NO_CJIT_OPTIONS)
710 end
711
712 def runNoCJITValidatePhases
713     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
714 end
715
716 def runDefaultFTL
717     run("default-ftl", *FTL_OPTIONS) if $enableFTL
718 end
719
720 def runFTLNoCJIT
721     run("ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
722 end
723
724 def runFTLNoCJITValidate
725     run("ftl-no-cjit-validate", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
726 end
727
728 def runFTLNoCJITNoInlineValidate
729     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
730 end
731
732 def runFTLNoCJITOSRValidation
733     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
734 end
735
736 def runDFGEager
737     run("dfg-eager", *EAGER_OPTIONS)
738 end
739
740 def runDFGEagerNoCJITValidate
741     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
742 end
743
744 def runFTLEager
745     run("ftl-eager", *(FTL_OPTIONS + EAGER_OPTIONS)) if $enableFTL
746 end
747
748 def runFTLEagerNoCJITValidate
749     run("ftl-eager-no-cjit", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
750 end
751
752 def runFTLEagerNoCJITOSRValidation
753     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
754 end
755
756 def runAlwaysTriggerCopyPhase
757     run("always-trigger-copy-phase", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
758 end
759
760 def runNoCJITNoASO
761     run("no-cjit-no-aso", "--enableArchitectureSpecificOptimizations=false", *NO_CJIT_OPTIONS)
762 end
763
764 def runFTLNoCJITNoSimpleOpt
765     run("ftl-no-cjit-no-simple-opt", "--llvmSimpleOpt=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
766 end
767
768 def runNoCJITNoAccessInlining
769     run("no-cjit-no-access-inlining", "--enableAccessInlining=false", *NO_CJIT_OPTIONS)
770 end
771
772 def runFTLNoCJITNoAccessInlining
773     run("ftl-no-cjit-no-access-inlining", "--enableAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
774 end
775
776 def runFTLNoCJITSmallPool
777     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
778 end
779
780 def runMiscNoCJITTest(*options)
781     run("misc-no-cjit", *(NO_CJIT_OPTIONS + options))
782 end
783
784 def runMiscFTLNoCJITTest(*options)
785     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + options))
786 end
787
788 def runDFGMaximalFlushPhase
789     run("dfg-maximal-flush-validate-no-cjit", "--validateGraph=true", "--enableMaximalFlushInsertionPhase=true", *NO_CJIT_OPTIONS)
790 end
791
792 def defaultRun
793     runDefault
794     runAlwaysTriggerCopyPhase
795     if $jitTests
796         runNoLLInt
797         runNoCJITValidatePhases
798         runDFGEager
799         runDFGEagerNoCJITValidate
800         runDefaultFTL
801         runFTLNoCJITValidate
802         runFTLNoCJITNoInlineValidate
803         runFTLEager
804         runFTLEagerNoCJITValidate
805         runFTLNoCJITSmallPool
806         runDFGMaximalFlushPhase
807     end
808 end
809
810 def defaultQuickRun
811     if $enableFTL
812         runDefaultFTL
813         runFTLNoCJITValidate
814     else
815         runDefault
816         if $jitTests
817             runNoCJITValidate
818         end
819     end
820 end
821
822 def defaultSpotCheckNoMaximalFlush
823     defaultQuickRun
824     runFTLNoCJITNoSimpleOpt
825     runFTLNoCJITOSRValidation
826     runNoCJITNoAccessInlining
827     runFTLNoCJITNoAccessInlining
828 end
829
830 def defaultSpotCheck
831     defaultSpotCheckNoMaximalFlush
832     runDFGMaximalFlushPhase
833 end
834
835 # This is expected to not do eager runs because eager runs can have a lot of recompilations
836 # for reasons that don't arise in the real world. It's used for tests that assert convergence
837 # by counting recompilations.
838 def defaultNoEagerRun
839     runDefault
840     runAlwaysTriggerCopyPhase
841     if $jitTests
842         runNoLLInt
843         runNoCJITValidatePhases
844         runDefaultFTL
845         runFTLNoCJITValidate
846         runFTLNoCJITNoInlineValidate
847     end
848 end
849
850 def runProfiler
851     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
852         skip
853         return
854     end
855
856     profilerOutput = uniqueFilename(".json")
857     if $canRunDisplayProfilerOutput
858         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
859     else
860         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
861         run("profiler-simple", "-p", profilerOutput.to_s)
862     end
863 end
864
865 def runExceptionFuzz
866     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
867     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
868 end
869
870 def runExecutableAllocationFuzz(name, *options)
871     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
872     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
873 end
874
875 def runTypeProfiler
876     if !$jitTests
877         return
878     end
879
880     if $enableFTL
881         run("ftl-no-cjit-type-profiler", "--enableTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
882     else
883         run("no-cjit-type-profiler", "--enableTypeProfiler=true", *NO_CJIT_OPTIONS)
884     end
885 end
886
887 def runControlFlowProfiler
888     if !$jitTests
889         return
890     end
891
892     if $enableFTL
893         run("ftl-no-cjit-type-profiler", "--enableControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
894     else
895         run("no-cjit-type-profiler", "--enableControlFlowProfiler=true", *NO_CJIT_OPTIONS)
896     end
897 end
898
899 def runES6(mode)
900     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
901     case mode
902     when :normal
903         errorHandler = simpleErrorHandler
904     when :fail
905         errorHandler = expectedFailErrorHandler
906     else
907         raise "Invalid mode: #{mode}"
908     end
909     addRunCommand("default", args, noisyOutputHandler, errorHandler)
910 end
911
912 def runModules
913     run("default-modules", "-m")
914     run("always-trigger-copy-phase-modules", "-m", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
915
916     if !$jitTests
917         return
918     end
919
920     run("no-llint-modules", "-m", "--useLLInt=false")
921     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
922     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
923     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
924     if $enableFTL
925         run("default-ftl-modules", "-m", *FTL_OPTIONS)
926         run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
927         run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
928         run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
929         run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
930         run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
931     end
932 end
933
934 def runLayoutTest(kind, *options)
935     raise unless $benchmark.to_s =~ /\.js$/
936     testName = $~.pre_match
937     if kind
938         kind = "layout-" + kind
939     else
940         kind = "layout"
941     end
942
943     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
944     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
945
946     args = [pathToVM.to_s] + BASE_OPTIONS + options +
947         [(Pathname.new("resources") + "standalone-pre.js").to_s,
948          $benchmark.to_s,
949          (Pathname.new("resources") + "standalone-post.js").to_s]
950     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
951 end
952
953 def runLayoutTestDefault
954     runLayoutTest(nil)
955 end
956
957 def runLayoutTestNoLLInt
958     runLayoutTest("no-llint", "--useLLInt=false")
959 end
960
961 def runLayoutTestNoCJIT
962     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
963 end
964
965 def runLayoutTestDFGEagerNoCJIT
966     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
967 end
968
969 def runLayoutTestDefaultFTL
970     runLayoutTest("ftl", "--testTheFTL=true", *FTL_OPTIONS) if $enableFTL
971 end
972
973 def runLayoutTestFTLNoCJIT
974     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
975 end
976
977 def runLayoutTestFTLEagerNoCJIT
978     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
979 end
980
981 def noFTLRunLayoutTest
982     if !$jitTests
983         return
984     end
985
986     runLayoutTestNoLLInt
987     runLayoutTestNoCJIT
988     runLayoutTestDFGEagerNoCJIT
989 end
990
991 def defaultRunLayoutTest
992     runLayoutTestDefault
993     if $jitTests
994         noFTLRunLayoutTest
995         runLayoutTestDefaultFTL
996         runLayoutTestFTLNoCJIT
997         runLayoutTestFTLEagerNoCJIT
998     end
999 end
1000
1001 def noNoLLIntRunLayoutTest
1002     runLayoutTestDefault
1003     if $jitTests
1004         runLayoutTestNoCJIT
1005         runLayoutTestDFGEagerNoCJIT
1006         runLayoutTestDefaultFTL
1007         runLayoutTestFTLNoCJIT
1008         runLayoutTestFTLEagerNoCJIT
1009     end
1010 end
1011
1012 def prepareExtraRelativeFiles(extraFiles, destination)
1013     Dir.chdir($outputDir) {
1014         extraFiles.each {
1015             | file |
1016             FileUtils.cp $extraFilesBaseDir + file, destination + file
1017         }
1018     }
1019 end
1020
1021 def baseDirForCollection(collectionName)
1022     Pathname(".tests") + collectionName
1023 end
1024
1025 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1026     raise unless absoluteBase.absolute?
1027     Dir.chdir($outputDir) {
1028         collectionBaseDir = baseDirForCollection($collectionName)
1029         extraFiles.each {
1030             | file |
1031             destination = collectionBaseDir + file
1032             FileUtils.mkdir_p destination.dirname unless destination.directory?
1033             FileUtils.cp absoluteBase + file, destination
1034         }
1035     }
1036 end
1037
1038 def runMozillaTest(kind, mode, extraFiles, *options)
1039     if kind
1040         kind = "mozilla-" + kind
1041     else
1042         kind = "mozilla"
1043     end
1044     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1045     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1046     case mode
1047     when :normal
1048         errorHandler = mozillaErrorHandler
1049     when :negative
1050         errorHandler = mozillaExit3ErrorHandler
1051     when :fail
1052         errorHandler = mozillaFailErrorHandler
1053     when :skip
1054         return
1055     else
1056         raise "Invalid mode: #{mode}"
1057     end
1058     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1059 end
1060
1061 def runMozillaTestDefault(mode, *extraFiles)
1062     runMozillaTest(nil, mode, extraFiles)
1063 end
1064
1065 def runMozillaTestDefaultFTL(mode, *extraFiles)
1066     runMozillaTest("ftl", mode, extraFiles, *FTL_OPTIONS) if $enableFTL
1067 end
1068
1069 def runMozillaTestLLInt(mode, *extraFiles)
1070     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1071 end
1072
1073 def runMozillaTestBaselineJIT(mode, *extraFiles)
1074     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1075 end
1076
1077 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1078     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1079 end
1080
1081 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1082     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
1083 end
1084
1085 def defaultRunMozillaTest(mode, *extraFiles)
1086     runMozillaTestDefault(mode, *extraFiles)
1087     if $jitTests
1088         runMozillaTestLLInt(mode, *extraFiles)
1089         runMozillaTestBaselineJIT(mode, *extraFiles)
1090         runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1091         runMozillaTestDefaultFTL(mode, *extraFiles)
1092         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1093     end
1094 end
1095
1096 def runNoisyTest(kind, *options)
1097     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
1098 end
1099
1100 def runNoisyTestDefault
1101     runNoisyTest("default")
1102 end
1103
1104 def runNoisyTestDefaultFTL
1105     runNoisyTest("ftl", *FTL_OPTIONS) if $enableFTL
1106 end
1107
1108 def runNoisyTestNoCJIT
1109     runNoisyTest($enableFTL ? "ftl-no-cjit" : "no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(($enableFTL ? FTL_OPTIONS : []) + NO_CJIT_OPTIONS))
1110 end
1111
1112 def runNoisyTestEagerNoCJIT
1113     runNoisyTest($enableFTL ? "ftl-eager-no-cjit" : "eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(($enableFTL ? FTL_OPTIONS : []) + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1114 end
1115
1116 def defaultRunNoisyTest
1117     runNoisyTestDefault
1118     if $jitTests
1119         runNoisyTestDefaultFTL
1120         runNoisyTestNoCJIT
1121         runNoisyTestEagerNoCJIT
1122     end
1123 end
1124
1125 def skip
1126     $didAddRunCommand = true
1127     puts "Skipping #{$collectionName}/#{$benchmark}"
1128 end
1129
1130 def largeHeap
1131     if $memoryLimited
1132         $didAddRunCommand = true
1133         puts "Skipping #{$collectionName}/#{$benchmark}"
1134     end
1135 end
1136
1137 def allJSFiles(path)
1138     if path.file?
1139         [path]
1140     else
1141         result = []
1142         Dir.foreach(path) {
1143             | filename |
1144             next unless filename =~ /\.js$/
1145             next unless (path + filename).file?
1146             result << path + filename
1147         }
1148         result
1149     end
1150 end
1151
1152 def uniqueifyName(names, name)
1153     result = name.to_s
1154     toAdd = 1
1155     while names[result]
1156         result = "#{name}-#{toAdd}"
1157         toAdd += 1
1158     end
1159     names[result] = true
1160     result
1161 end
1162
1163 def simplifyCollectionName(collectionPath)
1164     outerDir = collectionPath.dirname
1165     name = collectionPath.basename
1166     lastName = name
1167     if collectionPath.directory?
1168         while lastName.to_s =~ /test/
1169             lastName = outerDir.basename
1170             name = lastName + name
1171             outerDir = outerDir.dirname
1172         end
1173     end
1174     uniqueifyName($collectionNames, name)
1175 end
1176
1177 def prepareCollection(name)
1178     FileUtils.mkdir_p $outputDir + name
1179
1180     absoluteCollection = $collection.realpath
1181
1182     Dir.chdir($outputDir) {
1183         bundleDir = baseDirForCollection(name)
1184
1185         # Create the proper directory structures.
1186         FileUtils.mkdir_p bundleDir
1187         if bundleDir.basename == $collection.basename
1188             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1189             $collection = bundleDir
1190         else
1191             FileUtils.cp_r absoluteCollection, bundleDir
1192             $collection = bundleDir + $collection.basename
1193         end
1194
1195         $extraFilesBaseDir = absoluteCollection
1196     }
1197 end
1198
1199 $collectionNames = {}
1200
1201 def handleCollectionFile(collection)
1202     collectionName = simplifyCollectionName(collection)
1203    
1204     paths = {}
1205     subCollections = []
1206     YAML::load(IO::read(collection)).each {
1207         | entry |
1208         if entry["collection"]
1209             subCollections << entry["collection"]
1210             next
1211         end
1212         
1213         if Pathname.new(entry["path"]).absolute?
1214             raise "Absolute path: " + entry["path"] + " in #{collection}"
1215         end
1216         
1217         if paths[entry["path"]]
1218             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1219         end
1220         
1221         subCollection = collection.dirname + entry["path"]
1222         
1223         if subCollection.file?
1224             subCollectionName = Pathname.new(entry["path"]).dirname
1225         else
1226             subCollectionName = entry["path"]
1227         end
1228         
1229         $collection = subCollection
1230         $collectionName = Pathname.new(collectionName)
1231         Pathname.new(subCollectionName).each_filename {
1232             | filename |
1233             next if filename =~ /^\./
1234             $collectionName += filename
1235         }
1236         $collectionName = $collectionName.to_s
1237         
1238         prepareCollection($collectionName)
1239       
1240         Dir.chdir($outputDir) {
1241             pathsToSearch = [$collection]
1242             if entry["tests"]
1243                 if entry["tests"].is_a? Array
1244                     pathsToSearch = entry["tests"].map {
1245                         | testName |
1246                         pathsToSearch[0] + testName
1247                     }
1248                 else
1249                     pathsToSearch[0] += entry["tests"]
1250                 end
1251             end
1252             pathsToSearch.each {
1253                 | pathToSearch |
1254                 allJSFiles(pathToSearch).each {
1255                     | path |
1256                     
1257                     $benchmark = path.basename
1258                     $benchmarkDirectory = path.dirname
1259                     
1260                     $runCommandOptions = {}
1261                     eval entry["cmd"]
1262                 }
1263             }
1264         }
1265     }
1266     
1267     subCollections.each {
1268         | subCollection |
1269         handleCollection(collection.dirname + subCollection)
1270     }
1271 end
1272
1273 def handleCollectionDirectory(collection)
1274     collectionName = simplifyCollectionName(collection)
1275     
1276     $collection = collection
1277     $collectionName = collectionName
1278     prepareCollection(collectionName)
1279    
1280     Dir.chdir($outputDir) {
1281         $benchmarkDirectory = $collection
1282         allJSFiles($collection).each {
1283             | path |
1284             
1285             $benchmark = path.basename
1286             
1287             $runCommandOptions = {}
1288             defaultRun unless parseRunCommands
1289         }
1290     }
1291 end
1292
1293 def handleCollection(collection)
1294     collection = Pathname.new(collection)
1295     
1296     if collection.file?
1297         handleCollectionFile(collection)
1298     else
1299         handleCollectionDirectory(collection)
1300     end
1301 end
1302
1303 def appendFailure(plan)
1304     File.open($outputDir + "failed", "a") {
1305         | outp |
1306         outp.puts plan.name
1307     }
1308     $numFailures += 1
1309 end
1310
1311 def prepareBundle
1312     raise if $bundle
1313
1314     if $doNotMessWithVMPath
1315         if !$remote and !$tarball
1316             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1317             $jscPath = Pathname.new($jscPath).realpath
1318         else
1319             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1320         end
1321     else
1322         originalJSCPath = $jscPath
1323         vmDir = $outputDir + ".vm"
1324         FileUtils.mkdir_p vmDir
1325         
1326         frameworkPath = frameworkFromJSCPath($jscPath)
1327         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1328         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1329         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1330
1331         if frameworkPath
1332             source = frameworkPath
1333             destination = Pathname.new(".vm")
1334         else
1335             source = originalJSCPath
1336             destination = $jscPath
1337
1338             Dir.chdir($outputDir) {
1339                 FileUtils.mkdir_p $jscPath.dirname
1340             }
1341         end
1342
1343         Dir.chdir($outputDir) {
1344             if $copyVM
1345                 FileUtils.cp_r source, destination
1346             else
1347                 begin 
1348                     FileUtils.ln_s source, destination
1349                 rescue Exception
1350                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1351                     FileUtils.cp_r source, destination
1352                 end
1353             end
1354
1355             if $remote and $hostOS == "linux"
1356                 if $enableFTL
1357                     begin
1358                         FileUtils.cp_r originalJSCPath + "../../lib/libllvmForJSC.so" , $jscPath.dirname
1359                     rescue
1360                         $stderr.puts "Warning: unable to copy libllvmForJSC.so to the bundle."
1361                     end
1362                 end
1363                 begin
1364                     dependencies = `ldd #{source}`
1365                     dependencies.split(/\n/).each {
1366                         | dependency |
1367                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1368                     }
1369                 rescue
1370                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1371                 end
1372             end
1373         }
1374     end
1375     
1376     Dir.chdir($outputDir) {
1377         FileUtils.cp_r HELPERS_PATH, ".helpers"
1378     }
1379
1380     ARGV.each {
1381         | collection |
1382         handleCollection(collection)
1383     }
1384
1385     puts
1386 end
1387
1388 def cleanOldResults
1389     raise unless $bundle
1390
1391     eachResultFile($outputDir) {
1392         | path |
1393         FileUtils.rm_f path
1394     }
1395 end
1396
1397 def cleanEmptyResultFiles
1398     eachResultFile($outputDir) {
1399         | path |
1400         next unless path.basename.to_s =~ /\.out$/
1401         next unless FileTest.size(path) == 0
1402         FileUtils.rm_f path
1403     }
1404 end
1405
1406 def eachResultFile(startingDir, &block)
1407     dirsToClean = [startingDir]
1408     until dirsToClean.empty?
1409         nextDir = dirsToClean.pop
1410         Dir.foreach(nextDir) {
1411             | entry |
1412             next if entry =~ /^\./
1413             path = nextDir + entry
1414             if path.directory?
1415                 dirsToClean.push(path)
1416             else
1417                 block.call(path)
1418             end
1419         }
1420     end
1421 end
1422
1423 def prepareTestRunner
1424     raise if $bundle
1425
1426     $runlist.each_with_index {
1427         | plan, index |
1428         plan.index = index
1429     }
1430
1431     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1432     toDelete = []
1433     Dir.foreach($runnerDir) {
1434         | filename |
1435         if filename =~ /^test_/
1436             toDelete << filename
1437         end
1438     }
1439     
1440     toDelete.each {
1441         | filename |
1442         File.unlink($runnerDir + filename)
1443     }
1444
1445     $runlist.each {
1446         | plan |
1447         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1448     }
1449
1450     case $testRunnerType
1451     when :make
1452         prepareMakeTestRunner
1453     when :shell
1454         prepareShellTestRunner
1455     else
1456         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1457     end
1458 end
1459
1460 def prepareShellTestRunner
1461     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
1462 end
1463
1464 def prepareMakeTestRunner
1465     # The goals of our parallel test runner are scalability and simplicity. The
1466     # simplicity part is particularly important. We don't want to have to have
1467     # a full-time contributor just philosophising about parallel testing.
1468     #
1469     # As such, we just pass off all of the hard work to 'make'. This creates a
1470     # dummy directory ("$outputDir/.runner") in which we create a dummy
1471     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
1472     # That is, for each test we know we will run, there is a rule in the
1473     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
1474     # Makefile results in 'make' doing all of the hard work:
1475     #
1476     # - Load balancing just works. Most systems have a great load balancer in
1477     #   'make'. If your system doesn't then just install a real 'make'.
1478     #
1479     # - Interruptions just work. For example Ctrl-C handling in 'make' is
1480     #   exactly right. You don't have to worry about zombie processes.
1481     #
1482     # We then do some tricks to make failure detection work and to make this
1483     # totally sound. If a test fails, we don't want the whole 'make' job to
1484     # stop. We also don't have any facility for makefile-escaping of path names.
1485     # We do have such a thing for shell-escaping, though. We fix both problems
1486     # by having the actual work for each of the test rules be done in a shell
1487     # script on the side. There is one such script per test. The script responds
1488     # to failure by printing something on the console and then touching a
1489     # failure file for that test, but then still returns 0. This makes 'make'
1490     # continue past that failure and complete all the tests anyway.
1491     #
1492     # In the end, this script collects all of the failures by searching for
1493     # files in the .runner directory whose name matches /^test_fail_/, where
1494     # the thing after the 'fail_' is the test index. Those are the files that
1495     # would be created by the test scripts if they detect failure. We're
1496     # basically using the filesystem as a concurrent database of test failures.
1497     # Even if two tests fail at the same time, since they're touching different
1498     # files we won't miss any failures.
1499     runIndices = []
1500     $runlist.each {
1501         | plan |
1502         runIndices << plan.index
1503     }
1504     
1505     File.open($runnerDir + "Makefile", "w") {
1506         | outp |
1507         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1508         runIndices.each {
1509             | index |
1510             plan = $runlist[index]
1511             outp.puts "test_done_#{index}:"
1512             outp.puts "\tsh test_script_#{plan.index}"
1513         }
1514     }
1515 end
1516
1517     
1518 puts
1519
1520 def cleanRunnerDirectory
1521     raise unless $bundle
1522     Dir.foreach($runnerDir) {
1523         | filename |
1524         next unless filename =~ /^test_fail/
1525         FileUtils.rm_f $runnerDir + filename
1526     }
1527 end
1528
1529 def sshRead(cmd)
1530     raise unless $remote
1531
1532     result = ""
1533     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1534       | inp |
1535       inp.each_line {
1536         | line |
1537         result += line
1538       }
1539     }
1540     raise "#{$?}" unless $?.success?
1541     result
1542 end
1543
1544 def runCommandOnTester(cmd)
1545     if $remote
1546         result = sshRead(cmd)
1547     else
1548         result = `#{cmd}`
1549     end
1550 end
1551
1552 def numberOfProcessors
1553     begin
1554         numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1555     rescue
1556         numProcessors = 0
1557     end
1558
1559     if numProcessors == 0
1560         begin
1561             numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1562         rescue
1563             numProcessors == 0
1564         end
1565     end
1566
1567     if numProcessors == 0
1568         numProcessors = 1
1569     end
1570     return numProcessors
1571 end
1572
1573 def runAndMonitorTestRunnerCommand(*cmd)
1574     numberOfTests = 0
1575     Dir.chdir($runnerDir) {
1576         # -1 for the runscript, and -2 for '..' and '.'
1577         numberOfTests = Dir.entries(".").count - 3
1578     }
1579     unless $progressMeter
1580         mysys(cmd.join(' '))
1581     else
1582        running = {}
1583        didRun = {}
1584        didFail = {}
1585        blankLine = true
1586        prevStringLength = 0
1587        IO.popen(cmd.join(' '), mode="r") {
1588            | inp |
1589            inp.each_line {
1590                | line |
1591                line.chomp!
1592                if line =~ /^Running /
1593                    running[$~.post_match] = true
1594                elsif line =~ /^PASS: /
1595                    didRun[$~.post_match] = true
1596                elsif line =~ /^FAIL: /
1597                    didRun[$~.post_match] = true
1598                    didFail[$~.post_match] = true
1599                else
1600                    unless blankLine
1601                        print("\r" + " " * prevStringLength + "\r")
1602                    end
1603                    puts line
1604                    blankLine = true
1605                end
1606
1607                def lpad(str, chars)
1608                    str = str.to_s
1609                    if str.length > chars
1610                        str
1611                    else
1612                       "%#{chars}s"%(str)
1613                    end
1614                end
1615
1616                string  = ""
1617                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1618                unless didFail.empty?
1619                    string += " (failed #{didFail.size})"
1620                end
1621                string += " "
1622                (running.size - didRun.size).times {
1623                    string += "."
1624                }
1625                if string.length < prevStringLength
1626                    print string
1627                    print(" " * (prevStringLength - string.length))
1628                end
1629                print string
1630                prevStringLength = string.length
1631                blankLine = false
1632                $stdout.flush
1633            }
1634        }
1635        puts
1636        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1637     end
1638 end
1639
1640 def runTestRunner
1641     case $testRunnerType
1642     when :shell
1643         testRunnerCommand = "sh runscript"
1644     when :make
1645         testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
1646     else
1647         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1648     end
1649
1650     if $remote
1651         if !$remoteDirectory
1652             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1653         end
1654         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1655         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1656         remoteScript = "\""
1657         remoteScript += "cd #{$remoteDirectory} && "
1658         remoteScript += "rm -rf #{$outputDir.basename} && "
1659         remoteScript += "tar xzf #{$tarFileName} && "
1660         remoteScript += "cd #{$outputDir.basename}/.runner && "
1661         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1662         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1663         remoteScript += "export JSC_timeout=#{Shellwords.shellescape(ENV['JSC_timeout'])} && "
1664         remoteScript += "#{testRunnerCommand}\""
1665         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1666     else
1667         Dir.chdir($runnerDir) {
1668             runAndMonitorTestRunnerCommand(testRunnerCommand)
1669         }
1670     end
1671 end
1672
1673 def detectFailures
1674     raise if $bundle
1675
1676     if $remote
1677         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1678         output.split(/\n/).each {
1679             | line |
1680             next unless line =~ /test_fail_/
1681             appendFailure($runlist[$~.post_match.to_i])
1682         }
1683     else
1684         Dir.foreach($runnerDir) {
1685             | filename |
1686             next unless filename =~ /test_fail_/
1687             appendFailure($runlist[$~.post_match.to_i])
1688         }
1689     end
1690 end
1691
1692 def compressBundle
1693     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
1694     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1695     raise unless system(cmd)
1696 end
1697
1698 def clean(file)
1699     FileUtils.rm_rf file unless $bundle
1700 end
1701
1702 clean($outputDir + "failed")
1703 clean($outputDir + ".vm")
1704 clean($outputDir + ".helpers")
1705 clean($outputDir + ".runner")
1706 clean($outputDir + ".tests")
1707 clean($outputDir + "_payload")
1708
1709 Dir.mkdir($outputDir) unless $outputDir.directory?
1710
1711 $outputDir = $outputDir.realpath
1712 $runnerDir = $outputDir + ".runner"
1713
1714 if !$numChildProcesses
1715     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
1716         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
1717     else
1718         $numChildProcesses = numberOfProcessors
1719     end
1720 end
1721
1722 if $enableFTL and ENV["JSC_timeout"]
1723     # Currently, using the FTL is a performance regression particularly in real
1724     # (i.e. non-loopy) benchmarks. Account for this in the timeout.
1725     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i * 2).to_s
1726 end
1727
1728 if ENV["JSC_timeout"]
1729     # In the worst case, the processors just interfere with each other.
1730     # Increase the timeout proportionally to the number of processors.
1731     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
1732 end
1733
1734 def runBundle
1735     raise unless $bundle
1736
1737     cleanRunnerDirectory
1738     cleanOldResults
1739     runTestRunner
1740     cleanEmptyResultFiles
1741 end
1742
1743 def runNormal
1744     raise if $bundle or $tarball
1745
1746     prepareBundle
1747     prepareTestRunner
1748     runTestRunner
1749     cleanEmptyResultFiles
1750     detectFailures
1751 end
1752
1753 def runTarball
1754     raise unless $tarball
1755
1756     prepareBundle 
1757     prepareTestRunner
1758     compressBundle
1759 end
1760
1761 def runRemote
1762     raise unless $remote
1763
1764     prepareBundle
1765     prepareTestRunner
1766     compressBundle
1767     runTestRunner
1768     detectFailures
1769 end
1770
1771 if $bundle
1772     runBundle
1773 elsif $remote
1774     runRemote
1775 elsif $tarball
1776     runTarball
1777 else
1778     runNormal
1779 end