JavaScriptCore should have some ES6 conformance tests
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2015 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
41 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
42 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
43 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
44 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
45 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
46
47 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
48
49 begin
50     require 'shellwords'
51 rescue Exception => e
52     $stderr.puts "Warning: did not find shellwords, not running any tests."
53     exit 0
54 end
55
56 $canRunDisplayProfilerOutput = false
57
58 begin
59     require 'rubygems'
60     require 'json'
61     require 'highline'
62     $canRunDisplayProfilerOutput = true
63 rescue Exception => e
64     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
65     $stderr.puts "Error: #{e.inspect}"
66 end
67
68 def printCommandArray(*cmd)
69     begin
70         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
71     rescue
72         commandArray = cmd.join(' ')
73     end
74     $stderr.puts ">> #{commandArray}"
75 end
76
77 def mysys(*cmd)
78     printCommandArray(*cmd) if $verbosity >= 1
79     raise "Command failed: #{$?.inspect}" unless system(*cmd)
80 end
81
82 def escapeAll(array)
83     array.map {
84         | v |
85         raise "Detected a non-string in #{inspect}" unless v.is_a? String
86         Shellwords.shellescape(v)
87     }.join(' ')
88 end
89
90
91 $jscPath = nil
92 $doNotMessWithVMPath = false
93 $enableFTL = false
94 $jitTests = true
95 $memoryLimited = false
96 $outputDir = Pathname.new("results")
97 $verbosity = 0
98 $bundle = nil
99 $tarball = false
100 $tarFileName = "payload.tar.gz"
101 $copyVM = false
102 $testRunnerType = nil
103 $remoteUser = nil
104 $remoteHost = nil
105 $remotePort = nil
106 $remoteDirectory = nil
107 $architecture = nil
108 $hostOS = nil
109 $filter = nil
110
111
112 def usage
113     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
114     puts
115     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
116     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
117     puts "                            --jsc specifies an already present JavaScriptCore to test."
118     puts "--ftl-jit                   Indicate that we have the FTL JIT."
119     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
120     puts "                            Skip tests tagged with //@large-heap"
121     puts "--no-jit                    Do not run JIT specific tests."
122     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
123     puts "--verbose            (-v)   Print more things while running."
124     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
125     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
126     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
127     puts "                            e.g. x86, x86_64, arm."
128     puts "--os                        Specify os instead of determining from JavaScriptCore build."
129     puts "                            e.g. darwin, linux & windows."
130     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
131     puts "                            In general the shell runner is slower than the make runner."
132     puts "--make-runner               Uses the faster make-based runner."
133     puts "--remote                    Specify a remote host on which to run tests from command line argument."
134     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
135     puts "--child-processes    (-c)   Specify the number of child processes."
136     puts "--filter                    Only run tests whose name matches the given regular expression."
137     puts "--help               (-h)   Print this message."
138     exit 1
139 end
140
141 jscArg = nil
142
143 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
144                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
145                ['--no-copy', GetoptLong::NO_ARGUMENT],
146                ['--ftl-jit', GetoptLong::NO_ARGUMENT],
147                ['--memory-limited', GetoptLong::NO_ARGUMENT],
148                ['--no-jit', GetoptLong::NO_ARGUMENT],
149                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
150                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
151                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
152                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
153                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
154                ['--os', GetoptLong::REQUIRED_ARGUMENT],
155                ['--shell-runner', GetoptLong::NO_ARGUMENT],
156                ['--make-runner', GetoptLong::NO_ARGUMENT],
157                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
158                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
159                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
160                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
161                ['--verbose', '-v', GetoptLong::NO_ARGUMENT]).each {
162     | opt, arg |
163     case opt
164     when '--help'
165         usage
166     when '--jsc'
167         jscArg = arg
168     when '--no-copy'
169         $doNotMessWithVMPath = true
170     when '--output-dir'
171         $outputDir = Pathname.new(arg)
172     when '--ftl-jit'
173         $enableFTL = true
174     when '--memory-limited'
175         $memoryLimited = true
176     when '--no-jit'
177         $jitTests = false
178     when '--verbose'
179         $verbosity += 1
180     when '--run-bundle'
181         $bundle = Pathname.new(arg)
182     when '--tarball'
183         $tarball = true
184         $copyVM = true
185         $tarFileName = arg unless arg == ''
186     when '--force-vm-copy'
187         $copyVM = true
188     when '--shell-runner'
189         $testRunnerType = :shell
190     when '--make-runner'
191         $testRunnerType = :make
192     when '--remote'
193         $copyVM = true
194         $tarball = true
195         $remote = true
196         uri = URI("ssh://" + arg)
197         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
198     when '--remote-config-file'
199         $remoteConfigFile = arg
200     when '--child-processes'
201         $numChildProcesses = arg.to_i
202     when '--filter'
203         $filter = Regexp.new(arg)
204     when '--arch'
205         $architecture = arg
206     when '--os'
207         $hostOS = arg
208     end
209 }
210
211 if $remoteConfigFile
212     file = File.read($remoteConfigFile)
213     config = JSON.parse(file)
214
215     if !$remote and config['remote']
216         $copyVM = true
217         $tarball = true
218         $remote = true
219         uri = URI("ssh://" + config['remote'])
220         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
221     end
222
223     if config['remoteDirectory']
224         $remoteDirectory = config['remoteDirectory']
225     end
226 end
227
228 unless jscArg
229     $stderr.puts "Error: must specify -jsc <path>"
230     exit 1
231 end
232
233 if $enableFTL and !$jitTests
234     $stderr.puts "Error: can only specify one of --no-jit and --ftl-jit"
235     exit 1
236 end
237
238 if $doNotMessWithVMPath
239     $jscPath = Pathname.new(jscArg)
240 else
241     $jscPath = Pathname.new(jscArg).realpath
242 end
243
244 $progressMeter = ($verbosity == 0 and $stdout.tty?)
245
246 if $bundle
247     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
248     $outputDir = $bundle
249 end
250
251 # Try to determine architecture. Return nil on failure.
252 def machOArchitectureCode
253     begin 
254         otoolLines = `otool -aSfh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
255         otoolLines.each_with_index {
256             | value, index |
257             if value =~ /magic/ and value =~ /cputype/
258                 return otoolLines[index + 1].split[1].to_i
259             end
260         }
261     rescue
262         $stderr.puts "Warning: unable to execute otool."
263     end
264     $stderr.puts "Warning: unable to determine architecture."
265     nil
266 end
267
268 def determineArchitectureFromMachOBinary
269     code = machOArchitectureCode
270     return nil unless code
271     is64BitFlag = 0x01000000
272     case code
273     when 7
274         "x86"
275     when 7 | is64BitFlag
276         "x86-64"
277     when 12
278         "arm"
279     when 12 | is64BitFlag
280         "arm64"
281     else
282         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
283         nil
284     end
285 end
286
287 def determineArchitectureFromELFBinary
288     f = File.open($jscPath.to_s)
289     data = f.read(19)
290
291     if !(data[0,4] == "\x7F\x45\x4C\x46")
292         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
293         return nil
294     end
295
296     code = data[18].ord
297     case code
298     when 3
299         "x86"
300     when 62
301         "x86-64"
302     when 40
303         "arm"
304     when 183
305         "arm64"
306     else
307         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
308         nil
309     end
310 end
311
312 def determineArchitecture
313     case $hostOS
314     when "darwin"
315         determineArchitectureFromMachOBinary
316     when "linux"
317         determineArchitectureFromELFBinary
318     else
319         $stderr.puts "Warning: unable to determine architecture on this platform."
320         nil
321     end
322 end
323
324 def determineOS
325     case RbConfig::CONFIG["host_os"]
326     when /darwin/i
327         "darwin"
328     when /linux/i
329         "linux"
330     when /mswin|mingw|cygwin/
331         "windows"
332     else
333         $stderr.puts "Warning: unable to determine host operating system"
334         nil
335     end
336 end
337
338 $hostOS = determineOS unless $hostOS
339 $architecture = determineArchitecture unless $architecture
340
341 if !$testRunnerType
342     if $remote and $hostOS == "darwin"
343         $testRunnerType = :shell
344     else
345         $testRunnerType = :make
346     end
347 end
348
349 $numFailures = 0
350
351 BASE_OPTIONS = ["--useFTLJIT=false", "--enableFunctionDotArguments=true"]
352 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000"]
353 NO_CJIT_OPTIONS = ["--enableConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100"]
354 FTL_OPTIONS = ["--useFTLJIT=true", "--ftlCrashesIfCantInitializeLLVM=true"]
355
356 $runlist = []
357
358 def frameworkFromJSCPath(jscPath)
359     parentDirectory = jscPath.dirname
360     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
361         parentDirectory.dirname
362     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
363         jscPath.dirname + "JavaScriptCore.framework"
364     else
365         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
366         nil
367     end
368 end
369
370 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
371     dir = Pathname.new(".")
372     $benchmarkDirectory.each_filename {
373         | pathComponent |
374         dir += ".."
375     }
376     dir + resourcePath
377 end
378
379 def pathToVM
380     pathToBundleResourceFromBenchmarkDirectory($jscPath)
381 end
382
383 def pathToHelpers
384     pathToBundleResourceFromBenchmarkDirectory(".helpers")
385 end
386
387 def prefixCommand(prefix)
388     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
389 end
390
391 def redirectAndPrefixCommand(prefix)
392     prefixCommand(prefix) + " 2>&1"
393 end
394
395 def pipeAndPrefixCommand(outputFilename, prefix)
396     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
397 end
398
399 # Output handler for tests that are expected to be silent.
400 def silentOutputHandler
401     Proc.new {
402         | name |
403         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
404     }
405 end
406
407 # Output handler for tests that are expected to produce meaningful output.
408 def noisyOutputHandler
409     Proc.new {
410         | name |
411         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
412     }
413 end
414
415 # Error handler for tests that fail exactly when they return non-zero exit status.
416 # This is useful when a test is expected to fail.
417 def simpleErrorHandler
418     Proc.new {
419         | outp, plan |
420         outp.puts "if test -e #{plan.failFile}"
421         outp.puts "then"
422         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
423         outp.puts "    " + plan.failCommand
424         outp.puts "else"
425         outp.puts "    " + plan.successCommand
426         outp.puts "fi"
427     }
428 end
429
430 # Error handler for tests that fail exactly when they return zero exit status.
431 def expectedFailErrorHandler
432     Proc.new {
433         | outp, plan |
434         outp.puts "if test -e #{plan.failFile}"
435         outp.puts "then"
436         outp.puts "    " + plan.successCommand
437         outp.puts "else"
438         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
439         outp.puts "    " + plan.failCommand
440         outp.puts "fi"
441     }
442 end
443
444 # Error handler for tests that fail exactly when they return non-zero exit status and produce
445 # lots of spew. This will echo that spew when the test fails.
446 def noisyErrorHandler
447     Proc.new {
448         | outp, plan |
449         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
450     
451         outp.puts "if test -e #{plan.failFile}"
452         outp.puts "then"
453         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
454         outp.puts "    " + plan.failCommand
455         outp.puts "else"
456         outp.puts "    " + plan.successCommand
457         outp.puts "fi"
458     }
459 end
460
461 # Error handler for tests that diff their output with some expectation.
462 def diffErrorHandler(expectedFilename)
463     Proc.new {
464         | outp, plan |
465         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
466         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
467         
468         outp.puts "if test -e #{plan.failFile}"
469         outp.puts "then"
470         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
471         outp.puts "    " + plan.failCommand
472         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
473         outp.puts "then"
474         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
475         outp.puts "    if [ $? -eq 0 ]"
476         outp.puts "    then"
477         outp.puts "    " + plan.successCommand
478         outp.puts "    else"
479         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
480         outp.puts "        " + plan.failCommand
481         outp.puts "    fi"
482         outp.puts "else"
483         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
484         outp.puts "    " + plan.failCommand
485         outp.puts "fi"
486     }
487 end
488
489 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
490 # tests.
491 def mozillaErrorHandler
492     Proc.new {
493         | outp, plan |
494         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
495
496         outp.puts "if test -e #{plan.failFile}"
497         outp.puts "then"
498         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
499         outp.puts "    " + plan.failCommand
500         outp.puts "elif grep -i -q failed! #{outputFilename}"
501         outp.puts "then"
502         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
503         outp.puts "    " + plan.failCommand
504         outp.puts "else"
505         outp.puts "    " + plan.successCommand
506         outp.puts "fi"
507     }
508 end
509
510 # Error handler for tests that report error by saying "failed!", and are expected to
511 # fail. This is used by Mozilla tests.
512 def mozillaFailErrorHandler
513     Proc.new {
514         | outp, plan |
515         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
516
517         outp.puts "if test -e #{plan.failFile}"
518         outp.puts "then"
519         outp.puts "    " + plan.successCommand
520         outp.puts "elif grep -i -q failed! #{outputFilename}"
521         outp.puts "then"
522         outp.puts "    " + plan.successCommand
523         outp.puts "else"
524         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
525         outp.puts "    " + plan.failCommand
526         outp.puts "fi"
527     }
528 end
529
530 # Error handler for tests that report error by saying "failed!", and are expected to have
531 # an exit code of 3.
532 def mozillaExit3ErrorHandler
533     Proc.new {
534         | outp, plan |
535         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
536
537         outp.puts "if test -e #{plan.failFile}"
538         outp.puts "then"
539         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
540         outp.puts "    then"
541         outp.puts "        if grep -i -q failed! #{outputFilename}"
542         outp.puts "        then"
543         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
544         outp.puts "            " + plan.failCommand
545         outp.puts "        else"
546         outp.puts "            " + plan.successCommand
547         outp.puts "        fi"
548         outp.puts "    else"
549         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
550         outp.puts "        " + plan.failCommand
551         outp.puts "    fi"
552         outp.puts "else"
553         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
554         outp.puts "    " + plan.failCommand
555         outp.puts "fi"
556     }
557 end
558
559 $runCommandOptions = {}
560
561 class Plan
562     attr_reader :directory, :arguments, :name, :outputHandler, :errorHandler
563     attr_accessor :index
564     
565     def initialize(directory, arguments, name, outputHandler, errorHandler)
566         @directory = directory
567         @arguments = arguments
568         @name = name
569         @outputHandler = outputHandler
570         @errorHandler = errorHandler
571         @isSlow = !!$runCommandOptions[:isSlow]
572     end
573     
574     def shellCommand
575         # It's important to remember that the test is actually run in a subshell, so if we change directory
576         # in the subshell when we return we will be in our original directory. This is nice because we don't
577         # have to bend over backwards to do things relative to the root.
578         "(cd ../#{Shellwords.shellescape(@directory.to_s)} && \"$@\" " + escapeAll(@arguments) + ")"
579     end
580     
581     def reproScriptCommand
582         # We have to find our way back to the .runner directory since that's where all of the relative
583         # paths assume they start out from.
584         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
585         script += "cd $CURRENT_DIR\n"
586         Pathname.new(@name).dirname.each_filename {
587             | pathComponent |
588             script += "cd ..\n"
589         }
590         script += "cd .runner\n"
591
592         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
593         script += "export JSC_timeout=#{Shellwords.shellescape(ENV['JSC_timeout'])}\n"
594         script += "#{shellCommand} || exit 1"
595         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
596     end
597     
598     def failCommand
599         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
600     end
601     
602     def successCommand
603         if $progressMeter or $verbosity >= 2
604             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
605         else
606             "rm -f #{failFile}"
607         end
608     end
609     
610     def failFile
611         "test_fail_#{@index}"
612     end
613     
614     def writeRunScript(filename)
615         File.open(filename, "w") {
616             | outp |
617             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
618             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
619             cmd += @outputHandler.call(@name)
620             if $verbosity >= 3
621                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
622             end
623             outp.puts cmd
624             @errorHandler.call(outp, self)
625         }
626     end
627 end
628
629 $uniqueFilenameCounter = 0
630 def uniqueFilename(extension)
631     payloadDir = $outputDir + "_payload"
632     Dir.mkdir payloadDir unless payloadDir.directory?
633     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
634     $uniqueFilenameCounter += 1
635     result
636 end
637
638 def baseOutputName(kind)
639     "#{$collectionName}/#{$benchmark}.#{kind}"
640 end
641
642 def addRunCommand(kind, command, outputHandler, errorHandler)
643     $didAddRunCommand = true
644     name = baseOutputName(kind)
645     if $filter and name !~ $filter
646         return
647     end
648     plan = Plan.new($benchmarkDirectory, command, name, outputHandler, errorHandler)
649     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
650         $runlist.unshift plan
651     else
652         $runlist << plan
653     end
654 end
655
656 # Returns true if there were run commands found in the file ($benchmarkDirectory +
657 # $benchmark), in which case those run commands have already been executed. Otherwise
658 # returns false, in which case you're supposed to add your own run commands.
659 def parseRunCommands
660     oldDidAddRunCommand = $didAddRunCommand
661     $didAddRunCommand = false
662
663     Dir.chdir($outputDir) {
664         File.open($benchmarkDirectory + $benchmark) {
665             | inp |
666             inp.each_line {
667                 | line |
668                 begin
669                     doesMatch = line =~ /^\/\/@/
670                 rescue Exception => e
671                     # Apparently this happens in the case of some UTF8 stuff in some files, where
672                     # Ruby tries to be strict and throw exceptions.
673                     next
674                 end
675                 next unless doesMatch
676                 eval $~.post_match
677             }
678         }
679     }
680
681     result = $didAddRunCommand
682     $didAddRunCommand = result or oldDidAddRunCommand
683     result
684 end
685
686 def slow!
687     $runCommandOptions[:isSlow] = true
688 end
689
690 def run(kind, *options)
691     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
692 end
693
694 def runDefault
695     run("default")
696 end
697
698 def runWithRAMSize(size)
699     run("ram-size-#{size}", "--forceRAMSize=#{size}")
700 end
701
702 def runNoLLInt
703     if $jitTests
704         run("no-llint", "--useLLInt=false")
705     end
706 end
707
708 def runNoCJITValidate
709     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *NO_CJIT_OPTIONS)
710 end
711
712 def runNoCJITValidatePhases
713     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
714 end
715
716 def runDefaultFTL
717     run("default-ftl", *FTL_OPTIONS) if $enableFTL
718 end
719
720 def runFTLNoCJIT
721     run("ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
722 end
723
724 def runFTLNoCJITValidate
725     run("ftl-no-cjit-validate", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
726 end
727
728 def runFTLNoCJITNoInlineValidate
729     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
730 end
731
732 def runFTLNoCJITOSRValidation
733     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
734 end
735
736 def runDFGEager
737     run("dfg-eager", *EAGER_OPTIONS)
738 end
739
740 def runDFGEagerNoCJITValidate
741     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
742 end
743
744 def runFTLEager
745     run("ftl-eager", *(FTL_OPTIONS + EAGER_OPTIONS)) if $enableFTL
746 end
747
748 def runFTLEagerNoCJITValidate
749     run("ftl-eager-no-cjit", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
750 end
751
752 def runFTLEagerNoCJITOSRValidation
753     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
754 end
755
756 def runAlwaysTriggerCopyPhase
757     run("always-trigger-copy-phase", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
758 end
759
760 def runNoCJITNoASO
761     run("no-cjit-no-aso", "--enableArchitectureSpecificOptimizations=false", *NO_CJIT_OPTIONS)
762 end
763
764 def runFTLNoCJITNoSimpleOpt
765     run("ftl-no-cjit-no-simple-opt", "--llvmSimpleOpt=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
766 end
767
768 def runNoCJITNoAccessInlining
769     run("no-cjit-no-access-inlining", "--enableAccessInlining=false", *NO_CJIT_OPTIONS)
770 end
771
772 def runFTLNoCJITNoAccessInlining
773     run("ftl-no-cjit-no-access-inlining", "--enableAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
774 end
775
776 def runFTLNoCJITSmallPool
777     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
778 end
779
780 def runMiscNoCJITTest(*options)
781     run("misc-no-cjit", *(NO_CJIT_OPTIONS + options))
782 end
783
784 def runMiscFTLNoCJITTest(*options)
785     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + options))
786 end
787
788 def defaultRun
789     runDefault
790     runAlwaysTriggerCopyPhase
791     if $jitTests
792         runNoLLInt
793         runNoCJITValidatePhases
794         runDFGEager
795         runDFGEagerNoCJITValidate
796         runDefaultFTL
797         runFTLNoCJITValidate
798         runFTLNoCJITNoInlineValidate
799         runFTLEager
800         runFTLEagerNoCJITValidate
801         runFTLNoCJITSmallPool
802     end
803 end
804
805 def defaultQuickRun
806     if $enableFTL
807         runDefaultFTL
808         runFTLNoCJITValidate
809     else
810         runDefault
811         if $jitTests
812             runNoCJITValidate
813         end
814     end
815 end
816
817 def defaultSpotCheck
818     defaultQuickRun
819     runFTLNoCJITNoSimpleOpt
820     runFTLNoCJITOSRValidation
821     runNoCJITNoAccessInlining
822     runFTLNoCJITNoAccessInlining
823 end
824
825 # This is expected to not do eager runs because eager runs can have a lot of recompilations
826 # for reasons that don't arise in the real world. It's used for tests that assert convergence
827 # by counting recompilations.
828 def defaultNoEagerRun
829     runDefault
830     runAlwaysTriggerCopyPhase
831     if $jitTests
832         runNoLLInt
833         runNoCJITValidatePhases
834         runDefaultFTL
835         runFTLNoCJITValidate
836         runFTLNoCJITNoInlineValidate
837     end
838 end
839
840 def runProfiler
841     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
842         skip
843         return
844     end
845
846     profilerOutput = uniqueFilename(".json")
847     if $canRunDisplayProfilerOutput
848         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
849     else
850         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
851         run("profiler-simple", "-p", profilerOutput.to_s)
852     end
853 end
854
855 def runExceptionFuzz
856     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
857     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
858 end
859
860 def runExecutableAllocationFuzz(name, *options)
861     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
862     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
863 end
864
865 def runTypeProfiler
866     if !$jitTests
867         return
868     end
869
870     if $enableFTL
871         run("ftl-no-cjit-type-profiler", "--enableTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
872     else
873         run("no-cjit-type-profiler", "--enableTypeProfiler=true", *NO_CJIT_OPTIONS)
874     end
875 end
876
877 def runControlFlowProfiler
878     if !$jitTests
879         return
880     end
881
882     if $enableFTL
883         run("ftl-no-cjit-type-profiler", "--enableControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
884     else
885         run("no-cjit-type-profiler", "--enableControlFlowProfiler=true", *NO_CJIT_OPTIONS)
886     end
887 end
888
889 def runES6(mode)
890     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
891     case mode
892     when :normal
893         errorHandler = simpleErrorHandler
894     when :fail
895         errorHandler = expectedFailErrorHandler
896     else
897         raise "Invalid mode: #{mode}"
898     end
899     addRunCommand("default", args, noisyOutputHandler, errorHandler)
900 end
901
902 def runLayoutTest(kind, *options)
903     raise unless $benchmark.to_s =~ /\.js$/
904     testName = $~.pre_match
905     if kind
906         kind = "layout-" + kind
907     else
908         kind = "layout"
909     end
910
911     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
912     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
913
914     args = [pathToVM.to_s] + BASE_OPTIONS + options +
915         [(Pathname.new("resources") + "standalone-pre.js").to_s,
916          $benchmark.to_s,
917          (Pathname.new("resources") + "standalone-post.js").to_s]
918     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
919 end
920
921 def runLayoutTestDefault
922     runLayoutTest(nil)
923 end
924
925 def runLayoutTestNoLLInt
926     runLayoutTest("no-llint", "--useLLInt=false")
927 end
928
929 def runLayoutTestNoCJIT
930     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
931 end
932
933 def runLayoutTestDFGEagerNoCJIT
934     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
935 end
936
937 def runLayoutTestDefaultFTL
938     runLayoutTest("ftl", "--testTheFTL=true", *FTL_OPTIONS) if $enableFTL
939 end
940
941 def runLayoutTestFTLNoCJIT
942     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
943 end
944
945 def runLayoutTestFTLEagerNoCJIT
946     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
947 end
948
949 def noFTLRunLayoutTest
950     if !$jitTests
951         return
952     end
953
954     runLayoutTestNoLLInt
955     runLayoutTestNoCJIT
956     runLayoutTestDFGEagerNoCJIT
957 end
958
959 def defaultRunLayoutTest
960     runLayoutTestDefault
961     if $jitTests
962         noFTLRunLayoutTest
963         runLayoutTestDefaultFTL
964         runLayoutTestFTLNoCJIT
965         runLayoutTestFTLEagerNoCJIT
966     end
967 end
968
969 def noNoLLIntRunLayoutTest
970     runLayoutTestDefault
971     if $jitTests
972         runLayoutTestNoCJIT
973         runLayoutTestDFGEagerNoCJIT
974         runLayoutTestDefaultFTL
975         runLayoutTestFTLNoCJIT
976         runLayoutTestFTLEagerNoCJIT
977     end
978 end
979
980 def prepareExtraRelativeFiles(extraFiles, destination)
981     Dir.chdir($outputDir) {
982         extraFiles.each {
983             | file |
984             FileUtils.cp $extraFilesBaseDir + file, destination + file
985         }
986     }
987 end
988
989 def baseDirForCollection(collectionName)
990     Pathname(".tests") + collectionName
991 end
992
993 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
994     raise unless absoluteBase.absolute?
995     Dir.chdir($outputDir) {
996         collectionBaseDir = baseDirForCollection($collectionName)
997         extraFiles.each {
998             | file |
999             destination = collectionBaseDir + file
1000             FileUtils.mkdir_p destination.dirname unless destination.directory?
1001             FileUtils.cp absoluteBase + file, destination
1002         }
1003     }
1004 end
1005
1006 def runMozillaTest(kind, mode, extraFiles, *options)
1007     if kind
1008         kind = "mozilla-" + kind
1009     else
1010         kind = "mozilla"
1011     end
1012     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1013     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1014     case mode
1015     when :normal
1016         errorHandler = mozillaErrorHandler
1017     when :negative
1018         errorHandler = mozillaExit3ErrorHandler
1019     when :fail
1020         errorHandler = mozillaFailErrorHandler
1021     when :skip
1022         return
1023     else
1024         raise "Invalid mode: #{mode}"
1025     end
1026     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1027 end
1028
1029 def runMozillaTestDefault(mode, *extraFiles)
1030     runMozillaTest(nil, mode, extraFiles)
1031 end
1032
1033 def runMozillaTestDefaultFTL(mode, *extraFiles)
1034     runMozillaTest("ftl", mode, extraFiles, *FTL_OPTIONS) if $enableFTL
1035 end
1036
1037 def runMozillaTestLLInt(mode, *extraFiles)
1038     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1039 end
1040
1041 def runMozillaTestBaselineJIT(mode, *extraFiles)
1042     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1043 end
1044
1045 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1046     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1047 end
1048
1049 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1050     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
1051 end
1052
1053 def defaultRunMozillaTest(mode, *extraFiles)
1054     runMozillaTestDefault(mode, *extraFiles)
1055     if $jitTests
1056         runMozillaTestLLInt(mode, *extraFiles)
1057         runMozillaTestBaselineJIT(mode, *extraFiles)
1058         runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1059         runMozillaTestDefaultFTL(mode, *extraFiles)
1060         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1061     end
1062 end
1063
1064 def runNoisyTest(kind, *options)
1065     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
1066 end
1067
1068 def runNoisyTestDefault
1069     runNoisyTest("default")
1070 end
1071
1072 def runNoisyTestDefaultFTL
1073     runNoisyTest("ftl", *FTL_OPTIONS) if $enableFTL
1074 end
1075
1076 def runNoisyTestNoCJIT
1077     runNoisyTest($enableFTL ? "ftl-no-cjit" : "no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(($enableFTL ? FTL_OPTIONS : []) + NO_CJIT_OPTIONS))
1078 end
1079
1080 def runNoisyTestEagerNoCJIT
1081     runNoisyTest($enableFTL ? "ftl-eager-no-cjit" : "eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(($enableFTL ? FTL_OPTIONS : []) + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1082 end
1083
1084 def defaultRunNoisyTest
1085     runNoisyTestDefault
1086     if $jitTests
1087         runNoisyTestDefaultFTL
1088         runNoisyTestNoCJIT
1089         runNoisyTestEagerNoCJIT
1090     end
1091 end
1092
1093 def skip
1094     $didAddRunCommand = true
1095     puts "Skipping #{$collectionName}/#{$benchmark}"
1096 end
1097
1098 def largeHeap
1099     if $memoryLimited
1100         $didAddRunCommand = true
1101         puts "Skipping #{$collectionName}/#{$benchmark}"
1102     end
1103 end
1104
1105 def allJSFiles(path)
1106     if path.file?
1107         [path]
1108     else
1109         result = []
1110         Dir.foreach(path) {
1111             | filename |
1112             next unless filename =~ /\.js$/
1113             next unless (path + filename).file?
1114             result << path + filename
1115         }
1116         result
1117     end
1118 end
1119
1120 def uniqueifyName(names, name)
1121     result = name.to_s
1122     toAdd = 1
1123     while names[result]
1124         result = "#{name}-#{toAdd}"
1125         toAdd += 1
1126     end
1127     names[result] = true
1128     result
1129 end
1130
1131 def simplifyCollectionName(collectionPath)
1132     outerDir = collectionPath.dirname
1133     name = collectionPath.basename
1134     lastName = name
1135     if collectionPath.directory?
1136         while lastName.to_s =~ /test/
1137             lastName = outerDir.basename
1138             name = lastName + name
1139             outerDir = outerDir.dirname
1140         end
1141     end
1142     uniqueifyName($collectionNames, name)
1143 end
1144
1145 def prepareCollection(name)
1146     FileUtils.mkdir_p $outputDir + name
1147
1148     absoluteCollection = $collection.realpath
1149
1150     Dir.chdir($outputDir) {
1151         bundleDir = baseDirForCollection(name)
1152
1153         # Create the proper directory structures.
1154         FileUtils.mkdir_p bundleDir
1155         if bundleDir.basename == $collection.basename
1156             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1157             $collection = bundleDir
1158         else
1159             FileUtils.cp_r absoluteCollection, bundleDir
1160             $collection = bundleDir + $collection.basename
1161         end
1162
1163         $extraFilesBaseDir = absoluteCollection
1164     }
1165 end
1166
1167 $collectionNames = {}
1168
1169 def handleCollectionFile(collection)
1170     collectionName = simplifyCollectionName(collection)
1171    
1172     paths = {}
1173     subCollections = []
1174     YAML::load(IO::read(collection)).each {
1175         | entry |
1176         if entry["collection"]
1177             subCollections << entry["collection"]
1178             next
1179         end
1180         
1181         if Pathname.new(entry["path"]).absolute?
1182             raise "Absolute path: " + entry["path"] + " in #{collection}"
1183         end
1184         
1185         if paths[entry["path"]]
1186             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1187         end
1188         
1189         subCollection = collection.dirname + entry["path"]
1190         
1191         if subCollection.file?
1192             subCollectionName = Pathname.new(entry["path"]).dirname
1193         else
1194             subCollectionName = entry["path"]
1195         end
1196         
1197         $collection = subCollection
1198         $collectionName = Pathname.new(collectionName)
1199         Pathname.new(subCollectionName).each_filename {
1200             | filename |
1201             next if filename =~ /^\./
1202             $collectionName += filename
1203         }
1204         $collectionName = $collectionName.to_s
1205         
1206         prepareCollection($collectionName)
1207       
1208         Dir.chdir($outputDir) {
1209             pathsToSearch = [$collection]
1210             if entry["tests"]
1211                 if entry["tests"].is_a? Array
1212                     pathsToSearch = entry["tests"].map {
1213                         | testName |
1214                         pathsToSearch[0] + testName
1215                     }
1216                 else
1217                     pathsToSearch[0] += entry["tests"]
1218                 end
1219             end
1220             pathsToSearch.each {
1221                 | pathToSearch |
1222                 allJSFiles(pathToSearch).each {
1223                     | path |
1224                     
1225                     $benchmark = path.basename
1226                     $benchmarkDirectory = path.dirname
1227                     
1228                     $runCommandOptions = {}
1229                     eval entry["cmd"]
1230                 }
1231             }
1232         }
1233     }
1234     
1235     subCollections.each {
1236         | subCollection |
1237         handleCollection(collection.dirname + subCollection)
1238     }
1239 end
1240
1241 def handleCollectionDirectory(collection)
1242     collectionName = simplifyCollectionName(collection)
1243     
1244     $collection = collection
1245     $collectionName = collectionName
1246     prepareCollection(collectionName)
1247    
1248     Dir.chdir($outputDir) {
1249         $benchmarkDirectory = $collection
1250         allJSFiles($collection).each {
1251             | path |
1252             
1253             $benchmark = path.basename
1254             
1255             $runCommandOptions = {}
1256             defaultRun unless parseRunCommands
1257         }
1258     }
1259 end
1260
1261 def handleCollection(collection)
1262     collection = Pathname.new(collection)
1263     
1264     if collection.file?
1265         handleCollectionFile(collection)
1266     else
1267         handleCollectionDirectory(collection)
1268     end
1269 end
1270
1271 def appendFailure(plan)
1272     File.open($outputDir + "failed", "a") {
1273         | outp |
1274         outp.puts plan.name
1275     }
1276     $numFailures += 1
1277 end
1278
1279 def prepareBundle
1280     raise if $bundle
1281
1282     if $doNotMessWithVMPath
1283         if !$remote and !$tarball
1284             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1285             $jscPath = Pathname.new($jscPath).realpath
1286         else
1287             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1288         end
1289     else
1290         originalJSCPath = $jscPath
1291         vmDir = $outputDir + ".vm"
1292         FileUtils.mkdir_p vmDir
1293         
1294         frameworkPath = frameworkFromJSCPath($jscPath)
1295         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1296         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1297         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1298
1299         if frameworkPath
1300             source = frameworkPath
1301             destination = Pathname.new(".vm")
1302         else
1303             source = originalJSCPath
1304             destination = $jscPath
1305
1306             Dir.chdir($outputDir) {
1307                 FileUtils.mkdir_p $jscPath.dirname
1308             }
1309         end
1310
1311         Dir.chdir($outputDir) {
1312             if $copyVM
1313                 FileUtils.cp_r source, destination
1314             else
1315                 begin 
1316                     FileUtils.ln_s source, destination
1317                 rescue Exception
1318                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1319                     FileUtils.cp_r source, destination
1320                 end
1321             end
1322
1323             if $remote and $hostOS == "linux"
1324                 if $enableFTL
1325                     begin
1326                         FileUtils.cp_r originalJSCPath + "../../lib/libllvmForJSC.so" , $jscPath.dirname
1327                     rescue
1328                         $stderr.puts "Warning: unable to copy libllvmForJSC.so to the bundle."
1329                     end
1330                 end
1331                 begin
1332                     dependencies = `ldd #{source}`
1333                     dependencies.split(/\n/).each {
1334                         | dependency |
1335                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1336                     }
1337                 rescue
1338                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1339                 end
1340             end
1341         }
1342     end
1343     
1344     Dir.chdir($outputDir) {
1345         FileUtils.cp_r HELPERS_PATH, ".helpers"
1346     }
1347
1348     ARGV.each {
1349         | collection |
1350         handleCollection(collection)
1351     }
1352
1353     puts
1354 end
1355
1356 def cleanOldResults
1357     raise unless $bundle
1358
1359     eachResultFile($outputDir) {
1360         | path |
1361         FileUtils.rm_f path
1362     }
1363 end
1364
1365 def cleanEmptyResultFiles
1366     eachResultFile($outputDir) {
1367         | path |
1368         next unless path.basename.to_s =~ /\.out$/
1369         next unless FileTest.size(path) == 0
1370         FileUtils.rm_f path
1371     }
1372 end
1373
1374 def eachResultFile(startingDir, &block)
1375     dirsToClean = [startingDir]
1376     until dirsToClean.empty?
1377         nextDir = dirsToClean.pop
1378         Dir.foreach(nextDir) {
1379             | entry |
1380             next if entry =~ /^\./
1381             path = nextDir + entry
1382             if path.directory?
1383                 dirsToClean.push(path)
1384             else
1385                 block.call(path)
1386             end
1387         }
1388     end
1389 end
1390
1391 def prepareTestRunner
1392     raise if $bundle
1393
1394     $runlist.each_with_index {
1395         | plan, index |
1396         plan.index = index
1397     }
1398
1399     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1400     toDelete = []
1401     Dir.foreach($runnerDir) {
1402         | filename |
1403         if filename =~ /^test_/
1404             toDelete << filename
1405         end
1406     }
1407     
1408     toDelete.each {
1409         | filename |
1410         File.unlink($runnerDir + filename)
1411     }
1412
1413     $runlist.each {
1414         | plan |
1415         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1416     }
1417
1418     case $testRunnerType
1419     when :make
1420         prepareMakeTestRunner
1421     when :shell
1422         prepareShellTestRunner
1423     else
1424         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1425     end
1426 end
1427
1428 def prepareShellTestRunner
1429     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
1430 end
1431
1432 def prepareMakeTestRunner
1433     # The goals of our parallel test runner are scalability and simplicity. The
1434     # simplicity part is particularly important. We don't want to have to have
1435     # a full-time contributor just philosophising about parallel testing.
1436     #
1437     # As such, we just pass off all of the hard work to 'make'. This creates a
1438     # dummy directory ("$outputDir/.runner") in which we create a dummy
1439     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
1440     # That is, for each test we know we will run, there is a rule in the
1441     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
1442     # Makefile results in 'make' doing all of the hard work:
1443     #
1444     # - Load balancing just works. Most systems have a great load balancer in
1445     #   'make'. If your system doesn't then just install a real 'make'.
1446     #
1447     # - Interruptions just work. For example Ctrl-C handling in 'make' is
1448     #   exactly right. You don't have to worry about zombie processes.
1449     #
1450     # We then do some tricks to make failure detection work and to make this
1451     # totally sound. If a test fails, we don't want the whole 'make' job to
1452     # stop. We also don't have any facility for makefile-escaping of path names.
1453     # We do have such a thing for shell-escaping, though. We fix both problems
1454     # by having the actual work for each of the test rules be done in a shell
1455     # script on the side. There is one such script per test. The script responds
1456     # to failure by printing something on the console and then touching a
1457     # failure file for that test, but then still returns 0. This makes 'make'
1458     # continue past that failure and complete all the tests anyway.
1459     #
1460     # In the end, this script collects all of the failures by searching for
1461     # files in the .runner directory whose name matches /^test_fail_/, where
1462     # the thing after the 'fail_' is the test index. Those are the files that
1463     # would be created by the test scripts if they detect failure. We're
1464     # basically using the filesystem as a concurrent database of test failures.
1465     # Even if two tests fail at the same time, since they're touching different
1466     # files we won't miss any failures.
1467     runIndices = []
1468     $runlist.each {
1469         | plan |
1470         runIndices << plan.index
1471     }
1472     
1473     File.open($runnerDir + "Makefile", "w") {
1474         | outp |
1475         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1476         runIndices.each {
1477             | index |
1478             plan = $runlist[index]
1479             outp.puts "test_done_#{index}:"
1480             outp.puts "\tsh test_script_#{plan.index}"
1481         }
1482     }
1483 end
1484
1485     
1486 puts
1487
1488 def cleanRunnerDirectory
1489     raise unless $bundle
1490     Dir.foreach($runnerDir) {
1491         | filename |
1492         next unless filename =~ /^test_fail/
1493         FileUtils.rm_f $runnerDir + filename
1494     }
1495 end
1496
1497 def sshRead(cmd)
1498     raise unless $remote
1499
1500     result = ""
1501     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1502       | inp |
1503       inp.each_line {
1504         | line |
1505         result += line
1506       }
1507     }
1508     raise "#{$?}" unless $?.success?
1509     result
1510 end
1511
1512 def runCommandOnTester(cmd)
1513     if $remote
1514         result = sshRead(cmd)
1515     else
1516         result = `#{cmd}`
1517     end
1518 end
1519
1520 def numberOfProcessors
1521     begin
1522         numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1523     rescue
1524         numProcessors = 0
1525     end
1526
1527     if numProcessors == 0
1528         begin
1529             numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1530         rescue
1531             numProcessors == 0
1532         end
1533     end
1534
1535     if numProcessors == 0
1536         numProcessors = 1
1537     end
1538     return numProcessors
1539 end
1540
1541 def runAndMonitorTestRunnerCommand(*cmd)
1542     numberOfTests = 0
1543     Dir.chdir($runnerDir) {
1544         # -1 for the runscript, and -2 for '..' and '.'
1545         numberOfTests = Dir.entries(".").count - 3
1546     }
1547     unless $progressMeter
1548         mysys(cmd.join(' '))
1549     else
1550        running = {}
1551        didRun = {}
1552        didFail = {}
1553        blankLine = true
1554        prevStringLength = 0
1555        IO.popen(cmd.join(' '), mode="r") {
1556            | inp |
1557            inp.each_line {
1558                | line |
1559                line.chomp!
1560                if line =~ /^Running /
1561                    running[$~.post_match] = true
1562                elsif line =~ /^PASS: /
1563                    didRun[$~.post_match] = true
1564                elsif line =~ /^FAIL: /
1565                    didRun[$~.post_match] = true
1566                    didFail[$~.post_match] = true
1567                else
1568                    unless blankLine
1569                        print("\r" + " " * prevStringLength + "\r")
1570                    end
1571                    puts line
1572                    blankLine = true
1573                end
1574
1575                def lpad(str, chars)
1576                    str = str.to_s
1577                    if str.length > chars
1578                        str
1579                    else
1580                       "%#{chars}s"%(str)
1581                    end
1582                end
1583
1584                string  = ""
1585                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1586                unless didFail.empty?
1587                    string += " (failed #{didFail.size})"
1588                end
1589                string += " "
1590                (running.size - didRun.size).times {
1591                    string += "."
1592                }
1593                if string.length < prevStringLength
1594                    print string
1595                    print(" " * (prevStringLength - string.length))
1596                end
1597                print string
1598                prevStringLength = string.length
1599                blankLine = false
1600                $stdout.flush
1601            }
1602        }
1603        puts
1604        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1605     end
1606 end
1607
1608 def runTestRunner
1609     case $testRunnerType
1610     when :shell
1611         testRunnerCommand = "sh runscript"
1612     when :make
1613         testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
1614     else
1615         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1616     end
1617
1618     if $remote
1619         if !$remoteDirectory
1620             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1621         end
1622         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1623         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1624         remoteScript = "\""
1625         remoteScript += "cd #{$remoteDirectory} && "
1626         remoteScript += "rm -rf #{$outputDir.basename} && "
1627         remoteScript += "tar xzf #{$tarFileName} && "
1628         remoteScript += "cd #{$outputDir.basename}/.runner && "
1629         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1630         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1631         remoteScript += "export JSC_timeout=#{Shellwords.shellescape(ENV['JSC_timeout'])} && "
1632         remoteScript += "#{testRunnerCommand}\""
1633         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1634     else
1635         Dir.chdir($runnerDir) {
1636             runAndMonitorTestRunnerCommand(testRunnerCommand)
1637         }
1638     end
1639 end
1640
1641 def detectFailures
1642     raise if $bundle
1643
1644     if $remote
1645         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1646         output.split(/\n/).each {
1647             | line |
1648             next unless line =~ /test_fail_/
1649             appendFailure($runlist[$~.post_match.to_i])
1650         }
1651     else
1652         Dir.foreach($runnerDir) {
1653             | filename |
1654             next unless filename =~ /test_fail_/
1655             appendFailure($runlist[$~.post_match.to_i])
1656         }
1657     end
1658 end
1659
1660 def compressBundle
1661     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
1662     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1663     raise unless system(cmd)
1664 end
1665
1666 def clean(file)
1667     FileUtils.rm_rf file unless $bundle
1668 end
1669
1670 clean($outputDir + "failed")
1671 clean($outputDir + ".vm")
1672 clean($outputDir + ".helpers")
1673 clean($outputDir + ".runner")
1674 clean($outputDir + ".tests")
1675 clean($outputDir + "_payload")
1676
1677 Dir.mkdir($outputDir) unless $outputDir.directory?
1678
1679 $outputDir = $outputDir.realpath
1680 $runnerDir = $outputDir + ".runner"
1681
1682 if !$numChildProcesses
1683     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
1684         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
1685     else
1686         $numChildProcesses = numberOfProcessors
1687     end
1688 end
1689
1690 if $enableFTL and ENV["JSC_timeout"]
1691     # Currently, using the FTL is a performance regression particularly in real
1692     # (i.e. non-loopy) benchmarks. Account for this in the timeout.
1693     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i * 2).to_s
1694 end
1695
1696 if ENV["JSC_timeout"]
1697     # In the worst case, the processors just interfere with each other.
1698     # Increase the timeout proportionally to the number of processors.
1699     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
1700 end
1701
1702 def runBundle
1703     raise unless $bundle
1704
1705     cleanRunnerDirectory
1706     cleanOldResults
1707     runTestRunner
1708     cleanEmptyResultFiles
1709 end
1710
1711 def runNormal
1712     raise if $bundle or $tarball
1713
1714     prepareBundle
1715     prepareTestRunner
1716     runTestRunner
1717     cleanEmptyResultFiles
1718     detectFailures
1719 end
1720
1721 def runTarball
1722     raise unless $tarball
1723
1724     prepareBundle 
1725     prepareTestRunner
1726     compressBundle
1727 end
1728
1729 def runRemote
1730     raise unless $remote
1731
1732     prepareBundle
1733     prepareTestRunner
1734     compressBundle
1735     runTestRunner
1736     detectFailures
1737 end
1738
1739 if $bundle
1740     runBundle
1741 elsif $remote
1742     runRemote
1743 elsif $tarball
1744     runTarball
1745 else
1746     runNormal
1747 end