Fix run-javascriptcore-tests/run-jsc-stress-tests to work with rootless, add a -...
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2015 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
41 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
42 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
43 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
44 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
45 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
46
47 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
48
49 begin
50     require 'shellwords'
51 rescue Exception => e
52     $stderr.puts "Warning: did not find shellwords, not running any tests."
53     exit 0
54 end
55
56 $canRunDisplayProfilerOutput = false
57
58 begin
59     require 'rubygems'
60     require 'json'
61     require 'highline'
62     $canRunDisplayProfilerOutput = true
63 rescue Exception => e
64     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
65     $stderr.puts "Error: #{e.inspect}"
66 end
67
68 def printCommandArray(*cmd)
69     begin
70         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
71     rescue
72         commandArray = cmd.join(' ')
73     end
74     $stderr.puts ">> #{commandArray}"
75 end
76
77 def mysys(*cmd)
78     printCommandArray(*cmd) if $verbosity >= 1
79     raise "Command failed: #{$?.inspect}" unless system(*cmd)
80 end
81
82 def escapeAll(array)
83     array.map {
84         | v |
85         raise "Detected a non-string in #{inspect}" unless v.is_a? String
86         Shellwords.shellescape(v)
87     }.join(' ')
88 end
89
90
91 $jscPath = nil
92 $doNotMessWithVMPath = false
93 $enableFTL = false
94 $jitTests = true
95 $memoryLimited = false
96 $outputDir = Pathname.new("results")
97 $verbosity = 0
98 $bundle = nil
99 $tarball = false
100 $tarFileName = "payload.tar.gz"
101 $copyVM = false
102 $testRunnerType = nil
103 $remoteUser = nil
104 $remoteHost = nil
105 $remotePort = nil
106 $remoteDirectory = nil
107 $architecture = nil
108 $hostOS = nil
109 $filter = nil
110 $envVars = []
111 $quickMode = false
112
113
114 def usage
115     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
116     puts
117     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
118     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
119     puts "                            --jsc specifies an already present JavaScriptCore to test."
120     puts "--ftl-jit                   Indicate that we have the FTL JIT."
121     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
122     puts "                            Skip tests tagged with //@large-heap"
123     puts "--no-jit                    Do not run JIT specific tests."
124     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
125     puts "--verbose            (-v)   Print more things while running."
126     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
127     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
128     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
129     puts "                            e.g. x86, x86_64, arm."
130     puts "--os                        Specify os instead of determining from JavaScriptCore build."
131     puts "                            e.g. darwin, linux & windows."
132     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
133     puts "                            In general the shell runner is slower than the make runner."
134     puts "--make-runner               Uses the faster make-based runner."
135     puts "--remote                    Specify a remote host on which to run tests from command line argument."
136     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
137     puts "--child-processes    (-c)   Specify the number of child processes."
138     puts "--filter                    Only run tests whose name matches the given regular expression."
139     puts "--help               (-h)   Print this message."
140     puts "--env-vars                  Add a list of environment variables to set before running jsc."
141     puts "                            Each environment variable should be separated by a space."
142     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
143     puts "                            it will override the default value."
144     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
145     exit 1
146 end
147
148 jscArg = nil
149
150 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
151                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
152                ['--no-copy', GetoptLong::NO_ARGUMENT],
153                ['--ftl-jit', GetoptLong::NO_ARGUMENT],
154                ['--memory-limited', GetoptLong::NO_ARGUMENT],
155                ['--no-jit', GetoptLong::NO_ARGUMENT],
156                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
157                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
158                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
159                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
160                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
161                ['--os', GetoptLong::REQUIRED_ARGUMENT],
162                ['--shell-runner', GetoptLong::NO_ARGUMENT],
163                ['--make-runner', GetoptLong::NO_ARGUMENT],
164                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
165                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
166                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
167                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
168                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
169                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
170                ['--quick', '-q', GetoptLong::NO_ARGUMENT]).each {
171     | opt, arg |
172     case opt
173     when '--help'
174         usage
175     when '--jsc'
176         jscArg = arg
177     when '--no-copy'
178         $doNotMessWithVMPath = true
179     when '--output-dir'
180         $outputDir = Pathname.new(arg)
181     when '--ftl-jit'
182         $enableFTL = true
183     when '--memory-limited'
184         $memoryLimited = true
185     when '--no-jit'
186         $jitTests = false
187     when '--verbose'
188         $verbosity += 1
189     when '--run-bundle'
190         $bundle = Pathname.new(arg)
191     when '--tarball'
192         $tarball = true
193         $copyVM = true
194         $tarFileName = arg unless arg == ''
195     when '--force-vm-copy'
196         $copyVM = true
197     when '--shell-runner'
198         $testRunnerType = :shell
199     when '--make-runner'
200         $testRunnerType = :make
201     when '--remote'
202         $copyVM = true
203         $tarball = true
204         $remote = true
205         uri = URI("ssh://" + arg)
206         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
207     when '--remote-config-file'
208         $remoteConfigFile = arg
209     when '--child-processes'
210         $numChildProcesses = arg.to_i
211     when '--filter'
212         $filter = Regexp.new(arg)
213     when '--arch'
214         $architecture = arg
215     when '--os'
216         $hostOS = arg
217     when '--env-vars'
218         $envVars = arg.gsub(/\s+/, ' ').split(' ')
219     when '--quick'
220         $quickMode = true
221     end
222 }
223
224 if $remoteConfigFile
225     file = File.read($remoteConfigFile)
226     config = JSON.parse(file)
227
228     if !$remote and config['remote']
229         $copyVM = true
230         $tarball = true
231         $remote = true
232         uri = URI("ssh://" + config['remote'])
233         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
234     end
235
236     if config['remoteDirectory']
237         $remoteDirectory = config['remoteDirectory']
238     end
239 end
240
241 unless jscArg
242     $stderr.puts "Error: must specify --jsc <path>"
243     exit 1
244 end
245
246 if $enableFTL and !$jitTests
247     $stderr.puts "Error: can only specify one of --no-jit and --ftl-jit"
248     exit 1
249 end
250
251 if $doNotMessWithVMPath
252     $jscPath = Pathname.new(jscArg)
253 else
254     $jscPath = Pathname.new(jscArg).realpath
255 end
256
257 $progressMeter = ($verbosity == 0 and $stdout.tty?)
258
259 if $bundle
260     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
261     $outputDir = $bundle
262 end
263
264 # Try to determine architecture. Return nil on failure.
265 def machOArchitectureCode
266     begin 
267         otoolLines = `otool -aSfh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
268         otoolLines.each_with_index {
269             | value, index |
270             if value =~ /magic/ and value =~ /cputype/
271                 return otoolLines[index + 1].split[1].to_i
272             end
273         }
274     rescue
275         $stderr.puts "Warning: unable to execute otool."
276     end
277     $stderr.puts "Warning: unable to determine architecture."
278     nil
279 end
280
281 def determineArchitectureFromMachOBinary
282     code = machOArchitectureCode
283     return nil unless code
284     is64BitFlag = 0x01000000
285     case code
286     when 7
287         "x86"
288     when 7 | is64BitFlag
289         "x86-64"
290     when 12
291         "arm"
292     when 12 | is64BitFlag
293         "arm64"
294     else
295         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
296         nil
297     end
298 end
299
300 def determineArchitectureFromELFBinary
301     f = File.open($jscPath.to_s)
302     data = f.read(19)
303
304     if !(data[0,4] == "\x7F\x45\x4C\x46")
305         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
306         return nil
307     end
308
309     code = data[18].ord
310     case code
311     when 3
312         "x86"
313     when 62
314         "x86-64"
315     when 40
316         "arm"
317     when 183
318         "arm64"
319     else
320         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
321         nil
322     end
323 end
324
325 def determineArchitecture
326     case $hostOS
327     when "darwin"
328         determineArchitectureFromMachOBinary
329     when "linux"
330         determineArchitectureFromELFBinary
331     else
332         $stderr.puts "Warning: unable to determine architecture on this platform."
333         nil
334     end
335 end
336
337 def determineOS
338     case RbConfig::CONFIG["host_os"]
339     when /darwin/i
340         "darwin"
341     when /linux/i
342         "linux"
343     when /mswin|mingw|cygwin/
344         "windows"
345     else
346         $stderr.puts "Warning: unable to determine host operating system"
347         nil
348     end
349 end
350
351 $hostOS = determineOS unless $hostOS
352 $architecture = determineArchitecture unless $architecture
353
354 if !$testRunnerType
355     if $remote and $hostOS == "darwin"
356         $testRunnerType = :shell
357     else
358         $testRunnerType = :make
359     end
360 end
361
362 $numFailures = 0
363
364 BASE_OPTIONS = ["--useFTLJIT=false", "--enableFunctionDotArguments=true"]
365 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000"]
366 NO_CJIT_OPTIONS = ["--enableConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100"]
367 FTL_OPTIONS = ["--useFTLJIT=true", "--ftlCrashesIfCantInitializeLLVM=true"]
368
369 $runlist = []
370
371 def frameworkFromJSCPath(jscPath)
372     parentDirectory = jscPath.dirname
373     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
374         parentDirectory.dirname
375     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
376         jscPath.dirname + "JavaScriptCore.framework"
377     else
378         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
379         nil
380     end
381 end
382
383 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
384     dir = Pathname.new(".")
385     $benchmarkDirectory.each_filename {
386         | pathComponent |
387         dir += ".."
388     }
389     dir + resourcePath
390 end
391
392 def pathToVM
393     pathToBundleResourceFromBenchmarkDirectory($jscPath)
394 end
395
396 def pathToHelpers
397     pathToBundleResourceFromBenchmarkDirectory(".helpers")
398 end
399
400 def prefixCommand(prefix)
401     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
402 end
403
404 def redirectAndPrefixCommand(prefix)
405     prefixCommand(prefix) + " 2>&1"
406 end
407
408 def pipeAndPrefixCommand(outputFilename, prefix)
409     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
410 end
411
412 # Output handler for tests that are expected to be silent.
413 def silentOutputHandler
414     Proc.new {
415         | name |
416         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
417     }
418 end
419
420 # Output handler for tests that are expected to produce meaningful output.
421 def noisyOutputHandler
422     Proc.new {
423         | name |
424         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
425     }
426 end
427
428 # Error handler for tests that fail exactly when they return non-zero exit status.
429 # This is useful when a test is expected to fail.
430 def simpleErrorHandler
431     Proc.new {
432         | outp, plan |
433         outp.puts "if test -e #{plan.failFile}"
434         outp.puts "then"
435         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
436         outp.puts "    " + plan.failCommand
437         outp.puts "else"
438         outp.puts "    " + plan.successCommand
439         outp.puts "fi"
440     }
441 end
442
443 # Error handler for tests that fail exactly when they return zero exit status.
444 def expectedFailErrorHandler
445     Proc.new {
446         | outp, plan |
447         outp.puts "if test -e #{plan.failFile}"
448         outp.puts "then"
449         outp.puts "    " + plan.successCommand
450         outp.puts "else"
451         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
452         outp.puts "    " + plan.failCommand
453         outp.puts "fi"
454     }
455 end
456
457 # Error handler for tests that fail exactly when they return non-zero exit status and produce
458 # lots of spew. This will echo that spew when the test fails.
459 def noisyErrorHandler
460     Proc.new {
461         | outp, plan |
462         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
463     
464         outp.puts "if test -e #{plan.failFile}"
465         outp.puts "then"
466         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
467         outp.puts "    " + plan.failCommand
468         outp.puts "else"
469         outp.puts "    " + plan.successCommand
470         outp.puts "fi"
471     }
472 end
473
474 # Error handler for tests that diff their output with some expectation.
475 def diffErrorHandler(expectedFilename)
476     Proc.new {
477         | outp, plan |
478         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
479         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
480         
481         outp.puts "if test -e #{plan.failFile}"
482         outp.puts "then"
483         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
484         outp.puts "    " + plan.failCommand
485         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
486         outp.puts "then"
487         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
488         outp.puts "    if [ $? -eq 0 ]"
489         outp.puts "    then"
490         outp.puts "    " + plan.successCommand
491         outp.puts "    else"
492         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
493         outp.puts "        " + plan.failCommand
494         outp.puts "    fi"
495         outp.puts "else"
496         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
497         outp.puts "    " + plan.failCommand
498         outp.puts "fi"
499     }
500 end
501
502 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
503 # tests.
504 def mozillaErrorHandler
505     Proc.new {
506         | outp, plan |
507         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
508
509         outp.puts "if test -e #{plan.failFile}"
510         outp.puts "then"
511         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
512         outp.puts "    " + plan.failCommand
513         outp.puts "elif grep -i -q failed! #{outputFilename}"
514         outp.puts "then"
515         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
516         outp.puts "    " + plan.failCommand
517         outp.puts "else"
518         outp.puts "    " + plan.successCommand
519         outp.puts "fi"
520     }
521 end
522
523 # Error handler for tests that report error by saying "failed!", and are expected to
524 # fail. This is used by Mozilla tests.
525 def mozillaFailErrorHandler
526     Proc.new {
527         | outp, plan |
528         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
529
530         outp.puts "if test -e #{plan.failFile}"
531         outp.puts "then"
532         outp.puts "    " + plan.successCommand
533         outp.puts "elif grep -i -q failed! #{outputFilename}"
534         outp.puts "then"
535         outp.puts "    " + plan.successCommand
536         outp.puts "else"
537         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
538         outp.puts "    " + plan.failCommand
539         outp.puts "fi"
540     }
541 end
542
543 # Error handler for tests that report error by saying "failed!", and are expected to have
544 # an exit code of 3.
545 def mozillaExit3ErrorHandler
546     Proc.new {
547         | outp, plan |
548         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
549
550         outp.puts "if test -e #{plan.failFile}"
551         outp.puts "then"
552         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
553         outp.puts "    then"
554         outp.puts "        if grep -i -q failed! #{outputFilename}"
555         outp.puts "        then"
556         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
557         outp.puts "            " + plan.failCommand
558         outp.puts "        else"
559         outp.puts "            " + plan.successCommand
560         outp.puts "        fi"
561         outp.puts "    else"
562         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
563         outp.puts "        " + plan.failCommand
564         outp.puts "    fi"
565         outp.puts "else"
566         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
567         outp.puts "    " + plan.failCommand
568         outp.puts "fi"
569     }
570 end
571
572 $runCommandOptions = {}
573
574 class Plan
575     attr_reader :directory, :arguments, :name, :outputHandler, :errorHandler
576     attr_accessor :index
577     
578     def initialize(directory, arguments, name, outputHandler, errorHandler)
579         @directory = directory
580         @arguments = arguments
581         @name = name
582         @outputHandler = outputHandler
583         @errorHandler = errorHandler
584         @isSlow = !!$runCommandOptions[:isSlow]
585     end
586     
587     def shellCommand
588         # It's important to remember that the test is actually run in a subshell, so if we change directory
589         # in the subshell when we return we will be in our original directory. This is nice because we don't
590         # have to bend over backwards to do things relative to the root.
591         script = "(cd ../#{Shellwords.shellescape(@directory.to_s)} && ("
592         $envVars.each { |var| script += "export " << var << "; " }
593         script += "\"$@\" " + escapeAll(@arguments) + "))"
594         return script
595     end
596     
597     def reproScriptCommand
598         # We have to find our way back to the .runner directory since that's where all of the relative
599         # paths assume they start out from.
600         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
601         script += "cd $CURRENT_DIR\n"
602         Pathname.new(@name).dirname.each_filename {
603             | pathComponent |
604             script += "cd ..\n"
605         }
606         script += "cd .runner\n"
607
608         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
609         script += "export JSC_timeout=#{Shellwords.shellescape(ENV['JSC_timeout'])}\n"
610         $envVars.each { |var| script += "export " << var << "\n" }
611         script += "#{shellCommand} || exit 1"
612         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
613     end
614     
615     def failCommand
616         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
617     end
618     
619     def successCommand
620         if $progressMeter or $verbosity >= 2
621             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
622         else
623             "rm -f #{failFile}"
624         end
625     end
626     
627     def failFile
628         "test_fail_#{@index}"
629     end
630     
631     def writeRunScript(filename)
632         File.open(filename, "w") {
633             | outp |
634             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
635             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
636             cmd += @outputHandler.call(@name)
637             if $verbosity >= 3
638                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
639             end
640             outp.puts cmd
641             @errorHandler.call(outp, self)
642         }
643     end
644 end
645
646 $uniqueFilenameCounter = 0
647 def uniqueFilename(extension)
648     payloadDir = $outputDir + "_payload"
649     Dir.mkdir payloadDir unless payloadDir.directory?
650     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
651     $uniqueFilenameCounter += 1
652     result
653 end
654
655 def baseOutputName(kind)
656     "#{$collectionName}/#{$benchmark}.#{kind}"
657 end
658
659 def addRunCommand(kind, command, outputHandler, errorHandler)
660     $didAddRunCommand = true
661     name = baseOutputName(kind)
662     if $filter and name !~ $filter
663         return
664     end
665     plan = Plan.new($benchmarkDirectory, command, name, outputHandler, errorHandler)
666     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
667         $runlist.unshift plan
668     else
669         $runlist << plan
670     end
671 end
672
673 # Returns true if there were run commands found in the file ($benchmarkDirectory +
674 # $benchmark), in which case those run commands have already been executed. Otherwise
675 # returns false, in which case you're supposed to add your own run commands.
676 def parseRunCommands
677     oldDidAddRunCommand = $didAddRunCommand
678     $didAddRunCommand = false
679
680     Dir.chdir($outputDir) {
681         File.open($benchmarkDirectory + $benchmark) {
682             | inp |
683             inp.each_line {
684                 | line |
685                 begin
686                     doesMatch = line =~ /^\/\/@/
687                 rescue Exception => e
688                     # Apparently this happens in the case of some UTF8 stuff in some files, where
689                     # Ruby tries to be strict and throw exceptions.
690                     next
691                 end
692                 next unless doesMatch
693                 eval $~.post_match
694             }
695         }
696     }
697
698     result = $didAddRunCommand
699     $didAddRunCommand = result or oldDidAddRunCommand
700     result
701 end
702
703 def slow!
704     $runCommandOptions[:isSlow] = true
705 end
706
707 def run(kind, *options)
708     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
709 end
710
711 def runDefault
712     run("default")
713 end
714
715 def runWithRAMSize(size)
716     run("ram-size-#{size}", "--forceRAMSize=#{size}")
717 end
718
719 def runNoJIT
720     run("no-jit", "--useJIT=false")
721 end
722
723 def runNoLLInt
724     if $jitTests
725         run("no-llint", "--useLLInt=false")
726     end
727 end
728
729 def runNoCJITValidate
730     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *NO_CJIT_OPTIONS)
731 end
732
733 def runNoCJITValidatePhases
734     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
735 end
736
737 def runDefaultFTL
738     run("default-ftl", *FTL_OPTIONS) if $enableFTL
739 end
740
741 def runFTLNoCJIT
742     run("ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
743 end
744
745 def runFTLNoCJITValidate
746     run("ftl-no-cjit-validate", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
747 end
748
749 def runFTLNoCJITNoInlineValidate
750     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
751 end
752
753 def runFTLNoCJITOSRValidation
754     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
755 end
756
757 def runDFGEager
758     run("dfg-eager", *EAGER_OPTIONS)
759 end
760
761 def runDFGEagerNoCJITValidate
762     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
763 end
764
765 def runFTLEager
766     run("ftl-eager", *(FTL_OPTIONS + EAGER_OPTIONS)) if $enableFTL
767 end
768
769 def runFTLEagerNoCJITValidate
770     run("ftl-eager-no-cjit", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
771 end
772
773 def runFTLEagerNoCJITOSRValidation
774     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
775 end
776
777 def runAlwaysTriggerCopyPhase
778     run("always-trigger-copy-phase", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
779 end
780
781 def runNoCJITNoASO
782     run("no-cjit-no-aso", "--enableArchitectureSpecificOptimizations=false", *NO_CJIT_OPTIONS)
783 end
784
785 def runFTLNoCJITNoSimpleOpt
786     run("ftl-no-cjit-no-simple-opt", "--llvmSimpleOpt=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
787 end
788
789 def runNoCJITNoAccessInlining
790     run("no-cjit-no-access-inlining", "--enableAccessInlining=false", *NO_CJIT_OPTIONS)
791 end
792
793 def runFTLNoCJITNoAccessInlining
794     run("ftl-no-cjit-no-access-inlining", "--enableAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
795 end
796
797 def runFTLNoCJITSmallPool
798     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
799 end
800
801 def runMiscNoCJITTest(*options)
802     run("misc-no-cjit", *(NO_CJIT_OPTIONS + options))
803 end
804
805 def runMiscFTLNoCJITTest(*options)
806     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + options))
807 end
808
809 def runDFGMaximalFlushPhase
810     run("dfg-maximal-flush-validate-no-cjit", "--validateGraph=true", "--enableMaximalFlushInsertionPhase=true", *NO_CJIT_OPTIONS)
811 end
812
813 def defaultRun
814     if $quickMode
815         defaultQuickRun
816     else
817         runDefault
818         runAlwaysTriggerCopyPhase
819         if $jitTests
820             runNoLLInt
821             runNoCJITValidatePhases
822             runDFGEager
823             runDFGEagerNoCJITValidate
824             runDefaultFTL
825             runFTLNoCJITValidate
826             runFTLNoCJITNoInlineValidate
827             runFTLEager
828             runFTLEagerNoCJITValidate
829             runFTLNoCJITSmallPool
830             runDFGMaximalFlushPhase
831         end
832     end
833 end
834
835 def defaultQuickRun
836     if $enableFTL and $jitTests
837         runDefaultFTL
838         runFTLNoCJITValidate
839     else
840         runDefault
841         if $jitTests
842             runNoCJITValidate
843         end
844     end
845 end
846
847 def defaultSpotCheckNoMaximalFlush
848     defaultQuickRun
849     runFTLNoCJITNoSimpleOpt
850     runFTLNoCJITOSRValidation
851     runNoCJITNoAccessInlining
852     runFTLNoCJITNoAccessInlining
853 end
854
855 def defaultSpotCheck
856     defaultSpotCheckNoMaximalFlush
857     runDFGMaximalFlushPhase
858 end
859
860 # This is expected to not do eager runs because eager runs can have a lot of recompilations
861 # for reasons that don't arise in the real world. It's used for tests that assert convergence
862 # by counting recompilations.
863 def defaultNoEagerRun
864     runDefault
865     runAlwaysTriggerCopyPhase
866     if $jitTests
867         runNoLLInt
868         runNoCJITValidatePhases
869         runDefaultFTL
870         runFTLNoCJITValidate
871         runFTLNoCJITNoInlineValidate
872     end
873 end
874
875 def runProfiler
876     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
877         skip
878         return
879     end
880
881     profilerOutput = uniqueFilename(".json")
882     if $canRunDisplayProfilerOutput
883         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
884     else
885         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
886         run("profiler-simple", "-p", profilerOutput.to_s)
887     end
888 end
889
890 def runExceptionFuzz
891     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
892     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
893 end
894
895 def runExecutableAllocationFuzz(name, *options)
896     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
897     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
898 end
899
900 def runTypeProfiler
901     if !$jitTests
902         return
903     end
904
905     if $enableFTL
906         run("ftl-no-cjit-type-profiler", "--enableTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
907     else
908         run("no-cjit-type-profiler", "--enableTypeProfiler=true", *NO_CJIT_OPTIONS)
909     end
910 end
911
912 def runControlFlowProfiler
913     if !$jitTests
914         return
915     end
916
917     if $enableFTL
918         run("ftl-no-cjit-type-profiler", "--enableControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
919     else
920         run("no-cjit-type-profiler", "--enableControlFlowProfiler=true", *NO_CJIT_OPTIONS)
921     end
922 end
923
924 def runES6(mode)
925     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
926     case mode
927     when :normal
928         errorHandler = simpleErrorHandler
929     when :fail
930         errorHandler = expectedFailErrorHandler
931     else
932         raise "Invalid mode: #{mode}"
933     end
934     addRunCommand("default", args, noisyOutputHandler, errorHandler)
935 end
936
937 def runModules
938     run("default-modules", "-m")
939     run("always-trigger-copy-phase-modules", "-m", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
940
941     if !$jitTests
942         return
943     end
944
945     run("no-llint-modules", "-m", "--useLLInt=false")
946     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
947     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
948     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
949     if $enableFTL
950         run("default-ftl-modules", "-m", *FTL_OPTIONS)
951         run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
952         run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
953         run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
954         run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
955         run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
956     end
957 end
958
959 def runLayoutTest(kind, *options)
960     raise unless $benchmark.to_s =~ /\.js$/
961     testName = $~.pre_match
962     if kind
963         kind = "layout-" + kind
964     else
965         kind = "layout"
966     end
967
968     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
969     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
970
971     args = [pathToVM.to_s] + BASE_OPTIONS + options +
972         [(Pathname.new("resources") + "standalone-pre.js").to_s,
973          $benchmark.to_s,
974          (Pathname.new("resources") + "standalone-post.js").to_s]
975     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
976 end
977
978 def runLayoutTestDefault
979     runLayoutTest(nil)
980 end
981
982 def runLayoutTestNoLLInt
983     runLayoutTest("no-llint", "--useLLInt=false")
984 end
985
986 def runLayoutTestNoCJIT
987     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
988 end
989
990 def runLayoutTestDFGEagerNoCJIT
991     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
992 end
993
994 def runLayoutTestDefaultFTL
995     runLayoutTest("ftl", "--testTheFTL=true", *FTL_OPTIONS) if $enableFTL
996 end
997
998 def runLayoutTestFTLNoCJIT
999     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
1000 end
1001
1002 def runLayoutTestFTLEagerNoCJIT
1003     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
1004 end
1005
1006 def noFTLRunLayoutTest
1007     if !$jitTests
1008         return
1009     end
1010
1011     runLayoutTestNoLLInt
1012     runLayoutTestNoCJIT
1013     runLayoutTestDFGEagerNoCJIT
1014 end
1015
1016 def defaultQuickRunLayoutTest
1017     runLayoutTestDefault
1018     if $jitTests
1019         runLayoutTestFTLNoCJIT
1020         runLayoutTestFTLEagerNoCJIT
1021     end
1022 end
1023
1024 def defaultRunLayoutTest
1025     if $quickMode
1026         defaultQuickRunLayoutTest
1027     else
1028         runLayoutTestDefault
1029         if $jitTests
1030             noFTLRunLayoutTest
1031             runLayoutTestDefaultFTL
1032             runLayoutTestFTLNoCJIT
1033             runLayoutTestFTLEagerNoCJIT
1034         end
1035     end
1036 end
1037
1038 def noNoLLIntRunLayoutTest
1039     runLayoutTestDefault
1040     if $jitTests
1041         runLayoutTestNoCJIT
1042         runLayoutTestDFGEagerNoCJIT
1043         runLayoutTestDefaultFTL
1044         runLayoutTestFTLNoCJIT
1045         runLayoutTestFTLEagerNoCJIT
1046     end
1047 end
1048
1049 def prepareExtraRelativeFiles(extraFiles, destination)
1050     Dir.chdir($outputDir) {
1051         extraFiles.each {
1052             | file |
1053             FileUtils.cp $extraFilesBaseDir + file, destination + file
1054         }
1055     }
1056 end
1057
1058 def baseDirForCollection(collectionName)
1059     Pathname(".tests") + collectionName
1060 end
1061
1062 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1063     raise unless absoluteBase.absolute?
1064     Dir.chdir($outputDir) {
1065         collectionBaseDir = baseDirForCollection($collectionName)
1066         extraFiles.each {
1067             | file |
1068             destination = collectionBaseDir + file
1069             FileUtils.mkdir_p destination.dirname unless destination.directory?
1070             FileUtils.cp absoluteBase + file, destination
1071         }
1072     }
1073 end
1074
1075 def runMozillaTest(kind, mode, extraFiles, *options)
1076     if kind
1077         kind = "mozilla-" + kind
1078     else
1079         kind = "mozilla"
1080     end
1081     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1082     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1083     case mode
1084     when :normal
1085         errorHandler = mozillaErrorHandler
1086     when :negative
1087         errorHandler = mozillaExit3ErrorHandler
1088     when :fail
1089         errorHandler = mozillaFailErrorHandler
1090     when :skip
1091         return
1092     else
1093         raise "Invalid mode: #{mode}"
1094     end
1095     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1096 end
1097
1098 def runMozillaTestDefault(mode, *extraFiles)
1099     runMozillaTest(nil, mode, extraFiles)
1100 end
1101
1102 def runMozillaTestDefaultFTL(mode, *extraFiles)
1103     runMozillaTest("ftl", mode, extraFiles, *FTL_OPTIONS) if $enableFTL
1104 end
1105
1106 def runMozillaTestLLInt(mode, *extraFiles)
1107     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1108 end
1109
1110 def runMozillaTestBaselineJIT(mode, *extraFiles)
1111     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1112 end
1113
1114 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1115     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1116 end
1117
1118 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1119     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
1120 end
1121
1122 def defaultQuickRunMozillaTest(mode, *extraFiles)
1123     if $enableFTL and $jitTests
1124         runMozillaTestDefaultFTL(mode, *extraFiles)
1125         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1126     else
1127         runMozillaTestDefault(mode, *extraFiles)
1128         if $jitTests
1129             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1130         end
1131     end
1132 end
1133
1134 def defaultRunMozillaTest(mode, *extraFiles)
1135     if $quickMode
1136         defaultQuickRunMozillaTest(mode, *extraFiles)
1137     else
1138         runMozillaTestDefault(mode, *extraFiles)
1139         if $jitTests
1140             runMozillaTestLLInt(mode, *extraFiles)
1141             runMozillaTestBaselineJIT(mode, *extraFiles)
1142             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1143             runMozillaTestDefaultFTL(mode, *extraFiles)
1144             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1145         end
1146     end
1147 end
1148
1149 def runNoisyTest(kind, *options)
1150     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
1151 end
1152
1153 def runNoisyTestDefault
1154     runNoisyTest("default")
1155 end
1156
1157 def runNoisyTestDefaultFTL
1158     runNoisyTest("ftl", *FTL_OPTIONS) if $enableFTL
1159 end
1160
1161 def runNoisyTestNoCJIT
1162     runNoisyTest($enableFTL ? "ftl-no-cjit" : "no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(($enableFTL ? FTL_OPTIONS : []) + NO_CJIT_OPTIONS))
1163 end
1164
1165 def runNoisyTestEagerNoCJIT
1166     runNoisyTest($enableFTL ? "ftl-eager-no-cjit" : "eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(($enableFTL ? FTL_OPTIONS : []) + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1167 end
1168
1169 def defaultRunNoisyTest
1170     runNoisyTestDefault
1171     if $jitTests
1172         runNoisyTestDefaultFTL
1173         runNoisyTestNoCJIT
1174         runNoisyTestEagerNoCJIT
1175     end
1176 end
1177
1178 def skip
1179     $didAddRunCommand = true
1180     puts "Skipping #{$collectionName}/#{$benchmark}"
1181 end
1182
1183 def largeHeap
1184     if $memoryLimited
1185         $didAddRunCommand = true
1186         puts "Skipping #{$collectionName}/#{$benchmark}"
1187     end
1188 end
1189
1190 def allJSFiles(path)
1191     if path.file?
1192         [path]
1193     else
1194         result = []
1195         Dir.foreach(path) {
1196             | filename |
1197             next unless filename =~ /\.js$/
1198             next unless (path + filename).file?
1199             result << path + filename
1200         }
1201         result
1202     end
1203 end
1204
1205 def uniqueifyName(names, name)
1206     result = name.to_s
1207     toAdd = 1
1208     while names[result]
1209         result = "#{name}-#{toAdd}"
1210         toAdd += 1
1211     end
1212     names[result] = true
1213     result
1214 end
1215
1216 def simplifyCollectionName(collectionPath)
1217     outerDir = collectionPath.dirname
1218     name = collectionPath.basename
1219     lastName = name
1220     if collectionPath.directory?
1221         while lastName.to_s =~ /test/
1222             lastName = outerDir.basename
1223             name = lastName + name
1224             outerDir = outerDir.dirname
1225         end
1226     end
1227     uniqueifyName($collectionNames, name)
1228 end
1229
1230 def prepareCollection(name)
1231     FileUtils.mkdir_p $outputDir + name
1232
1233     absoluteCollection = $collection.realpath
1234
1235     Dir.chdir($outputDir) {
1236         bundleDir = baseDirForCollection(name)
1237
1238         # Create the proper directory structures.
1239         FileUtils.mkdir_p bundleDir
1240         if bundleDir.basename == $collection.basename
1241             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1242             $collection = bundleDir
1243         else
1244             FileUtils.cp_r absoluteCollection, bundleDir
1245             $collection = bundleDir + $collection.basename
1246         end
1247
1248         $extraFilesBaseDir = absoluteCollection
1249     }
1250 end
1251
1252 $collectionNames = {}
1253
1254 def handleCollectionFile(collection)
1255     collectionName = simplifyCollectionName(collection)
1256    
1257     paths = {}
1258     subCollections = []
1259     YAML::load(IO::read(collection)).each {
1260         | entry |
1261         if entry["collection"]
1262             subCollections << entry["collection"]
1263             next
1264         end
1265         
1266         if Pathname.new(entry["path"]).absolute?
1267             raise "Absolute path: " + entry["path"] + " in #{collection}"
1268         end
1269         
1270         if paths[entry["path"]]
1271             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1272         end
1273         
1274         subCollection = collection.dirname + entry["path"]
1275         
1276         if subCollection.file?
1277             subCollectionName = Pathname.new(entry["path"]).dirname
1278         else
1279             subCollectionName = entry["path"]
1280         end
1281         
1282         $collection = subCollection
1283         $collectionName = Pathname.new(collectionName)
1284         Pathname.new(subCollectionName).each_filename {
1285             | filename |
1286             next if filename =~ /^\./
1287             $collectionName += filename
1288         }
1289         $collectionName = $collectionName.to_s
1290         
1291         prepareCollection($collectionName)
1292       
1293         Dir.chdir($outputDir) {
1294             pathsToSearch = [$collection]
1295             if entry["tests"]
1296                 if entry["tests"].is_a? Array
1297                     pathsToSearch = entry["tests"].map {
1298                         | testName |
1299                         pathsToSearch[0] + testName
1300                     }
1301                 else
1302                     pathsToSearch[0] += entry["tests"]
1303                 end
1304             end
1305             pathsToSearch.each {
1306                 | pathToSearch |
1307                 allJSFiles(pathToSearch).each {
1308                     | path |
1309                     
1310                     $benchmark = path.basename
1311                     $benchmarkDirectory = path.dirname
1312                     
1313                     $runCommandOptions = {}
1314                     eval entry["cmd"]
1315                 }
1316             }
1317         }
1318     }
1319     
1320     subCollections.each {
1321         | subCollection |
1322         handleCollection(collection.dirname + subCollection)
1323     }
1324 end
1325
1326 def handleCollectionDirectory(collection)
1327     collectionName = simplifyCollectionName(collection)
1328     
1329     $collection = collection
1330     $collectionName = collectionName
1331     prepareCollection(collectionName)
1332    
1333     Dir.chdir($outputDir) {
1334         $benchmarkDirectory = $collection
1335         allJSFiles($collection).each {
1336             | path |
1337             
1338             $benchmark = path.basename
1339             
1340             $runCommandOptions = {}
1341             defaultRun unless parseRunCommands
1342         }
1343     }
1344 end
1345
1346 def handleCollection(collection)
1347     collection = Pathname.new(collection)
1348     
1349     if collection.file?
1350         handleCollectionFile(collection)
1351     else
1352         handleCollectionDirectory(collection)
1353     end
1354 end
1355
1356 def appendFailure(plan)
1357     File.open($outputDir + "failed", "a") {
1358         | outp |
1359         outp.puts plan.name
1360     }
1361     $numFailures += 1
1362 end
1363
1364 def prepareBundle
1365     raise if $bundle
1366
1367     if $doNotMessWithVMPath
1368         if !$remote and !$tarball
1369             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1370             $jscPath = Pathname.new($jscPath).realpath
1371         else
1372             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1373         end
1374     else
1375         originalJSCPath = $jscPath
1376         vmDir = $outputDir + ".vm"
1377         FileUtils.mkdir_p vmDir
1378         
1379         frameworkPath = frameworkFromJSCPath($jscPath)
1380         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1381         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1382         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1383
1384         if frameworkPath
1385             source = frameworkPath
1386             destination = Pathname.new(".vm")
1387         else
1388             source = originalJSCPath
1389             destination = $jscPath
1390
1391             Dir.chdir($outputDir) {
1392                 FileUtils.mkdir_p $jscPath.dirname
1393             }
1394         end
1395
1396         Dir.chdir($outputDir) {
1397             if $copyVM
1398                 FileUtils.cp_r source, destination
1399             else
1400                 begin 
1401                     FileUtils.ln_s source, destination
1402                 rescue Exception
1403                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1404                     FileUtils.cp_r source, destination
1405                 end
1406             end
1407
1408             if $remote and $hostOS == "linux"
1409                 if $enableFTL
1410                     begin
1411                         FileUtils.cp_r originalJSCPath + "../../lib/libllvmForJSC.so" , $jscPath.dirname
1412                     rescue
1413                         $stderr.puts "Warning: unable to copy libllvmForJSC.so to the bundle."
1414                     end
1415                 end
1416                 begin
1417                     dependencies = `ldd #{source}`
1418                     dependencies.split(/\n/).each {
1419                         | dependency |
1420                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1421                     }
1422                 rescue
1423                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1424                 end
1425             end
1426         }
1427     end
1428     
1429     Dir.chdir($outputDir) {
1430         FileUtils.cp_r HELPERS_PATH, ".helpers"
1431     }
1432
1433     ARGV.each {
1434         | collection |
1435         handleCollection(collection)
1436     }
1437
1438     puts
1439 end
1440
1441 def cleanOldResults
1442     raise unless $bundle
1443
1444     eachResultFile($outputDir) {
1445         | path |
1446         FileUtils.rm_f path
1447     }
1448 end
1449
1450 def cleanEmptyResultFiles
1451     eachResultFile($outputDir) {
1452         | path |
1453         next unless path.basename.to_s =~ /\.out$/
1454         next unless FileTest.size(path) == 0
1455         FileUtils.rm_f path
1456     }
1457 end
1458
1459 def eachResultFile(startingDir, &block)
1460     dirsToClean = [startingDir]
1461     until dirsToClean.empty?
1462         nextDir = dirsToClean.pop
1463         Dir.foreach(nextDir) {
1464             | entry |
1465             next if entry =~ /^\./
1466             path = nextDir + entry
1467             if path.directory?
1468                 dirsToClean.push(path)
1469             else
1470                 block.call(path)
1471             end
1472         }
1473     end
1474 end
1475
1476 def prepareTestRunner
1477     raise if $bundle
1478
1479     $runlist.each_with_index {
1480         | plan, index |
1481         plan.index = index
1482     }
1483
1484     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1485     toDelete = []
1486     Dir.foreach($runnerDir) {
1487         | filename |
1488         if filename =~ /^test_/
1489             toDelete << filename
1490         end
1491     }
1492     
1493     toDelete.each {
1494         | filename |
1495         File.unlink($runnerDir + filename)
1496     }
1497
1498     $runlist.each {
1499         | plan |
1500         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1501     }
1502
1503     case $testRunnerType
1504     when :make
1505         prepareMakeTestRunner
1506     when :shell
1507         prepareShellTestRunner
1508     else
1509         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1510     end
1511 end
1512
1513 def prepareShellTestRunner
1514     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
1515 end
1516
1517 def prepareMakeTestRunner
1518     # The goals of our parallel test runner are scalability and simplicity. The
1519     # simplicity part is particularly important. We don't want to have to have
1520     # a full-time contributor just philosophising about parallel testing.
1521     #
1522     # As such, we just pass off all of the hard work to 'make'. This creates a
1523     # dummy directory ("$outputDir/.runner") in which we create a dummy
1524     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
1525     # That is, for each test we know we will run, there is a rule in the
1526     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
1527     # Makefile results in 'make' doing all of the hard work:
1528     #
1529     # - Load balancing just works. Most systems have a great load balancer in
1530     #   'make'. If your system doesn't then just install a real 'make'.
1531     #
1532     # - Interruptions just work. For example Ctrl-C handling in 'make' is
1533     #   exactly right. You don't have to worry about zombie processes.
1534     #
1535     # We then do some tricks to make failure detection work and to make this
1536     # totally sound. If a test fails, we don't want the whole 'make' job to
1537     # stop. We also don't have any facility for makefile-escaping of path names.
1538     # We do have such a thing for shell-escaping, though. We fix both problems
1539     # by having the actual work for each of the test rules be done in a shell
1540     # script on the side. There is one such script per test. The script responds
1541     # to failure by printing something on the console and then touching a
1542     # failure file for that test, but then still returns 0. This makes 'make'
1543     # continue past that failure and complete all the tests anyway.
1544     #
1545     # In the end, this script collects all of the failures by searching for
1546     # files in the .runner directory whose name matches /^test_fail_/, where
1547     # the thing after the 'fail_' is the test index. Those are the files that
1548     # would be created by the test scripts if they detect failure. We're
1549     # basically using the filesystem as a concurrent database of test failures.
1550     # Even if two tests fail at the same time, since they're touching different
1551     # files we won't miss any failures.
1552     runIndices = []
1553     $runlist.each {
1554         | plan |
1555         runIndices << plan.index
1556     }
1557     
1558     File.open($runnerDir + "Makefile", "w") {
1559         | outp |
1560         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1561         runIndices.each {
1562             | index |
1563             plan = $runlist[index]
1564             outp.puts "test_done_#{index}:"
1565             outp.puts "\tsh test_script_#{plan.index}"
1566         }
1567     }
1568 end
1569
1570 def cleanRunnerDirectory
1571     raise unless $bundle
1572     Dir.foreach($runnerDir) {
1573         | filename |
1574         next unless filename =~ /^test_fail/
1575         FileUtils.rm_f $runnerDir + filename
1576     }
1577 end
1578
1579 def sshRead(cmd)
1580     raise unless $remote
1581
1582     result = ""
1583     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1584       | inp |
1585       inp.each_line {
1586         | line |
1587         result += line
1588       }
1589     }
1590     raise "#{$?}" unless $?.success?
1591     result
1592 end
1593
1594 def runCommandOnTester(cmd)
1595     if $remote
1596         result = sshRead(cmd)
1597     else
1598         result = `#{cmd}`
1599     end
1600 end
1601
1602 def numberOfProcessors
1603     begin
1604         numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1605     rescue
1606         numProcessors = 0
1607     end
1608
1609     if numProcessors == 0
1610         begin
1611             numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1612         rescue
1613             numProcessors == 0
1614         end
1615     end
1616
1617     if numProcessors == 0
1618         numProcessors = 1
1619     end
1620     return numProcessors
1621 end
1622
1623 def runAndMonitorTestRunnerCommand(*cmd)
1624     numberOfTests = 0
1625     Dir.chdir($runnerDir) {
1626         # -1 for the runscript, and -2 for '..' and '.'
1627         numberOfTests = Dir.entries(".").count - 3
1628     }
1629     unless $progressMeter
1630         mysys(cmd.join(' '))
1631     else
1632        running = {}
1633        didRun = {}
1634        didFail = {}
1635        blankLine = true
1636        prevStringLength = 0
1637        IO.popen(cmd.join(' '), mode="r") {
1638            | inp |
1639            inp.each_line {
1640                | line |
1641                line.chomp!
1642                if line =~ /^Running /
1643                    running[$~.post_match] = true
1644                elsif line =~ /^PASS: /
1645                    didRun[$~.post_match] = true
1646                elsif line =~ /^FAIL: /
1647                    didRun[$~.post_match] = true
1648                    didFail[$~.post_match] = true
1649                else
1650                    unless blankLine
1651                        print("\r" + " " * prevStringLength + "\r")
1652                    end
1653                    puts line
1654                    blankLine = true
1655                end
1656
1657                def lpad(str, chars)
1658                    str = str.to_s
1659                    if str.length > chars
1660                        str
1661                    else
1662                       "%#{chars}s"%(str)
1663                    end
1664                end
1665
1666                string  = ""
1667                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1668                unless didFail.empty?
1669                    string += " (failed #{didFail.size})"
1670                end
1671                string += " "
1672                (running.size - didRun.size).times {
1673                    string += "."
1674                }
1675                if string.length < prevStringLength
1676                    print string
1677                    print(" " * (prevStringLength - string.length))
1678                end
1679                print string
1680                prevStringLength = string.length
1681                blankLine = false
1682                $stdout.flush
1683            }
1684        }
1685        puts
1686        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1687     end
1688 end
1689
1690 def runTestRunner
1691     case $testRunnerType
1692     when :shell
1693         testRunnerCommand = "sh runscript"
1694     when :make
1695         testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
1696     else
1697         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1698     end
1699
1700     if $remote
1701         if !$remoteDirectory
1702             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1703         end
1704         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1705         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1706         remoteScript = "\""
1707         remoteScript += "cd #{$remoteDirectory} && "
1708         remoteScript += "rm -rf #{$outputDir.basename} && "
1709         remoteScript += "tar xzf #{$tarFileName} && "
1710         remoteScript += "cd #{$outputDir.basename}/.runner && "
1711         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1712         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1713         remoteScript += "export JSC_timeout=#{Shellwords.shellescape(ENV['JSC_timeout'])} && "
1714         $envVars.each { |var| remoteScript += "export " << var << "\n" }
1715         remoteScript += "#{testRunnerCommand}\""
1716         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1717     else
1718         Dir.chdir($runnerDir) {
1719             runAndMonitorTestRunnerCommand(testRunnerCommand)
1720         }
1721     end
1722 end
1723
1724 def detectFailures
1725     raise if $bundle
1726
1727     if $remote
1728         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1729         output.split(/\n/).each {
1730             | line |
1731             next unless line =~ /test_fail_/
1732             appendFailure($runlist[$~.post_match.to_i])
1733         }
1734     else
1735         Dir.foreach($runnerDir) {
1736             | filename |
1737             next unless filename =~ /test_fail_/
1738             appendFailure($runlist[$~.post_match.to_i])
1739         }
1740     end
1741 end
1742
1743 def compressBundle
1744     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
1745     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1746     raise unless system(cmd)
1747 end
1748
1749 def clean(file)
1750     FileUtils.rm_rf file unless $bundle
1751 end
1752
1753 clean($outputDir + "failed")
1754 clean($outputDir + ".vm")
1755 clean($outputDir + ".helpers")
1756 clean($outputDir + ".runner")
1757 clean($outputDir + ".tests")
1758 clean($outputDir + "_payload")
1759
1760 Dir.mkdir($outputDir) unless $outputDir.directory?
1761
1762 $outputDir = $outputDir.realpath
1763 $runnerDir = $outputDir + ".runner"
1764
1765 if !$numChildProcesses
1766     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
1767         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
1768     else
1769         $numChildProcesses = numberOfProcessors
1770     end
1771 end
1772
1773 if $enableFTL and ENV["JSC_timeout"]
1774     # Currently, using the FTL is a performance regression particularly in real
1775     # (i.e. non-loopy) benchmarks. Account for this in the timeout.
1776     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i * 2).to_s
1777 end
1778
1779 if ENV["JSC_timeout"]
1780     # In the worst case, the processors just interfere with each other.
1781     # Increase the timeout proportionally to the number of processors.
1782     ENV["JSC_timeout"] = (ENV["JSC_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
1783 end
1784
1785 def runBundle
1786     raise unless $bundle
1787
1788     cleanRunnerDirectory
1789     cleanOldResults
1790     runTestRunner
1791     cleanEmptyResultFiles
1792 end
1793
1794 def runNormal
1795     raise if $bundle or $tarball
1796
1797     prepareBundle
1798     prepareTestRunner
1799     runTestRunner
1800     cleanEmptyResultFiles
1801     detectFailures
1802 end
1803
1804 def runTarball
1805     raise unless $tarball
1806
1807     prepareBundle 
1808     prepareTestRunner
1809     compressBundle
1810 end
1811
1812 def runRemote
1813     raise unless $remote
1814
1815     prepareBundle
1816     prepareTestRunner
1817     compressBundle
1818     runTestRunner
1819     detectFailures
1820 end
1821
1822 puts
1823 if $bundle
1824     runBundle
1825 elsif $remote
1826     runRemote
1827 elsif $tarball
1828     runTarball
1829 else
1830     runNormal
1831 end