41db5d4166d58673f99b3fb7596e07566873fdba
[WebKit-https.git] / Tools / Scripts / run-jsc-stress-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2013-2015 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer. 
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution. 
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 require 'fileutils'
27 require 'getoptlong'
28 require 'pathname'
29 require 'rbconfig'
30 require 'uri'
31 require 'yaml'
32
33 module URI
34     class SSH < Generic
35         DEFAULT_PORT = 22
36     end
37     @@schemes['SSH'] = SSH
38 end
39
40 class String
41     def scrub
42         encode("UTF-16be", :invalid=>:replace, :replace=>"?").encode('UTF-8')
43     end
44 end
45
46 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
47 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
48 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
49 LAYOUTTESTS_PATH = WEBKIT_PATH + "LayoutTests"
50 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
51 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
52
53 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers"
54
55 begin
56     require 'shellwords'
57 rescue Exception => e
58     $stderr.puts "Warning: did not find shellwords, not running any tests."
59     exit 0
60 end
61
62 $canRunDisplayProfilerOutput = false
63
64 begin
65     require 'rubygems'
66     require 'json'
67     require 'highline'
68     $canRunDisplayProfilerOutput = true
69 rescue Exception => e
70     $stderr.puts "Warning: did not find json or highline; some features will be disabled."
71     $stderr.puts "Run \"sudo gem install json highline\" to fix the issue."
72     $stderr.puts "Error: #{e.inspect}"
73 end
74
75 def printCommandArray(*cmd)
76     begin
77         commandArray = cmd.each{|value| Shellwords.shellescape(value.to_s)}.join(' ')
78     rescue
79         commandArray = cmd.join(' ')
80     end
81     $stderr.puts ">> #{commandArray}"
82 end
83
84 def mysys(*cmd)
85     printCommandArray(*cmd) if $verbosity >= 1
86     raise "Command failed: #{$?.inspect}" unless system(*cmd)
87 end
88
89 def escapeAll(array)
90     array.map {
91         | v |
92         raise "Detected a non-string in #{inspect}" unless v.is_a? String
93         Shellwords.shellescape(v)
94     }.join(' ')
95 end
96
97
98 $jscPath = nil
99 $doNotMessWithVMPath = false
100 $enableFTL = false
101 $jitTests = true
102 $memoryLimited = false
103 $outputDir = Pathname.new("results")
104 $verbosity = 0
105 $bundle = nil
106 $tarball = false
107 $tarFileName = "payload.tar.gz"
108 $copyVM = false
109 $testRunnerType = nil
110 $remoteUser = nil
111 $remoteHost = nil
112 $remotePort = nil
113 $remoteDirectory = nil
114 $architecture = nil
115 $hostOS = nil
116 $filter = nil
117 $envVars = []
118 $quickMode = false
119
120
121 def usage
122     puts "run-jsc-stress-tests -j <shell path> <collections path> [<collections path> ...]"
123     puts
124     puts "--jsc                (-j)   Path to JavaScriptCore build product. This option is required."
125     puts "--no-copy                   Do not copy the JavaScriptCore build product before testing."
126     puts "                            --jsc specifies an already present JavaScriptCore to test."
127     puts "--ftl-jit                   Indicate that we have the FTL JIT."
128     puts "--memory-limited            Indicate that we are targeting the test for a memory limited device."
129     puts "                            Skip tests tagged with //@large-heap"
130     puts "--no-jit                    Do not run JIT specific tests."
131     puts "--output-dir         (-o)   Path where to put results. Default is #{$outputDir}."
132     puts "--verbose            (-v)   Print more things while running."
133     puts "--run-bundle                Runs a bundle previously created by run-jsc-stress-tests."
134     puts "--tarball [fileName]        Creates a tarball of the final bundle.  Use name if supplied for tar file."
135     puts "--arch                      Specify architecture instead of determining from JavaScriptCore build."
136     puts "                            e.g. x86, x86_64, arm."
137     puts "--os                        Specify os instead of determining from JavaScriptCore build."
138     puts "                            e.g. darwin, linux & windows."
139     puts "--shell-runner              Uses the shell-based test runner instead of the default make-based runner."
140     puts "                            In general the shell runner is slower than the make runner."
141     puts "--make-runner               Uses the faster make-based runner."
142     puts "--remote                    Specify a remote host on which to run tests from command line argument."
143     puts "--remote-config-file        Specify a remote host on which to run tests from JSON file."
144     puts "--child-processes    (-c)   Specify the number of child processes."
145     puts "--filter                    Only run tests whose name matches the given regular expression."
146     puts "--help               (-h)   Print this message."
147     puts "--env-vars                  Add a list of environment variables to set before running jsc."
148     puts "                            Each environment variable should be separated by a space."
149     puts "                            e.g. \"foo=bar x=y\" (no quotes). Note, if you pass DYLD_FRAMEWORK_PATH"
150     puts "                            it will override the default value."
151     puts "--quick              (-q)   Only run with the default and no-cjit-validate modes."
152     exit 1
153 end
154
155 jscArg = nil
156 ifJSCArgIsntProvidedAreWeReleaseBuild = true
157
158 GetoptLong.new(['--help', '-h', GetoptLong::NO_ARGUMENT],
159                ['--jsc', '-j', GetoptLong::REQUIRED_ARGUMENT],
160                ['--no-copy', GetoptLong::NO_ARGUMENT],
161                ['--ftl-jit', GetoptLong::NO_ARGUMENT],
162                ['--memory-limited', GetoptLong::NO_ARGUMENT],
163                ['--no-jit', GetoptLong::NO_ARGUMENT],
164                ['--output-dir', '-o', GetoptLong::REQUIRED_ARGUMENT],
165                ['--run-bundle', GetoptLong::REQUIRED_ARGUMENT],
166                ['--tarball', GetoptLong::OPTIONAL_ARGUMENT],
167                ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
168                ['--arch', GetoptLong::REQUIRED_ARGUMENT],
169                ['--os', GetoptLong::REQUIRED_ARGUMENT],
170                ['--shell-runner', GetoptLong::NO_ARGUMENT],
171                ['--make-runner', GetoptLong::NO_ARGUMENT],
172                ['--remote', GetoptLong::REQUIRED_ARGUMENT],
173                ['--remote-config-file', GetoptLong::REQUIRED_ARGUMENT],
174                ['--child-processes', '-c', GetoptLong::REQUIRED_ARGUMENT],
175                ['--filter', GetoptLong::REQUIRED_ARGUMENT],
176                ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
177                ['--env-vars', GetoptLong::REQUIRED_ARGUMENT],
178                ['--debug', GetoptLong::NO_ARGUMENT],
179                ['--release', GetoptLong::NO_ARGUMENT],
180                ['--quick', '-q', GetoptLong::NO_ARGUMENT]).each {
181     | opt, arg |
182     case opt
183     when '--help'
184         usage
185     when '--jsc'
186         jscArg = arg
187     when '--no-copy'
188         $doNotMessWithVMPath = true
189     when '--output-dir'
190         $outputDir = Pathname.new(arg)
191     when '--ftl-jit'
192         $enableFTL = true
193     when '--memory-limited'
194         $memoryLimited = true
195     when '--no-jit'
196         $jitTests = false
197     when '--verbose'
198         $verbosity += 1
199     when '--run-bundle'
200         $bundle = Pathname.new(arg)
201     when '--tarball'
202         $tarball = true
203         $copyVM = true
204         $tarFileName = arg unless arg == ''
205     when '--force-vm-copy'
206         $copyVM = true
207     when '--shell-runner'
208         $testRunnerType = :shell
209     when '--make-runner'
210         $testRunnerType = :make
211     when '--remote'
212         $copyVM = true
213         $tarball = true
214         $remote = true
215         uri = URI("ssh://" + arg)
216         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
217     when '--remote-config-file'
218         $remoteConfigFile = arg
219     when '--child-processes'
220         $numChildProcesses = arg.to_i
221     when '--filter'
222         $filter = Regexp.new(arg)
223     when '--arch'
224         $architecture = arg
225     when '--os'
226         $hostOS = arg
227     when '--env-vars'
228         $envVars = arg.gsub(/\s+/, ' ').split(' ')
229     when '--quick'
230         $quickMode = true
231     when '--debug'
232         ifJSCArgIsntProvidedAreWeReleaseBuild = false
233     when '--release'
234         ifJSCArgIsntProvidedAreWeReleaseBuild = true
235     end
236 }
237
238 if $remoteConfigFile
239     file = File.read($remoteConfigFile)
240     config = JSON.parse(file)
241
242     if !$remote and config['remote']
243         $copyVM = true
244         $tarball = true
245         $remote = true
246         uri = URI("ssh://" + config['remote'])
247         $remoteUser, $remoteHost, $remotePort = uri.user, uri.host, uri.port
248     end
249
250     if config['remoteDirectory']
251         $remoteDirectory = config['remoteDirectory']
252     end
253 end
254
255 unless jscArg
256     # If we're not provided a JSC path, try to come up with a sensible JSC path automagically.
257     command = SCRIPTS_PATH.join("webkit-build-directory").to_s
258     command += ifJSCArgIsntProvidedAreWeReleaseBuild ? " --release" : " --debug"
259     command += " --executablePath"
260
261     output = `#{command}`.split("\n")
262     if !output.length
263         $stderr.puts "Error: must specify --jsc <path>"
264         exit 1
265     end
266
267     output = output[0]
268     jscArg = Pathname.new(output).join("jsc")
269     jscArg = Pathname.new(output).join("JavaScriptCore.framework", "Resources", "jsc") if !File.file?(jscArg)
270     jscArg = Pathname.new(output).join("bin", "jsc") if !File.file?(jscArg) # Support CMake build.
271     if !File.file?(jscArg)
272         $stderr.puts "Error: must specify --jsc <path>"
273         exit 1
274     end
275
276     puts "Using the following jsc path: #{jscArg}"
277 end
278
279 if $enableFTL and !$jitTests
280     $stderr.puts "Error: can only specify one of --no-jit and --ftl-jit"
281     exit 1
282 end
283
284 if $doNotMessWithVMPath
285     $jscPath = Pathname.new(jscArg)
286 else
287     $jscPath = Pathname.new(jscArg).realpath
288 end
289
290 $progressMeter = ($verbosity == 0 and $stdout.tty?)
291
292 if $bundle
293     $jscPath = $bundle + ".vm" + "JavaScriptCore.framework" + "Resources" + "jsc"
294     $outputDir = $bundle
295 end
296
297 # Try to determine architecture. Return nil on failure.
298 def machOArchitectureCode
299     begin 
300         otoolLines = `otool -aSfh #{Shellwords.shellescape($jscPath.to_s)}`.split("\n")
301         otoolLines.each_with_index {
302             | value, index |
303             if value =~ /magic/ and value =~ /cputype/
304                 return otoolLines[index + 1].split[1].to_i
305             end
306         }
307     rescue
308         $stderr.puts "Warning: unable to execute otool."
309     end
310     $stderr.puts "Warning: unable to determine architecture."
311     nil
312 end
313
314 def determineArchitectureFromMachOBinary
315     code = machOArchitectureCode
316     return nil unless code
317     is64BitFlag = 0x01000000
318     case code
319     when 7
320         "x86"
321     when 7 | is64BitFlag
322         "x86-64"
323     when 12
324         "arm"
325     when 12 | is64BitFlag
326         "arm64"
327     else
328         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
329         nil
330     end
331 end
332
333 def determineArchitectureFromELFBinary
334     f = File.open($jscPath.to_s)
335     data = f.read(19)
336
337     if !(data[0,4] == "\x7F\x45\x4C\x46")
338         $stderr.puts "Warning: Missing ELF magic in file #{Shellwords.shellescape($jscPath.to_s)}"
339         return nil
340     end
341
342     code = data[18].ord
343     case code
344     when 3
345         "x86"
346     when 62
347         "x86-64"
348     when 40
349         "arm"
350     when 183
351         "arm64"
352     else
353         $stderr.puts "Warning: unable to determine architecture from code: #{code}"
354         nil
355     end
356 end
357
358 def determineArchitecture
359     case $hostOS
360     when "darwin"
361         determineArchitectureFromMachOBinary
362     when "linux"
363         determineArchitectureFromELFBinary
364     else
365         $stderr.puts "Warning: unable to determine architecture on this platform."
366         nil
367     end
368 end
369
370 def determineOS
371     case RbConfig::CONFIG["host_os"]
372     when /darwin/i
373         "darwin"
374     when /linux/i
375         "linux"
376     when /mswin|mingw|cygwin/
377         "windows"
378     else
379         $stderr.puts "Warning: unable to determine host operating system"
380         nil
381     end
382 end
383
384 $hostOS = determineOS unless $hostOS
385 $architecture = determineArchitecture unless $architecture
386
387 if !$testRunnerType
388     if $remote and $hostOS == "darwin"
389         $testRunnerType = :shell
390     else
391         $testRunnerType = :make
392     end
393 end
394
395 $numFailures = 0
396 $numPasses = 0
397
398 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true"]
399 EAGER_OPTIONS = ["--thresholdForJITAfterWarmUp=10", "--thresholdForJITSoon=10", "--thresholdForOptimizeAfterWarmUp=20", "--thresholdForOptimizeAfterLongWarmUp=20", "--thresholdForOptimizeSoon=20", "--thresholdForFTLOptimizeAfterWarmUp=20", "--thresholdForFTLOptimizeSoon=20", "--maximumEvalCacheableSourceLength=150000"]
400 NO_CJIT_OPTIONS = ["--useConcurrentJIT=false", "--thresholdForJITAfterWarmUp=100"]
401 FTL_OPTIONS = ["--useFTLJIT=true", "--ftlCrashesIfCantInitializeLLVM=true"]
402
403 $runlist = []
404
405 def frameworkFromJSCPath(jscPath)
406     parentDirectory = jscPath.dirname
407     if parentDirectory.basename.to_s == "Resources" and parentDirectory.dirname.basename.to_s == "JavaScriptCore.framework"
408         parentDirectory.dirname
409     elsif parentDirectory.basename.to_s =~ /^Debug/ or parentDirectory.basename.to_s =~ /^Release/
410         jscPath.dirname + "JavaScriptCore.framework"
411     else
412         $stderr.puts "Warning: cannot identify JSC framework, doing generic VM copy."
413         nil
414     end
415 end
416
417 def pathToBundleResourceFromBenchmarkDirectory(resourcePath)
418     dir = Pathname.new(".")
419     $benchmarkDirectory.each_filename {
420         | pathComponent |
421         dir += ".."
422     }
423     dir + resourcePath
424 end
425
426 def pathToVM
427     pathToBundleResourceFromBenchmarkDirectory($jscPath)
428 end
429
430 def pathToHelpers
431     pathToBundleResourceFromBenchmarkDirectory(".helpers")
432 end
433
434 def prefixCommand(prefix)
435     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
436 end
437
438 def redirectAndPrefixCommand(prefix)
439     prefixCommand(prefix) + " 2>&1"
440 end
441
442 def pipeAndPrefixCommand(outputFilename, prefix)
443     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
444 end
445
446 # Output handler for tests that are expected to be silent.
447 def silentOutputHandler
448     Proc.new {
449         | name |
450         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
451     }
452 end
453
454 # Output handler for tests that are expected to produce meaningful output.
455 def noisyOutputHandler
456     Proc.new {
457         | name |
458         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
459     }
460 end
461
462 # Error handler for tests that fail exactly when they return non-zero exit status.
463 # This is useful when a test is expected to fail.
464 def simpleErrorHandler
465     Proc.new {
466         | outp, plan |
467         outp.puts "if test -e #{plan.failFile}"
468         outp.puts "then"
469         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
470         outp.puts "    " + plan.failCommand
471         outp.puts "else"
472         outp.puts "    " + plan.successCommand
473         outp.puts "fi"
474     }
475 end
476
477 # Error handler for tests that fail exactly when they return zero exit status.
478 def expectedFailErrorHandler
479     Proc.new {
480         | outp, plan |
481         outp.puts "if test -e #{plan.failFile}"
482         outp.puts "then"
483         outp.puts "    " + plan.successCommand
484         outp.puts "else"
485         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
486         outp.puts "    " + plan.failCommand
487         outp.puts "fi"
488     }
489 end
490
491 # Error handler for tests that fail exactly when they return non-zero exit status and produce
492 # lots of spew. This will echo that spew when the test fails.
493 def noisyErrorHandler
494     Proc.new {
495         | outp, plan |
496         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
497     
498         outp.puts "if test -e #{plan.failFile}"
499         outp.puts "then"
500         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
501         outp.puts "    " + plan.failCommand
502         outp.puts "else"
503         outp.puts "    " + plan.successCommand
504         outp.puts "fi"
505     }
506 end
507
508 # Error handler for tests that diff their output with some expectation.
509 def diffErrorHandler(expectedFilename)
510     Proc.new {
511         | outp, plan |
512         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
513         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
514         
515         outp.puts "if test -e #{plan.failFile}"
516         outp.puts "then"
517         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
518         outp.puts "    " + plan.failCommand
519         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
520         outp.puts "then"
521         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
522         outp.puts "    if [ $? -eq 0 ]"
523         outp.puts "    then"
524         outp.puts "    " + plan.successCommand
525         outp.puts "    else"
526         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
527         outp.puts "        " + plan.failCommand
528         outp.puts "    fi"
529         outp.puts "else"
530         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
531         outp.puts "    " + plan.failCommand
532         outp.puts "fi"
533     }
534 end
535
536 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
537 # tests.
538 def mozillaErrorHandler
539     Proc.new {
540         | outp, plan |
541         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
542
543         outp.puts "if test -e #{plan.failFile}"
544         outp.puts "then"
545         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
546         outp.puts "    " + plan.failCommand
547         outp.puts "elif grep -i -q failed! #{outputFilename}"
548         outp.puts "then"
549         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
550         outp.puts "    " + plan.failCommand
551         outp.puts "else"
552         outp.puts "    " + plan.successCommand
553         outp.puts "fi"
554     }
555 end
556
557 # Error handler for tests that report error by saying "failed!", and are expected to
558 # fail. This is used by Mozilla tests.
559 def mozillaFailErrorHandler
560     Proc.new {
561         | outp, plan |
562         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
563
564         outp.puts "if test -e #{plan.failFile}"
565         outp.puts "then"
566         outp.puts "    " + plan.successCommand
567         outp.puts "elif grep -i -q failed! #{outputFilename}"
568         outp.puts "then"
569         outp.puts "    " + plan.successCommand
570         outp.puts "else"
571         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
572         outp.puts "    " + plan.failCommand
573         outp.puts "fi"
574     }
575 end
576
577 # Error handler for tests that report error by saying "failed!", and are expected to have
578 # an exit code of 3.
579 def mozillaExit3ErrorHandler
580     Proc.new {
581         | outp, plan |
582         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
583
584         outp.puts "if test -e #{plan.failFile}"
585         outp.puts "then"
586         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
587         outp.puts "    then"
588         outp.puts "        if grep -i -q failed! #{outputFilename}"
589         outp.puts "        then"
590         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
591         outp.puts "            " + plan.failCommand
592         outp.puts "        else"
593         outp.puts "            " + plan.successCommand
594         outp.puts "        fi"
595         outp.puts "    else"
596         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
597         outp.puts "        " + plan.failCommand
598         outp.puts "    fi"
599         outp.puts "else"
600         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
601         outp.puts "    " + plan.failCommand
602         outp.puts "fi"
603     }
604 end
605
606 $runCommandOptions = {}
607
608 class Plan
609     attr_reader :directory, :arguments, :family, :name, :outputHandler, :errorHandler
610     attr_accessor :index
611     
612     def initialize(directory, arguments, family, name, outputHandler, errorHandler)
613         @directory = directory
614         @arguments = arguments
615         @family = family
616         @name = name
617         @outputHandler = outputHandler
618         @errorHandler = errorHandler
619         @isSlow = !!$runCommandOptions[:isSlow]
620     end
621     
622     def shellCommand
623         # It's important to remember that the test is actually run in a subshell, so if we change directory
624         # in the subshell when we return we will be in our original directory. This is nice because we don't
625         # have to bend over backwards to do things relative to the root.
626         script = "(cd ../#{Shellwords.shellescape(@directory.to_s)} && ("
627         $envVars.each { |var| script += "export " << var << "; " }
628         script += "\"$@\" " + escapeAll(@arguments) + "))"
629         return script
630     end
631     
632     def reproScriptCommand
633         # We have to find our way back to the .runner directory since that's where all of the relative
634         # paths assume they start out from.
635         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
636         script += "cd $CURRENT_DIR\n"
637         Pathname.new(@name).dirname.each_filename {
638             | pathComponent |
639             script += "cd ..\n"
640         }
641         script += "cd .runner\n"
642
643         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
644         script += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])}\n"
645         $envVars.each { |var| script += "export " << var << "\n" }
646         script += "#{shellCommand} || exit 1"
647         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
648     end
649     
650     def failCommand
651         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
652     end
653     
654     def successCommand
655         if $progressMeter or $verbosity >= 2
656             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
657         else
658             "rm -f #{failFile}"
659         end
660     end
661     
662     def failFile
663         "test_fail_#{@index}"
664     end
665     
666     def writeRunScript(filename)
667         File.open(filename, "w") {
668             | outp |
669             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
670             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
671             cmd += @outputHandler.call(@name)
672             if $verbosity >= 3
673                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
674             end
675             outp.puts cmd
676             @errorHandler.call(outp, self)
677         }
678     end
679 end
680
681 $uniqueFilenameCounter = 0
682 def uniqueFilename(extension)
683     payloadDir = $outputDir + "_payload"
684     Dir.mkdir payloadDir unless payloadDir.directory?
685     result = payloadDir.realpath + "temp-#{$uniqueFilenameCounter}#{extension}"
686     $uniqueFilenameCounter += 1
687     result
688 end
689
690 def baseOutputName(kind)
691     "#{$collectionName}/#{$benchmark}.#{kind}"
692 end
693
694 def addRunCommand(kind, command, outputHandler, errorHandler)
695     $didAddRunCommand = true
696     name = baseOutputName(kind)
697     if $filter and name !~ $filter
698         return
699     end
700     plan = Plan.new(
701         $benchmarkDirectory, command, "#{$collectionName}/#{$benchmark}", name, outputHandler,
702         errorHandler)
703     if $numChildProcesses > 1 and $runCommandOptions[:isSlow]
704         $runlist.unshift plan
705     else
706         $runlist << plan
707     end
708 end
709
710 # Returns true if there were run commands found in the file ($benchmarkDirectory +
711 # $benchmark), in which case those run commands have already been executed. Otherwise
712 # returns false, in which case you're supposed to add your own run commands.
713 def parseRunCommands
714     oldDidAddRunCommand = $didAddRunCommand
715     $didAddRunCommand = false
716
717     Dir.chdir($outputDir) {
718         File.open($benchmarkDirectory + $benchmark) {
719             | inp |
720             inp.each_line {
721                 | line |
722                 begin
723                     doesMatch = line =~ /^\/\/@/
724                 rescue Exception => e
725                     # Apparently this happens in the case of some UTF8 stuff in some files, where
726                     # Ruby tries to be strict and throw exceptions.
727                     next
728                 end
729                 next unless doesMatch
730                 eval $~.post_match
731             }
732         }
733     }
734
735     result = $didAddRunCommand
736     $didAddRunCommand = result or oldDidAddRunCommand
737     result
738 end
739
740 def slow!
741     $runCommandOptions[:isSlow] = true
742 end
743
744 def run(kind, *options)
745     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], silentOutputHandler, simpleErrorHandler)
746 end
747
748 def runDefault
749     run("default")
750 end
751
752 def runWithRAMSize(size)
753     run("ram-size-#{size}", "--forceRAMSize=#{size}")
754 end
755
756 def runNoJIT
757     run("no-jit", "--useJIT=false")
758 end
759
760 def runNoLLInt
761     if $jitTests
762         run("no-llint", "--useLLInt=false")
763     end
764 end
765
766 def runNoCJITValidate
767     run("no-cjit", "--validateBytecode=true", "--validateGraph=true", *NO_CJIT_OPTIONS)
768 end
769
770 def runNoCJITValidatePhases
771     run("no-cjit-validate-phases", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
772 end
773
774 def runDefaultFTL
775     run("default-ftl", *FTL_OPTIONS) if $enableFTL
776 end
777
778 def runFTLNoCJIT
779     run("ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
780 end
781
782 def runFTLNoCJITValidate
783     run("ftl-no-cjit-validate-sampling-profiler", "--validateGraph=true", "--useSamplingProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
784 end
785
786 def runFTLNoCJITNoPutStackValidate
787     run("ftl-no-cjit-no-put-stack-validate", "--validateGraph=true", "--usePutStackSinking=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
788 end
789
790 def runFTLNoCJITNoInlineValidate
791     run("ftl-no-cjit-no-inline-validate", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
792 end
793
794 def runFTLNoCJITOSRValidation
795     run("ftl-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
796 end
797
798 def runDFGEager
799     run("dfg-eager", *EAGER_OPTIONS)
800 end
801
802 def runDFGEagerNoCJITValidate
803     run("dfg-eager-no-cjit-validate", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
804 end
805
806 def runFTLEager
807     run("ftl-eager", *(FTL_OPTIONS + EAGER_OPTIONS)) if $enableFTL
808 end
809
810 def runFTLEagerNoCJITValidate
811     run("ftl-eager-no-cjit", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
812 end
813
814 def runFTLEagerNoCJITOSRValidation
815     run("ftl-eager-no-cjit-osr-validation", "--validateFTLOSRExitLiveness=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
816 end
817
818 def runAlwaysTriggerCopyPhase
819     run("always-trigger-copy-phase", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
820 end
821
822 def runNoCJITNoASO
823     run("no-cjit-no-aso", "--useArchitectureSpecificOptimizations=false", *NO_CJIT_OPTIONS)
824 end
825
826 def runFTLNoCJITNoSimpleOpt
827     run("ftl-no-cjit-no-simple-opt", "--llvmSimpleOpt=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
828 end
829
830 def runNoCJITNoAccessInlining
831     run("no-cjit-no-access-inlining", "--useAccessInlining=false", *NO_CJIT_OPTIONS)
832 end
833
834 def runFTLNoCJITNoAccessInlining
835     run("ftl-no-cjit-no-access-inlining", "--useAccessInlining=false", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
836 end
837
838 def runFTLNoCJITSmallPool
839     run("ftl-no-cjit-small-pool", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
840 end
841
842 def runMiscNoCJITTest(*options)
843     run("misc-no-cjit", *(NO_CJIT_OPTIONS + options))
844 end
845
846 def runMiscFTLNoCJITTest(*options)
847     run("misc-ftl-no-cjit", *(FTL_OPTIONS + NO_CJIT_OPTIONS + options))
848 end
849
850 def runDFGMaximalFlushPhase
851     run("dfg-maximal-flush-validate-no-cjit", "--validateGraph=true", "--useMaximalFlushInsertionPhase=true", *NO_CJIT_OPTIONS)
852 end
853
854 def defaultRun
855     if $quickMode
856         defaultQuickRun
857     else
858         runDefault
859         runAlwaysTriggerCopyPhase
860         if $jitTests
861             runNoLLInt
862             runNoCJITValidatePhases
863             runDFGEager
864             runDFGEagerNoCJITValidate
865             runDefaultFTL
866             runFTLNoCJITValidate
867             runFTLNoCJITNoPutStackValidate
868             runFTLNoCJITNoInlineValidate
869             runFTLEager
870             runFTLEagerNoCJITValidate
871             runFTLNoCJITSmallPool
872             runDFGMaximalFlushPhase
873         end
874     end
875 end
876
877 def defaultQuickRun
878     if $enableFTL and $jitTests
879         runDefaultFTL
880         runFTLNoCJITValidate
881     else
882         runDefault
883         if $jitTests
884             runNoCJITValidate
885         end
886     end
887 end
888
889 def defaultSpotCheckNoMaximalFlush
890     defaultQuickRun
891     runFTLNoCJITNoSimpleOpt
892     runFTLNoCJITOSRValidation
893     runNoCJITNoAccessInlining
894     runFTLNoCJITNoAccessInlining
895 end
896
897 def defaultSpotCheck
898     defaultSpotCheckNoMaximalFlush
899     runDFGMaximalFlushPhase
900 end
901
902 # This is expected to not do eager runs because eager runs can have a lot of recompilations
903 # for reasons that don't arise in the real world. It's used for tests that assert convergence
904 # by counting recompilations.
905 def defaultNoEagerRun
906     runDefault
907     runAlwaysTriggerCopyPhase
908     if $jitTests
909         runNoLLInt
910         runNoCJITValidatePhases
911         runDefaultFTL
912         runFTLNoCJITValidate
913         runFTLNoCJITNoInlineValidate
914     end
915 end
916
917 def defaultNoSamplingProfilerRun
918     runDefault
919     runAlwaysTriggerCopyPhase
920     if $jitTests
921         runNoLLInt
922         runNoCJITValidatePhases
923         runDFGEager
924         runDFGEagerNoCJITValidate
925         runDefaultFTL
926         runFTLNoCJITNoPutStackValidate
927         runFTLNoCJITNoInlineValidate
928         runFTLEager
929         runFTLEagerNoCJITValidate
930         runFTLNoCJITSmallPool
931         runDFGMaximalFlushPhase
932     end
933 end
934
935 def runProfiler
936     if $remote or ($architecture !~ /x86/i and $hostOS == "darwin") or ($hostOS == "windows")
937         skip
938         return
939     end
940
941     profilerOutput = uniqueFilename(".json")
942     if $canRunDisplayProfilerOutput
943         addRunCommand("profiler", ["ruby", (pathToHelpers + "profiler-test-helper").to_s, (SCRIPTS_PATH + "display-profiler-output").to_s, profilerOutput.to_s, pathToVM.to_s, "-p", profilerOutput.to_s, $benchmark.to_s], silentOutputHandler, simpleErrorHandler)
944     else
945         puts "Running simple version of #{$collectionName}/#{$benchmark} because some required Ruby features are unavailable."
946         run("profiler-simple", "-p", profilerOutput.to_s)
947     end
948 end
949
950 def runExceptionFuzz
951     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s])
952     addRunCommand("exception-fuzz", ["perl", (pathToHelpers + "js-exception-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
953 end
954
955 def runExecutableAllocationFuzz(name, *options)
956     subCommand = escapeAll([pathToVM.to_s, $benchmark.to_s] + options)
957     addRunCommand("executable-allocation-fuzz-" + name, ["perl", (pathToHelpers + "js-executable-allocation-fuzz").to_s, subCommand], silentOutputHandler, simpleErrorHandler)
958 end
959
960 def runTypeProfiler
961     if !$jitTests
962         return
963     end
964
965     if $enableFTL
966         run("ftl-no-cjit-type-profiler", "--useTypeProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
967     else
968         run("no-cjit-type-profiler", "--useTypeProfiler=true", *NO_CJIT_OPTIONS)
969     end
970 end
971
972 def runControlFlowProfiler
973     if !$jitTests
974         return
975     end
976
977     if $enableFTL
978         run("ftl-no-cjit-type-profiler", "--useControlFlowProfiler=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
979     else
980         run("no-cjit-type-profiler", "--useControlFlowProfiler=true", *NO_CJIT_OPTIONS)
981     end
982 end
983
984 def runES6(mode)
985     args = [pathToVM.to_s] + BASE_OPTIONS + [$benchmark.to_s]
986     case mode
987     when :normal
988         errorHandler = simpleErrorHandler
989     when :fail
990         errorHandler = expectedFailErrorHandler
991     else
992         raise "Invalid mode: #{mode}"
993     end
994     addRunCommand("default", args, noisyOutputHandler, errorHandler)
995 end
996
997 def runModules
998     run("default-modules", "-m")
999     run("always-trigger-copy-phase-modules", "-m", "--minHeapUtilization=2.0", "--minCopiedBlockUtilization=2.0")
1000
1001     if !$jitTests
1002         return
1003     end
1004
1005     run("no-llint-modules", "-m", "--useLLInt=false")
1006     run("no-cjit-validate-phases-modules", "-m", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *NO_CJIT_OPTIONS)
1007     run("dfg-eager-modules", "-m", *EAGER_OPTIONS)
1008     run("dfg-eager-no-cjit-validate-modules", "-m", "--validateGraph=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1009     if $enableFTL
1010         run("default-ftl-modules", "-m", *FTL_OPTIONS)
1011         run("ftl-no-cjit-validate-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1012         run("ftl-no-cjit-no-inline-validate-modules", "-m", "--validateGraph=true", "--maximumInliningDepth=1", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1013         run("ftl-eager-modules", "-m", *(FTL_OPTIONS + EAGER_OPTIONS))
1014         run("ftl-eager-no-cjit-modules", "-m", "--validateGraph=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1015         run("ftl-no-cjit-small-pool-modules", "-m", "--jitMemoryReservationSize=50000", *(FTL_OPTIONS + NO_CJIT_OPTIONS))
1016     end
1017 end
1018
1019 def runLayoutTest(kind, *options)
1020     raise unless $benchmark.to_s =~ /\.js$/
1021     testName = $~.pre_match
1022     if kind
1023         kind = "layout-" + kind
1024     else
1025         kind = "layout"
1026     end
1027
1028     prepareExtraRelativeFiles(["../#{testName}-expected.txt"], $benchmarkDirectory)
1029     prepareExtraAbsoluteFiles(LAYOUTTESTS_PATH, ["resources/standalone-pre.js", "resources/standalone-post.js"])
1030
1031     args = [pathToVM.to_s] + BASE_OPTIONS + options +
1032         [(Pathname.new("resources") + "standalone-pre.js").to_s,
1033          $benchmark.to_s,
1034          (Pathname.new("resources") + "standalone-post.js").to_s]
1035     addRunCommand(kind, args, noisyOutputHandler, diffErrorHandler(($benchmarkDirectory + "../#{testName}-expected.txt").to_s))
1036 end
1037
1038 def runLayoutTestDefault
1039     runLayoutTest(nil)
1040 end
1041
1042 def runLayoutTestNoLLInt
1043     runLayoutTest("no-llint", "--useLLInt=false")
1044 end
1045
1046 def runLayoutTestNoCJIT
1047     runLayoutTest("no-cjit", *NO_CJIT_OPTIONS)
1048 end
1049
1050 def runLayoutTestDFGEagerNoCJIT
1051     runLayoutTest("dfg-eager-no-cjit", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1052 end
1053
1054 def runLayoutTestDefaultFTL
1055     runLayoutTest("ftl", "--testTheFTL=true", *FTL_OPTIONS) if $enableFTL
1056 end
1057
1058 def runLayoutTestFTLNoCJIT
1059     runLayoutTest("ftl-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS)) if $enableFTL
1060 end
1061
1062 def runLayoutTestFTLEagerNoCJIT
1063     runLayoutTest("ftl-eager-no-cjit", "--testTheFTL=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
1064 end
1065
1066 def noFTLRunLayoutTest
1067     if !$jitTests
1068         return
1069     end
1070
1071     runLayoutTestNoLLInt
1072     runLayoutTestNoCJIT
1073     runLayoutTestDFGEagerNoCJIT
1074 end
1075
1076 def defaultQuickRunLayoutTest
1077     runLayoutTestDefault
1078     if $jitTests
1079         runLayoutTestFTLNoCJIT
1080         runLayoutTestFTLEagerNoCJIT
1081     end
1082 end
1083
1084 def defaultRunLayoutTest
1085     if $quickMode
1086         defaultQuickRunLayoutTest
1087     else
1088         runLayoutTestDefault
1089         if $jitTests
1090             noFTLRunLayoutTest
1091             runLayoutTestDefaultFTL
1092             runLayoutTestFTLNoCJIT
1093             runLayoutTestFTLEagerNoCJIT
1094         end
1095     end
1096 end
1097
1098 def noNoLLIntRunLayoutTest
1099     runLayoutTestDefault
1100     if $jitTests
1101         runLayoutTestNoCJIT
1102         runLayoutTestDFGEagerNoCJIT
1103         runLayoutTestDefaultFTL
1104         runLayoutTestFTLNoCJIT
1105         runLayoutTestFTLEagerNoCJIT
1106     end
1107 end
1108
1109 def prepareExtraRelativeFiles(extraFiles, destination)
1110     Dir.chdir($outputDir) {
1111         extraFiles.each {
1112             | file |
1113             FileUtils.cp $extraFilesBaseDir + file, destination + file
1114         }
1115     }
1116 end
1117
1118 def baseDirForCollection(collectionName)
1119     Pathname(".tests") + collectionName
1120 end
1121
1122 def prepareExtraAbsoluteFiles(absoluteBase, extraFiles)
1123     raise unless absoluteBase.absolute?
1124     Dir.chdir($outputDir) {
1125         collectionBaseDir = baseDirForCollection($collectionName)
1126         extraFiles.each {
1127             | file |
1128             destination = collectionBaseDir + file
1129             FileUtils.mkdir_p destination.dirname unless destination.directory?
1130             FileUtils.cp absoluteBase + file, destination
1131         }
1132     }
1133 end
1134
1135 def runMozillaTest(kind, mode, extraFiles, *options)
1136     if kind
1137         kind = "mozilla-" + kind
1138     else
1139         kind = "mozilla"
1140     end
1141     prepareExtraRelativeFiles(extraFiles.map{|v| (Pathname("..") + v).to_s}, $collection)
1142     args = [pathToVM.to_s] + BASE_OPTIONS + options + extraFiles.map{|v| v.to_s} + [$benchmark.to_s]
1143     case mode
1144     when :normal
1145         errorHandler = mozillaErrorHandler
1146     when :negative
1147         errorHandler = mozillaExit3ErrorHandler
1148     when :fail
1149         errorHandler = mozillaFailErrorHandler
1150     when :skip
1151         return
1152     else
1153         raise "Invalid mode: #{mode}"
1154     end
1155     addRunCommand(kind, args, noisyOutputHandler, errorHandler)
1156 end
1157
1158 def runMozillaTestDefault(mode, *extraFiles)
1159     runMozillaTest(nil, mode, extraFiles)
1160 end
1161
1162 def runMozillaTestDefaultFTL(mode, *extraFiles)
1163     runMozillaTest("ftl", mode, extraFiles, *FTL_OPTIONS) if $enableFTL
1164 end
1165
1166 def runMozillaTestLLInt(mode, *extraFiles)
1167     runMozillaTest("llint", mode, extraFiles, "--useJIT=false")
1168 end
1169
1170 def runMozillaTestBaselineJIT(mode, *extraFiles)
1171     runMozillaTest("baseline", mode, extraFiles, "--useLLInt=false", "--useDFGJIT=false")
1172 end
1173
1174 def runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1175     runMozillaTest("dfg-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(NO_CJIT_OPTIONS + EAGER_OPTIONS))
1176 end
1177
1178 def runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1179     runMozillaTest("ftl-eager-no-cjit-validate-phases", mode, extraFiles, "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(FTL_OPTIONS + NO_CJIT_OPTIONS + EAGER_OPTIONS)) if $enableFTL
1180 end
1181
1182 def defaultQuickRunMozillaTest(mode, *extraFiles)
1183     if $enableFTL and $jitTests
1184         runMozillaTestDefaultFTL(mode, *extraFiles)
1185         runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1186     else
1187         runMozillaTestDefault(mode, *extraFiles)
1188         if $jitTests
1189             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1190         end
1191     end
1192 end
1193
1194 def defaultRunMozillaTest(mode, *extraFiles)
1195     if $quickMode
1196         defaultQuickRunMozillaTest(mode, *extraFiles)
1197     else
1198         runMozillaTestDefault(mode, *extraFiles)
1199         if $jitTests
1200             runMozillaTestLLInt(mode, *extraFiles)
1201             runMozillaTestBaselineJIT(mode, *extraFiles)
1202             runMozillaTestDFGEagerNoCJITValidatePhases(mode, *extraFiles)
1203             runMozillaTestDefaultFTL(mode, *extraFiles)
1204             runMozillaTestFTLEagerNoCJITValidatePhases(mode, *extraFiles)
1205         end
1206     end
1207 end
1208
1209 def runNoisyTest(kind, *options)
1210     addRunCommand(kind, [pathToVM.to_s] + BASE_OPTIONS + options + [$benchmark.to_s], noisyOutputHandler, noisyErrorHandler)
1211 end
1212
1213 def runNoisyTestDefault
1214     runNoisyTest("default")
1215 end
1216
1217 def runNoisyTestDefaultFTL
1218     runNoisyTest("ftl", *FTL_OPTIONS) if $enableFTL
1219 end
1220
1221 def runNoisyTestNoCJIT
1222     runNoisyTest($enableFTL ? "ftl-no-cjit" : "no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(($enableFTL ? FTL_OPTIONS : []) + NO_CJIT_OPTIONS))
1223 end
1224
1225 def runNoisyTestEagerNoCJIT
1226     runNoisyTest($enableFTL ? "ftl-eager-no-cjit" : "eager-no-cjit", "--validateBytecode=true", "--validateGraphAtEachPhase=true", *(($enableFTL ? FTL_OPTIONS : []) + NO_CJIT_OPTIONS + EAGER_OPTIONS))
1227 end
1228
1229 def defaultRunNoisyTest
1230     runNoisyTestDefault
1231     if $jitTests
1232         runNoisyTestDefaultFTL
1233         runNoisyTestNoCJIT
1234         runNoisyTestEagerNoCJIT
1235     end
1236 end
1237
1238 def skip
1239     $didAddRunCommand = true
1240     puts "Skipping #{$collectionName}/#{$benchmark}"
1241 end
1242
1243 def largeHeap
1244     if $memoryLimited
1245         $didAddRunCommand = true
1246         puts "Skipping #{$collectionName}/#{$benchmark}"
1247     end
1248 end
1249
1250 def allJSFiles(path)
1251     if path.file?
1252         [path]
1253     else
1254         result = []
1255         Dir.foreach(path) {
1256             | filename |
1257             next unless filename =~ /\.js$/
1258             next unless (path + filename).file?
1259             result << path + filename
1260         }
1261         result
1262     end
1263 end
1264
1265 def uniqueifyName(names, name)
1266     result = name.to_s
1267     toAdd = 1
1268     while names[result]
1269         result = "#{name}-#{toAdd}"
1270         toAdd += 1
1271     end
1272     names[result] = true
1273     result
1274 end
1275
1276 def simplifyCollectionName(collectionPath)
1277     outerDir = collectionPath.dirname
1278     name = collectionPath.basename
1279     lastName = name
1280     if collectionPath.directory?
1281         while lastName.to_s =~ /test/
1282             lastName = outerDir.basename
1283             name = lastName + name
1284             outerDir = outerDir.dirname
1285         end
1286     end
1287     uniqueifyName($collectionNames, name)
1288 end
1289
1290 def prepareCollection(name)
1291     FileUtils.mkdir_p $outputDir + name
1292
1293     absoluteCollection = $collection.realpath
1294
1295     Dir.chdir($outputDir) {
1296         bundleDir = baseDirForCollection(name)
1297
1298         # Create the proper directory structures.
1299         FileUtils.mkdir_p bundleDir
1300         if bundleDir.basename == $collection.basename
1301             FileUtils.cp_r absoluteCollection, bundleDir.dirname
1302             $collection = bundleDir
1303         else
1304             FileUtils.cp_r absoluteCollection, bundleDir
1305             $collection = bundleDir + $collection.basename
1306         end
1307
1308         $extraFilesBaseDir = absoluteCollection
1309     }
1310 end
1311
1312 $collectionNames = {}
1313
1314 def handleCollectionFile(collection)
1315     collectionName = simplifyCollectionName(collection)
1316    
1317     paths = {}
1318     subCollections = []
1319     YAML::load(IO::read(collection)).each {
1320         | entry |
1321         if entry["collection"]
1322             subCollections << entry["collection"]
1323             next
1324         end
1325         
1326         if Pathname.new(entry["path"]).absolute?
1327             raise "Absolute path: " + entry["path"] + " in #{collection}"
1328         end
1329         
1330         if paths[entry["path"]]
1331             raise "Duplicate path: " + entry["path"] + " in #{collection}"
1332         end
1333         
1334         subCollection = collection.dirname + entry["path"]
1335         
1336         if subCollection.file?
1337             subCollectionName = Pathname.new(entry["path"]).dirname
1338         else
1339             subCollectionName = entry["path"]
1340         end
1341         
1342         $collection = subCollection
1343         $collectionName = Pathname.new(collectionName)
1344         Pathname.new(subCollectionName).each_filename {
1345             | filename |
1346             next if filename =~ /^\./
1347             $collectionName += filename
1348         }
1349         $collectionName = $collectionName.to_s
1350         
1351         prepareCollection($collectionName)
1352       
1353         Dir.chdir($outputDir) {
1354             pathsToSearch = [$collection]
1355             if entry["tests"]
1356                 if entry["tests"].is_a? Array
1357                     pathsToSearch = entry["tests"].map {
1358                         | testName |
1359                         pathsToSearch[0] + testName
1360                     }
1361                 else
1362                     pathsToSearch[0] += entry["tests"]
1363                 end
1364             end
1365             pathsToSearch.each {
1366                 | pathToSearch |
1367                 allJSFiles(pathToSearch).each {
1368                     | path |
1369                     
1370                     $benchmark = path.basename
1371                     $benchmarkDirectory = path.dirname
1372                     
1373                     $runCommandOptions = {}
1374                     eval entry["cmd"]
1375                 }
1376             }
1377         }
1378     }
1379     
1380     subCollections.each {
1381         | subCollection |
1382         handleCollection(collection.dirname + subCollection)
1383     }
1384 end
1385
1386 def handleCollectionDirectory(collection)
1387     collectionName = simplifyCollectionName(collection)
1388     
1389     $collection = collection
1390     $collectionName = collectionName
1391     prepareCollection(collectionName)
1392    
1393     Dir.chdir($outputDir) {
1394         $benchmarkDirectory = $collection
1395         allJSFiles($collection).each {
1396             | path |
1397             
1398             $benchmark = path.basename
1399             
1400             $runCommandOptions = {}
1401             defaultRun unless parseRunCommands
1402         }
1403     }
1404 end
1405
1406 def handleCollection(collection)
1407     collection = Pathname.new(collection)
1408     
1409     if collection.file?
1410         handleCollectionFile(collection)
1411     else
1412         handleCollectionDirectory(collection)
1413     end
1414 end
1415
1416 def appendFailure(plan)
1417     File.open($outputDir + "failed", "a") {
1418         | outp |
1419         outp.puts plan.name
1420     }
1421     $numFailures += 1
1422 end
1423
1424 def appendPass(plan)
1425     File.open($outputDir + "passed", "a") {
1426         | outp |
1427         outp.puts plan.name
1428     }
1429     $numPasses += 1
1430 end
1431
1432 def appendResult(plan, didPass)
1433     File.open($outputDir + "results", "a") {
1434         | outp |
1435         outp.puts "#{plan.name}: #{didPass ? 'PASS' : 'FAIL'}"
1436     }
1437 end
1438
1439 def prepareBundle
1440     raise if $bundle
1441
1442     if $doNotMessWithVMPath
1443         if !$remote and !$tarball
1444             $testingFrameworkPath = frameworkFromJSCPath($jscPath).realpath
1445             $jscPath = Pathname.new($jscPath).realpath
1446         else
1447             $testingFrameworkPath = frameworkFromJSCPath($jscPath)
1448         end
1449     else
1450         originalJSCPath = $jscPath
1451         vmDir = $outputDir + ".vm"
1452         FileUtils.mkdir_p vmDir
1453         
1454         frameworkPath = frameworkFromJSCPath($jscPath)
1455         destinationFrameworkPath = Pathname.new(".vm") + "JavaScriptCore.framework"
1456         $jscPath = destinationFrameworkPath + "Resources" + "jsc"
1457         $testingFrameworkPath = Pathname.new("..") + destinationFrameworkPath
1458
1459         if frameworkPath
1460             source = frameworkPath
1461             destination = Pathname.new(".vm")
1462         else
1463             source = originalJSCPath
1464             destination = $jscPath
1465
1466             Dir.chdir($outputDir) {
1467                 FileUtils.mkdir_p $jscPath.dirname
1468             }
1469         end
1470
1471         Dir.chdir($outputDir) {
1472             if $copyVM
1473                 FileUtils.cp_r source, destination
1474             else
1475                 begin 
1476                     FileUtils.ln_s source, destination
1477                 rescue Exception
1478                     $stderr.puts "Warning: unable to create soft link, trying to copy."
1479                     FileUtils.cp_r source, destination
1480                 end
1481             end
1482
1483             if $remote and $hostOS == "linux"
1484                 if $enableFTL
1485                     begin
1486                         FileUtils.cp_r originalJSCPath + "../../lib/libllvmForJSC.so" , $jscPath.dirname
1487                     rescue
1488                         $stderr.puts "Warning: unable to copy libllvmForJSC.so to the bundle."
1489                     end
1490                 end
1491                 begin
1492                     dependencies = `ldd #{source}`
1493                     dependencies.split(/\n/).each {
1494                         | dependency |
1495                         FileUtils.cp_r $&, $jscPath.dirname if dependency =~ /#{WEBKIT_PATH}[^ ]*/
1496                     }
1497                 rescue
1498                     $stderr.puts "Warning: unable to determine or copy library dependnecies of JSC."
1499                 end
1500             end
1501         }
1502     end
1503     
1504     Dir.chdir($outputDir) {
1505         FileUtils.cp_r HELPERS_PATH, ".helpers"
1506     }
1507
1508     ARGV.each {
1509         | collection |
1510         handleCollection(collection)
1511     }
1512
1513     puts
1514 end
1515
1516 def cleanOldResults
1517     raise unless $bundle
1518
1519     eachResultFile($outputDir) {
1520         | path |
1521         FileUtils.rm_f path
1522     }
1523 end
1524
1525 def cleanEmptyResultFiles
1526     eachResultFile($outputDir) {
1527         | path |
1528         next unless path.basename.to_s =~ /\.out$/
1529         next unless FileTest.size(path) == 0
1530         FileUtils.rm_f path
1531     }
1532 end
1533
1534 def eachResultFile(startingDir, &block)
1535     dirsToClean = [startingDir]
1536     until dirsToClean.empty?
1537         nextDir = dirsToClean.pop
1538         Dir.foreach(nextDir) {
1539             | entry |
1540             next if entry =~ /^\./
1541             path = nextDir + entry
1542             if path.directory?
1543                 dirsToClean.push(path)
1544             else
1545                 block.call(path)
1546             end
1547         }
1548     end
1549 end
1550
1551 def prepareTestRunner
1552     raise if $bundle
1553
1554     $runlist.each_with_index {
1555         | plan, index |
1556         plan.index = index
1557     }
1558
1559     Dir.mkdir($runnerDir) unless $runnerDir.directory?
1560     toDelete = []
1561     Dir.foreach($runnerDir) {
1562         | filename |
1563         if filename =~ /^test_/
1564             toDelete << filename
1565         end
1566     }
1567     
1568     toDelete.each {
1569         | filename |
1570         File.unlink($runnerDir + filename)
1571     }
1572
1573     $runlist.each {
1574         | plan |
1575         plan.writeRunScript($runnerDir + "test_script_#{plan.index}")
1576     }
1577
1578     case $testRunnerType
1579     when :make
1580         prepareMakeTestRunner
1581     when :shell
1582         prepareShellTestRunner
1583     else
1584         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1585     end
1586 end
1587
1588 def prepareShellTestRunner
1589     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
1590 end
1591
1592 def prepareMakeTestRunner
1593     # The goals of our parallel test runner are scalability and simplicity. The
1594     # simplicity part is particularly important. We don't want to have to have
1595     # a full-time contributor just philosophising about parallel testing.
1596     #
1597     # As such, we just pass off all of the hard work to 'make'. This creates a
1598     # dummy directory ("$outputDir/.runner") in which we create a dummy
1599     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
1600     # That is, for each test we know we will run, there is a rule in the
1601     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
1602     # Makefile results in 'make' doing all of the hard work:
1603     #
1604     # - Load balancing just works. Most systems have a great load balancer in
1605     #   'make'. If your system doesn't then just install a real 'make'.
1606     #
1607     # - Interruptions just work. For example Ctrl-C handling in 'make' is
1608     #   exactly right. You don't have to worry about zombie processes.
1609     #
1610     # We then do some tricks to make failure detection work and to make this
1611     # totally sound. If a test fails, we don't want the whole 'make' job to
1612     # stop. We also don't have any facility for makefile-escaping of path names.
1613     # We do have such a thing for shell-escaping, though. We fix both problems
1614     # by having the actual work for each of the test rules be done in a shell
1615     # script on the side. There is one such script per test. The script responds
1616     # to failure by printing something on the console and then touching a
1617     # failure file for that test, but then still returns 0. This makes 'make'
1618     # continue past that failure and complete all the tests anyway.
1619     #
1620     # In the end, this script collects all of the failures by searching for
1621     # files in the .runner directory whose name matches /^test_fail_/, where
1622     # the thing after the 'fail_' is the test index. Those are the files that
1623     # would be created by the test scripts if they detect failure. We're
1624     # basically using the filesystem as a concurrent database of test failures.
1625     # Even if two tests fail at the same time, since they're touching different
1626     # files we won't miss any failures.
1627     runIndices = []
1628     $runlist.each {
1629         | plan |
1630         runIndices << plan.index
1631     }
1632     
1633     File.open($runnerDir + "Makefile", "w") {
1634         | outp |
1635         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
1636         runIndices.each {
1637             | index |
1638             plan = $runlist[index]
1639             outp.puts "test_done_#{index}:"
1640             outp.puts "\tsh test_script_#{plan.index}"
1641         }
1642     }
1643 end
1644
1645 def cleanRunnerDirectory
1646     raise unless $bundle
1647     Dir.foreach($runnerDir) {
1648         | filename |
1649         next unless filename =~ /^test_fail/
1650         FileUtils.rm_f $runnerDir + filename
1651     }
1652 end
1653
1654 def sshRead(cmd)
1655     raise unless $remote
1656
1657     result = ""
1658     IO.popen("ssh -p #{$remotePort} #{$remoteUser}@#{$remoteHost} '#{cmd}'", "r") {
1659       | inp |
1660       inp.each_line {
1661         | line |
1662         result += line
1663       }
1664     }
1665     raise "#{$?}" unless $?.success?
1666     result
1667 end
1668
1669 def runCommandOnTester(cmd)
1670     if $remote
1671         result = sshRead(cmd)
1672     else
1673         result = `#{cmd}`
1674     end
1675 end
1676
1677 def numberOfProcessors
1678     begin
1679         numProcessors = runCommandOnTester("sysctl -n hw.activecpu 2>/dev/null").to_i
1680     rescue
1681         numProcessors = 0
1682     end
1683
1684     if numProcessors == 0
1685         begin
1686             numProcessors = runCommandOnTester("nproc --all 2>/dev/null").to_i
1687         rescue
1688             numProcessors == 0
1689         end
1690     end
1691
1692     if numProcessors == 0
1693         numProcessors = 1
1694     end
1695     return numProcessors
1696 end
1697
1698 def runAndMonitorTestRunnerCommand(*cmd)
1699     numberOfTests = 0
1700     Dir.chdir($runnerDir) {
1701         # -1 for the runscript, and -2 for '..' and '.'
1702         numberOfTests = Dir.entries(".").count - 3
1703     }
1704     unless $progressMeter
1705         mysys(cmd.join(' '))
1706     else
1707        running = {}
1708        didRun = {}
1709        didFail = {}
1710        blankLine = true
1711        prevStringLength = 0
1712        IO.popen(cmd.join(' '), mode="r") {
1713            | inp |
1714            inp.each_line {
1715                | line |
1716                line = line.scrub.chomp
1717                if line =~ /^Running /
1718                    running[$~.post_match] = true
1719                elsif line =~ /^PASS: /
1720                    didRun[$~.post_match] = true
1721                elsif line =~ /^FAIL: /
1722                    didRun[$~.post_match] = true
1723                    didFail[$~.post_match] = true
1724                else
1725                    unless blankLine
1726                        print("\r" + " " * prevStringLength + "\r")
1727                    end
1728                    puts line
1729                    blankLine = true
1730                end
1731
1732                def lpad(str, chars)
1733                    str = str.to_s
1734                    if str.length > chars
1735                        str
1736                    else
1737                       "%#{chars}s"%(str)
1738                    end
1739                end
1740
1741                string  = ""
1742                string += "\r#{lpad(didRun.size, numberOfTests.to_s.size)}/#{numberOfTests}"
1743                unless didFail.empty?
1744                    string += " (failed #{didFail.size})"
1745                end
1746                string += " "
1747                (running.size - didRun.size).times {
1748                    string += "."
1749                }
1750                if string.length < prevStringLength
1751                    print string
1752                    print(" " * (prevStringLength - string.length))
1753                end
1754                print string
1755                prevStringLength = string.length
1756                blankLine = false
1757                $stdout.flush
1758            }
1759        }
1760        puts
1761        raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success?
1762     end
1763 end
1764
1765 def runTestRunner
1766     case $testRunnerType
1767     when :shell
1768         testRunnerCommand = "sh runscript"
1769     when :make
1770         testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
1771     else
1772         raise "Unknown test runner type: #{$testRunnerType.to_s}"
1773     end
1774
1775     if $remote
1776         if !$remoteDirectory
1777             $remoteDirectory = JSON::parse(sshRead("cat ~/.bencher"))["tempPath"]
1778         end
1779         mysys("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", "mkdir -p #{$remoteDirectory}")
1780         mysys("scp", "-P", $remotePort.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{$remoteUser}@#{$remoteHost}:#{$remoteDirectory}")
1781         remoteScript = "\""
1782         remoteScript += "cd #{$remoteDirectory} && "
1783         remoteScript += "rm -rf #{$outputDir.basename} && "
1784         remoteScript += "tar xzf #{$tarFileName} && "
1785         remoteScript += "cd #{$outputDir.basename}/.runner && "
1786         remoteScript += "export DYLD_FRAMEWORK_PATH=\\\"\\$(cd #{$testingFrameworkPath.dirname}; pwd)\\\" && "
1787         remoteScript += "export LD_LIBRARY_PATH=#{$remoteDirectory}/#{$outputDir.basename}/#{$jscPath.dirname} && "
1788         remoteScript += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])} && "
1789         $envVars.each { |var| remoteScript += "export " << var << "\n" }
1790         remoteScript += "#{testRunnerCommand}\""
1791         runAndMonitorTestRunnerCommand("ssh", "-p", $remotePort.to_s, "#{$remoteUser}@#{$remoteHost}", remoteScript)
1792     else
1793         Dir.chdir($runnerDir) {
1794             runAndMonitorTestRunnerCommand(testRunnerCommand)
1795         }
1796     end
1797 end
1798
1799 def detectFailures
1800     raise if $bundle
1801
1802     failures = []
1803     
1804     if $remote
1805         output = sshRead("cd #{$remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"test_fail_*\"")
1806         output.split(/\n/).each {
1807             | line |
1808             next unless line =~ /test_fail_/
1809             failures << $~.post_match.to_i
1810         }
1811     else
1812         Dir.foreach($runnerDir) {
1813             | filename |
1814             next unless filename =~ /test_fail_/
1815             failures << $~.post_match.to_i
1816         }
1817     end
1818
1819     failureSet = {}
1820
1821     failures.each {
1822         | failure | 
1823         appendFailure($runlist[failure])
1824         failureSet[failure] = true
1825     }
1826
1827     familyMap = {}
1828     $runlist.each_with_index {
1829         | plan, index |
1830         unless familyMap[plan.family]
1831             familyMap[plan.family] = []
1832         end
1833         if failureSet[index]
1834             appendResult(plan, false)
1835             familyMap[plan.family] << {:result => "FAIL", :plan => plan};
1836             next
1837         else
1838             appendResult(plan, true)
1839             familyMap[plan.family] << {:result => "PASS", :plan => plan};
1840         end
1841         appendPass(plan)
1842     }
1843
1844     File.open($outputDir + "resultsByFamily", "w") {
1845         | outp |
1846         first = true
1847         familyMap.keys.sort.each {
1848             | familyName |
1849             if first
1850                 first = false
1851             else
1852                 outp.puts
1853             end
1854             
1855             outp.print "#{familyName}:"
1856
1857             numPassed = 0
1858             familyMap[familyName].each {
1859                 | entry |
1860                 if entry[:result] == "PASS"
1861                     numPassed += 1
1862                 end
1863             }
1864
1865             if numPassed == familyMap[familyName].size
1866                 outp.puts " PASSED"
1867             elsif numPassed == 0
1868                 outp.puts " FAILED"
1869             else
1870                 outp.puts
1871                 familyMap[familyName].each {
1872                     | entry |
1873                     outp.puts "    #{entry[:plan].name}: #{entry[:result]}"
1874                 }
1875             end
1876         }
1877     }
1878 end
1879
1880 def compressBundle
1881     cmd = "cd #{$outputDir}/.. && tar -czf #{$tarFileName} #{$outputDir.basename}"
1882     $stderr.puts ">> #{cmd}" if $verbosity >= 2
1883     raise unless system(cmd)
1884 end
1885
1886 def clean(file)
1887     FileUtils.rm_rf file unless $bundle
1888 end
1889
1890 clean($outputDir + "failed")
1891 clean($outputDir + "passed")
1892 clean($outputDir + "results")
1893 clean($outputDir + "resultsByFamily")
1894 clean($outputDir + ".vm")
1895 clean($outputDir + ".helpers")
1896 clean($outputDir + ".runner")
1897 clean($outputDir + ".tests")
1898 clean($outputDir + "_payload")
1899
1900 Dir.mkdir($outputDir) unless $outputDir.directory?
1901
1902 $outputDir = $outputDir.realpath
1903 $runnerDir = $outputDir + ".runner"
1904
1905 if !$numChildProcesses
1906     if ENV["WEBKIT_TEST_CHILD_PROCESSES"]
1907         $numChildProcesses = ENV["WEBKIT_TEST_CHILD_PROCESSES"].to_i
1908     else
1909         $numChildProcesses = numberOfProcessors
1910     end
1911 end
1912
1913 if $enableFTL and ENV["JSCTEST_timeout"]
1914     # Currently, using the FTL is a performance regression particularly in real
1915     # (i.e. non-loopy) benchmarks. Account for this in the timeout.
1916     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i * 2).to_s
1917 end
1918
1919 if ENV["JSCTEST_timeout"]
1920     # In the worst case, the processors just interfere with each other.
1921     # Increase the timeout proportionally to the number of processors.
1922     ENV["JSCTEST_timeout"] = (ENV["JSCTEST_timeout"].to_i.to_f * Math.sqrt($numChildProcesses)).to_i.to_s
1923 end
1924
1925 def runBundle
1926     raise unless $bundle
1927
1928     cleanRunnerDirectory
1929     cleanOldResults
1930     runTestRunner
1931     cleanEmptyResultFiles
1932 end
1933
1934 def runNormal
1935     raise if $bundle or $tarball
1936
1937     prepareBundle
1938     prepareTestRunner
1939     runTestRunner
1940     cleanEmptyResultFiles
1941     detectFailures
1942 end
1943
1944 def runTarball
1945     raise unless $tarball
1946
1947     prepareBundle 
1948     prepareTestRunner
1949     compressBundle
1950 end
1951
1952 def runRemote
1953     raise unless $remote
1954
1955     prepareBundle
1956     prepareTestRunner
1957     compressBundle
1958     runTestRunner
1959     detectFailures
1960 end
1961
1962 puts
1963 if $bundle
1964     runBundle
1965 elsif $remote
1966     runRemote
1967 elsif $tarball
1968     runTarball
1969 else
1970     runNormal
1971 end