pytest is not correctly auto-installed
[WebKit-https.git] / Tools / Scripts / import-test262-tests
1 #!/usr/bin/env ruby
2
3 # Copyright (C) 2016 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
7 # are met:
8 #
9 # 1.  Redistributions of source code must retain the above copyright
10 #     notice, this list of conditions and the following disclaimer.
11 # 2.  Redistributions in binary form must reproduce the above copyright
12 #     notice, this list of conditions and the following disclaimer in the
13 #     documentation and/or other materials provided with the distribution.
14 #
15 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
16 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
17 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
18 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
19 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
20 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
22 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25
26 # Steps to update test262 test resources and yaml:
27 #
28 #   1. Run importer to update the yaml and test resources.
29 #       $ import-test262-tests /path/to/test262
30 #   2. Run the tests to produce a list of failures.
31 #       $ run-jsc-stress-tests JSTests/test262.yaml
32 #   3. Run importer to update the yaml with a new list of failures.
33 #       $ import-test262-tests --failures ./results/failed /path/to/test262
34
35 require 'fileutils'
36 require 'getoptlong'
37 require 'pathname'
38 require 'yaml'
39 require 'find'
40 require 'shellwords'
41
42 THIS_SCRIPT_PATH = Pathname.new(__FILE__).realpath
43 SCRIPTS_PATH = THIS_SCRIPT_PATH.dirname
44 WEBKIT_PATH = SCRIPTS_PATH.dirname.dirname
45 TEST262_YAML_PATH = WEBKIT_PATH + "JSTests/test262.yaml"
46 TEST262_REVISION_PATH = WEBKIT_PATH + "JSTests/test262/test262-Revision.txt"
47
48 raise unless SCRIPTS_PATH.basename.to_s == "Scripts"
49 raise unless SCRIPTS_PATH.dirname.basename.to_s == "Tools"
50
51 def usage
52     puts "#{File.basename $0} [options] <path-to-test262-repository>"
53     puts
54     puts "-h, --help                   print this help message."
55     puts "-f, --failures FAILURES      Supplied file will be used to determine which tests fail."
56     puts "                             If a failures file is not provided all tests are assumed to pass,"
57     puts "                             and test resources are not updated, only the yaml is updated."
58     exit 1
59 end
60
61 $failures = nil
62 JS_TEST_REGEXP = /.*\.js/
63 JS_FIXTURE_REGEXP = /.*(_FIXTURE\.js|_\.js)/
64 GET_YAML_REGEXP = /\/\*---(?<yaml>.*?)---\*\//m
65
66 GetoptLong.new(["--help", "-h", GetoptLong::NO_ARGUMENT],
67                ["--failures", "-f", GetoptLong::REQUIRED_ARGUMENT]).each {
68     | opt, arg |
69     case opt
70     when "--help"
71         usage
72     when "--failures"
73         $failures = {}
74         File.open(Pathname.new(arg)).readlines.each {
75             | line |
76             match = line.match(/test262\/(.*?\.default.*?$)/)
77             if (match)
78                 $failures[match[1]] = true
79             end
80         }
81     end
82 }
83
84 def didPassForMode(path, strict)
85     if $failures
86         if strict == :strict
87             return !$failures.key?(path.to_s + ".default-strict")
88         else
89             return !$failures.key?(path.to_s + ".default")
90         end
91     else
92         return true
93     end
94 end
95
96 class Test
97     attr_writer :failsWithException, :isModule, :isAsync
98     attr_accessor :includeFiles, :needsStrict, :needsNonStrict
99     attr_reader :path
100
101     def initialize(path)
102         @path = path
103         @failsWithException = nil
104         @includeFiles = []
105         @needsStrict = true
106         @needsNonStrict = true
107         @isModule = false
108         @isAsync = false
109     end
110
111     def check
112         # Throw an exception here since I'm not sure if the test infrastructure works in these cases.
113         raise if !@needsStrict and !@needsNonStrict
114         raise if @isModule and !@needsNonStrict
115     end
116
117     def formatFlags(strict)
118         flags = []
119         flags << strict if strict == :strict
120         flags << :module if @isModule
121         flags << :async if @isAsync
122
123         return flags.to_s
124     end
125
126     def formatCmdArguments(strict)
127         raise if strict == :strict ? !@needsStrict : !@needsNonStrict
128         passed = didPassForMode(@path, strict)
129         cmd = "runTest262"
130         cmd += passed ? " :normal, " : " :fail, "
131         cmd += @failsWithException ? @failsWithException.inspect : "\"NoException\""
132         cmd += ", #{@includeFiles.inspect}"
133         cmd += ", #{formatFlags(strict)}"
134     end
135
136     def finalizeIncludes
137         if @isAsync
138             @includeFiles << "doneprintHandle.js"
139         end
140
141         dir = Pathname.new(".")
142         @path.dirname.each_filename {
143             | part |
144             dir += ".."
145         }
146         dir += "harness"
147
148         @includeFiles.map! { | file | (dir + file).to_s }
149     end
150
151 end
152
153 def processTestFile(path)
154     /\/\*---(?<yaml>.*?)---\*\//m =~ File::read(path)
155
156     test = Test.new(path)
157     # These should be included by all the tests
158     test.includeFiles = ["assert.js", "sta.js"]
159
160     begin
161         yamlElements = YAML::load(yaml)
162     rescue Exception => e
163         puts "Failed to parse YAML for #{path}, threw exception:"
164         puts e.inspect
165     end
166     yamlElements.each {
167         | option |
168         case option[0]
169         when "negative"
170             test.failsWithException = option[1]["type"].to_s
171         when "includes"
172             test.includeFiles += option[1]
173         when "flags"
174             option[1].each {
175                 | flag |
176                 case flag
177                 when "raw", "noStrict"
178                     test.needsStrict = false
179                 when "onlyStrict"
180                     test.needsNonStrict = false
181                 when "module"
182                     test.isModule = true
183                     test.needsStrict = false
184                 when "async"
185                     test.isAsync = true
186                 when "generated"
187                 else
188                     raise "Invalid Metadata flag option, #{flag}, when parsing #{$benchmarkDirectory + $benchmark}"
189                 end
190             }
191         end
192     }
193
194     test.finalizeIncludes
195     test.check
196
197     return test
198 end
199
200 class Fixture
201     attr_reader :path, :needsNonStrict, :needsStrict
202
203     def initialize(path)
204         @path = path
205         @needsNonStrict = true
206         @needsStrict = false
207     end
208
209     def formatCmdArguments(strict)
210         return "prepareTest262Fixture"
211     end
212 end
213
214 def processFixtureFile(path)
215     Fixture.new(path)
216 end
217
218 def processFilesRecursively(path)
219     tests = []
220
221     Dir.chdir(path) {
222         Find.find(Pathname.new("test")) {
223             | file |
224             if File.file?(file) and JS_TEST_REGEXP =~ file.to_s
225                 path = Pathname.new(file)
226                 if JS_FIXTURE_REGEXP =~ file.to_s
227                     tests << processFixtureFile(path)
228                 else
229                     tests << processTestFile(path)
230                 end
231             end
232         }
233     }
234
235     return tests
236 end
237
238 def printYAML(tests)
239     File.open(TEST262_YAML_PATH, "w+") {
240         | outp |
241         outp.puts "---"
242         tests.each {
243             | test |
244             if test.needsNonStrict
245                 outp.puts "- path: test262/" + test.path.to_s
246                 outp.puts "  cmd: " + test.formatCmdArguments(:nonStrict)
247             end
248             if test.needsStrict
249                 outp.puts "- path: test262/" + test.path.to_s
250                 outp.puts "  cmd: " + test.formatCmdArguments(:strict)
251             end
252         }
253     }
254 end
255
256 def printRevision(test262Path)
257     url = "unknown"
258     branchname = url
259     revision = url
260     Dir.chdir(test262Path) {
261         tracking = `git rev-parse --abbrev-ref --symbolic-full-name @{u}`.chomp
262         remoteName, branchName = tracking.split("/")
263         url = `git remote get-url #{remoteName}`
264         revision = `git rev-parse HEAD`.strip
265     }
266
267     File.open(TEST262_REVISION_PATH, "w+") {
268         | f |
269         puts "test262 remote url: " + url
270         puts "test262 revision: " + revision
271         f.puts "test262 remote url: " + url
272         f.puts "test262 revision: " + revision
273     }
274 end
275
276 def replaceResources(test262Path)
277     harnessSource = File.join(test262Path, "harness")
278     harnessDestination = File.join(WEBKIT_PATH, "JSTests", "test262", "harness")
279     testSource = File.join(test262Path, "test")
280     testDestination = File.join(WEBKIT_PATH, "JSTests", "test262", "test")
281
282     FileUtils.rm_r harnessDestination.to_s
283     FileUtils.cp_r harnessSource.to_s, harnessDestination.to_s
284
285     FileUtils.rm_r testDestination.to_s
286     FileUtils.cp_r testSource.to_s, testDestination.to_s
287 end
288
289 if ARGV.empty?
290     usage
291 end
292
293 puts "Writing #{File.basename(TEST262_REVISION_PATH)}..."
294 test262Path = Pathname.new(ARGV[0])
295 printRevision(test262Path)
296
297 puts "Enumerating tests..."
298 tests = processFilesRecursively(test262Path)
299
300 puts "Writing #{File.basename(TEST262_YAML_PATH)}..."
301 printYAML(tests)
302
303 if !$failures
304     puts "Replacing test262 resources..."
305     replaceResources(test262Path)
306 end