Add support for webkitpy tests EWS
[WebKit-https.git] / Tools / Scripts / webkitpy / tool / steps / runtests.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 # Copyright (C) 2017 Apple Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import logging
31 import optparse
32 import os
33 import platform
34 import re
35 import sys
36 from webkitpy.tool.steps.abstractstep import AbstractStep
37 from webkitpy.tool.steps.options import Options
38 from webkitpy.common.system.executive import ScriptError, Executive
39 from webkitpy.common.host import Host
40 from webkitpy.layout_tests.controllers.layout_test_finder import LayoutTestFinder
41
42 _log = logging.getLogger(__name__)
43
44
45 class RunTests(AbstractStep):
46     # FIXME: This knowledge really belongs in the commit-queue.
47     NON_INTERACTIVE_FAILURE_LIMIT_COUNT = 30
48
49     @classmethod
50     def options(cls):
51         return AbstractStep.options() + [
52             Options.build_style,
53             Options.test,
54             Options.iterate_on_new_tests,
55             Options.non_interactive,
56             Options.quiet,
57             Options.group,
58         ]
59
60     def run(self, state):
61         if self._options.group == "jsc":
62             self._run_javascriptcore_tests()
63             return
64
65         if self._options.group == "api":
66             self._run_api_tests()
67             return
68
69         if self._options.group == "bindings":
70             self._run_bindings_tests()
71             return
72
73         if self._options.group == "webkitpy":
74             self._run_webkitpy_tests()
75             return
76
77         if self._options.iterate_on_new_tests:
78             _log.info("Running run-webkit-tests on new tests")
79             self._run_webkit_tests(self._options.iterate_on_new_tests)
80
81         if not self._options.test:
82             return
83
84         python_unittests_command = self._tool.deprecated_port().run_python_unittests_command()
85         if python_unittests_command:
86             _log.info("Running Python unit tests")
87             if self._options.non_interactive:
88                 filesystem = self._tool.filesystem
89                 python_unittest_results_directory = self._tool.port_factory.get().python_unittest_results_directory()
90                 filesystem.maybe_make_directory(python_unittest_results_directory)
91
92                 python_unittests_command.append('--json')
93                 output = self._tool.executive.run_command(python_unittests_command, cwd=self._tool.scm().checkout_root, error_handler=Executive.ignore_error, return_stderr=False)
94                 filesystem.write_text_file(filesystem.join(python_unittest_results_directory, "results.json"), output)
95             else:
96                 self._tool.executive.run_and_throw_if_fail(python_unittests_command, cwd=self._tool.scm().checkout_root)
97
98         if not self._options.non_interactive:
99             # FIXME: We should teach the commit-queue and the EWS how to run these tests.
100
101             perl_unittests_command = self._tool.deprecated_port().run_perl_unittests_command()
102             if perl_unittests_command:
103                 _log.info("Running Perl unit tests")
104                 self._tool.executive.run_and_throw_if_fail(perl_unittests_command, cwd=self._tool.scm().checkout_root)
105
106             javascriptcore_tests_command = self._tool.deprecated_port().run_javascriptcore_tests_command()
107             if javascriptcore_tests_command:
108                 _log.info("Running JavaScriptCore tests")
109                 self._tool.executive.run_and_throw_if_fail(javascriptcore_tests_command, quiet=True, cwd=self._tool.scm().checkout_root)
110
111         _log.info("Running run-webkit-tests")
112         self._run_webkit_tests()
113
114     def _new_or_modified_tests(self):
115         touched_files = self._tool.scm().changed_files()
116         touched_files.extend(self._tool.scm().untracked_files())
117         if not touched_files:
118             return None
119
120         configuration = "Debug" if (self._options.build_style == "debug") else "Release"
121         port = Host().port_factory.get(self._tool.deprecated_port().port_flag_name, optparse.Values({'configuration': configuration}))
122         return LayoutTestFinder(port, optparse.Values({'skipped': 'always', 'skip_failing_tests': False, 'http': True})).find_touched_tests(touched_files)
123
124     def _run_webkit_tests(self, iterate_on_new_tests=0):
125         args = self._tool.deprecated_port().run_webkit_tests_command(build_style=self._options.build_style)
126         if self._options.non_interactive:
127             args.extend([
128                 "--no-new-test-results",
129                 "--no-show-results",
130                 "--exit-after-n-failures=%s" % self.NON_INTERACTIVE_FAILURE_LIMIT_COUNT,
131             ])
132
133             # old-run-webkit-tests does not support --skip-failing-tests
134             # Using --quiet one Windows fails when we try to use /dev/null, disabling for now until we find a fix
135             if sys.platform != "cygwin":
136                 args.append("--quiet")
137                 args.append("--skip-failing-tests")
138             else:
139                 args.append("--no-build");
140
141         if self._options.quiet:
142             args.append("--quiet")
143
144         if iterate_on_new_tests:
145             new_tests = self._new_or_modified_tests()
146             if not new_tests:
147                 _log.info("No new or modified tests")
148                 return
149             _log.info("Touched tests are: " + str(new_tests))
150             for test in new_tests:
151                 args.append(test)
152             args.append("--iterations=%d" % iterate_on_new_tests)
153
154         self._tool.executive.run_and_throw_if_fail(args, cwd=self._tool.scm().checkout_root)
155
156     def _run_javascriptcore_tests(self):
157         args = self._tool.deprecated_port().run_javascriptcore_tests_command(self._options.build_style)
158
159         results_directory = self._tool.port_factory.get(options=self._options).jsc_results_directory()
160         results_file_path = self._tool.filesystem.join(results_directory, "jsc_test_results.json")
161         args.append("--json-output=%s" % results_file_path)
162         self._tool.executive.run_and_throw_if_fail(args, cwd=self._tool.scm().checkout_root)
163
164     def _run_bindings_tests(self):
165         args = self._tool.deprecated_port().run_bindings_tests_command()
166         results_directory = self._tool.port_factory.get(options=self._options).bindings_results_directory()
167         self._tool.filesystem.maybe_make_directory(results_directory)
168         results_file_path = self._tool.filesystem.join(results_directory, "bindings_test_results.json")
169         args.append("--json-output=%s" % results_file_path)
170         self._tool.executive.run_and_throw_if_fail(args, cwd=self._tool.scm().checkout_root)
171
172     def _run_webkitpy_tests(self):
173         args = self._tool.deprecated_port().run_python_unittests_command()
174         results_directory = self._tool.port_factory.get(options=self._options).python_unittest_results_directory()
175         self._tool.filesystem.maybe_make_directory(results_directory)
176         results_file_path = self._tool.filesystem.join(results_directory, "webkitpy_test_results.json")
177         args.append("--json-output={}".format(results_file_path))
178         self._tool.executive.run_and_throw_if_fail(args, cwd=self._tool.scm().checkout_root)
179
180     def _run_api_tests(self):
181         args = self._tool.deprecated_port().run_api_tests_command(self._options.build_style)
182         results_directory = self._tool.port_factory.get(options=self._options).api_results_directory()
183         results_file_path = self._tool.filesystem.join(results_directory, "api_test_results.json")
184         args.append("--json-output=%s" % results_file_path)
185         self._tool.executive.run_and_throw_if_fail(args, cwd=self._tool.scm().checkout_root)