REGRESSION (r217572): run-webkit-tests exits without emitting newline character
[WebKit.git] / Tools / Scripts / webkitpy / layout_tests / controllers / layout_test_runner_unittest.py
1 # Copyright (C) 2012 Google Inc. All rights reserved.
2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 import unittest
31
32 from webkitpy.common.host_mock import MockHost
33 from webkitpy.common.system.systemhost_mock import MockSystemHost
34 from webkitpy.layout_tests import run_webkit_tests
35 from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner, Sharder, TestRunInterruptedException
36 from webkitpy.layout_tests.models import test_expectations
37 from webkitpy.layout_tests.models import test_failures
38 from webkitpy.layout_tests.models.test_run_results import TestRunResults
39 from webkitpy.layout_tests.models.test_input import TestInput
40 from webkitpy.layout_tests.models.test_results import TestResult
41 from webkitpy.port.test import TestPort
42
43
44 TestExpectations = test_expectations.TestExpectations
45
46
47 class FakePrinter(object):
48     num_started = 0
49     num_tests = 0
50
51     def print_expected(self, run_results, get_tests_with_result_type):
52         pass
53
54     def print_workers_and_shards(self, num_workers, num_shards):
55         pass
56
57     def print_started_test(self, test_name):
58         pass
59
60     def print_finished_test(self, result, expected, exp_str, got_str):
61         pass
62
63     def write(self, msg):
64         pass
65
66     def write_update(self, msg):
67         pass
68
69     def flush(self):
70         pass
71
72
73 class LayoutTestRunnerTests(unittest.TestCase):
74     def _runner(self, port=None):
75         # FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need.
76         options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopard'])[0]
77         options.child_processes = '1'
78
79         host = MockHost()
80         port = port or host.port_factory.get(options.platform, options=options)
81         return LayoutTestRunner(options, port, FakePrinter(), port.results_directory(), lambda test_name: False)
82
83     def _run_tests(self, runner, tests):
84         test_inputs = [TestInput(test, 6000) for test in tests]
85         expectations = TestExpectations(runner._port, tests)
86         expectations.parse_all_expectations()
87         runner.run_tests(expectations, test_inputs, set(),
88             num_workers=1, needs_http=any('http' in test for test in tests), needs_websockets=any(['websocket' in test for test in tests]), needs_web_platform_test_server=any(['imported/w3c' in test for test in tests]), retrying=False)
89
90     def test_interrupt_if_at_failure_limits(self):
91         runner = self._runner()
92         runner._options.exit_after_n_failures = None
93         runner._options.exit_after_n_crashes_or_times = None
94         test_names = ['passes/text.html', 'passes/image.html']
95         runner._test_inputs = [TestInput(test_name, 6000) for test_name in test_names]
96
97         expectations = TestExpectations(runner._port, test_names)
98         expectations.parse_all_expectations()
99         run_results = TestRunResults(expectations, len(test_names))
100         run_results.unexpected_failures = 100
101         run_results.unexpected_crashes = 50
102         run_results.unexpected_timeouts = 50
103         # No exception when the exit_after* options are None.
104         runner._interrupt_if_at_failure_limits(run_results)
105
106         # No exception when we haven't hit the limit yet.
107         runner._options.exit_after_n_failures = 101
108         runner._options.exit_after_n_crashes_or_timeouts = 101
109         runner._interrupt_if_at_failure_limits(run_results)
110
111         # Interrupt if we've exceeded either limit:
112         runner._options.exit_after_n_crashes_or_timeouts = 10
113         self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results)
114         self.assertEqual(run_results.results_by_name['passes/text.html'].type, test_expectations.SKIP)
115         self.assertEqual(run_results.results_by_name['passes/image.html'].type, test_expectations.SKIP)
116
117         runner._options.exit_after_n_crashes_or_timeouts = None
118         runner._options.exit_after_n_failures = 10
119         exception = self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results)
120
121     def test_update_summary_with_result(self):
122         # Reftests expected to be image mismatch should be respected when pixel_tests=False.
123         runner = self._runner()
124         runner._options.pixel_tests = False
125         test = 'failures/expected/reftest.html'
126         expectations = TestExpectations(runner._port, tests=[test])
127         expectations.parse_all_expectations()
128         runner._expectations = expectations
129
130         run_results = TestRunResults(expectations, 1)
131         result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()], reftest_type=['!='])
132         runner._update_summary_with_result(run_results, result)
133         self.assertEqual(1, run_results.expected)
134         self.assertEqual(0, run_results.unexpected)
135
136         run_results = TestRunResults(expectations, 1)
137         result = TestResult(test_name=test, failures=[], reftest_type=['=='])
138         runner._update_summary_with_result(run_results, result)
139         self.assertEqual(0, run_results.expected)
140         self.assertEqual(1, run_results.unexpected)
141
142     def test_servers_started(self):
143
144         def start_http_server():
145             self.http_started = True
146
147         def start_websocket_server():
148             self.websocket_started = True
149
150         def start_web_platform_test_server():
151             self.web_platform_test_server_started = True
152
153         def stop_http_server():
154             self.http_stopped = True
155
156         def stop_websocket_server():
157             self.websocket_stopped = True
158
159         def stop_web_platform_test_server():
160             self.web_platform_test_server_stopped = True
161
162         def is_http_server_running():
163             return self.http_started and not self.http_stopped
164
165         def is_websocket_server_running():
166             return self.websocket_started and not self.websocket_stopped
167
168         def is_wpt_server_running():
169             return self.websocket_started and not self.web_platform_test_server_stopped
170
171         host = MockHost()
172         port = host.port_factory.get('test-mac-leopard')
173         port.start_http_server = start_http_server
174         port.start_websocket_server = start_websocket_server
175         port.start_web_platform_test_server = start_web_platform_test_server
176         port.stop_http_server = stop_http_server
177         port.stop_websocket_server = stop_websocket_server
178         port.stop_web_platform_test_server = stop_web_platform_test_server
179         port.is_http_server_running = is_http_server_running
180         port.is_websocket_server_running = is_websocket_server_running
181         port.is_wpt_server_running = is_wpt_server_running
182
183         self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
184         self.web_platform_test_server_started = self.web_platform_test_server_stopped = False
185         runner = self._runner(port=port)
186         runner._needs_http = True
187         runner._needs_websockets = False
188         runner._needs_web_platform_test_server = False
189         runner.start_servers()
190         self.assertEqual(self.http_started, True)
191         self.assertEqual(self.websocket_started, False)
192         self.assertEqual(self.web_platform_test_server_started, False)
193         runner.stop_servers()
194         self.assertEqual(self.http_stopped, True)
195         self.assertEqual(self.websocket_stopped, False)
196         self.assertEqual(self.web_platform_test_server_stopped, False)
197
198         self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
199         self.web_platform_test_server_started = self.web_platform_test_server_stopped = False
200         runner._needs_http = True
201         runner._needs_websockets = True
202         runner._needs_web_platform_test_server = False
203         runner.start_servers()
204         self.assertEqual(self.http_started, True)
205         self.assertEqual(self.websocket_started, True)
206         self.assertEqual(self.web_platform_test_server_started, False)
207         runner.stop_servers()
208         self.assertEqual(self.http_stopped, True)
209         self.assertEqual(self.websocket_stopped, True)
210         self.assertEqual(self.web_platform_test_server_stopped, False)
211
212         self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
213         self.web_platform_test_server_started = self.web_platform_test_server_stopped = False
214         runner._needs_http = False
215         runner._needs_websockets = False
216         runner._needs_web_platform_test_server = True
217         runner.start_servers()
218         self.assertEqual(self.http_started, False)
219         self.assertEqual(self.websocket_started, False)
220         self.assertEqual(self.web_platform_test_server_started, True)
221         runner.stop_servers()
222         self.assertEqual(self.http_stopped, False)
223         self.assertEqual(self.websocket_stopped, False)
224         self.assertEqual(self.web_platform_test_server_stopped, True)
225
226         self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
227         self.web_platform_test_server_started = self.web_platform_test_server_stopped = False
228         runner._needs_http = False
229         runner._needs_websockets = False
230         runner._needs_web_platform_test_server = False
231         runner.start_servers()
232         self.assertEqual(self.http_started, False)
233         self.assertEqual(self.websocket_started, False)
234         self.assertEqual(self.web_platform_test_server_started, False)
235         runner.stop_servers()
236         self.assertEqual(self.http_stopped, False)
237         self.assertEqual(self.websocket_stopped, False)
238         self.assertEqual(self.web_platform_test_server_stopped, False)
239
240
241 class SharderTests(unittest.TestCase):
242
243     test_list = [
244         "http/tests/websocket/tests/unicode.htm",
245         "animations/keyframes.html",
246         "http/tests/security/view-source-no-refresh.html",
247         "http/tests/websocket/tests/websocket-protocol-ignored.html",
248         "fast/css/display-none-inline-style-change-crash.html",
249         "http/tests/xmlhttprequest/supported-xml-content-types.html",
250         "dom/html/level2/html/HTMLAnchorElement03.html",
251         "ietestcenter/Javascript/11.1.5_4-4-c-1.html",
252         "dom/html/level2/html/HTMLAnchorElement06.html",
253     ]
254
255     def get_test_input(self, test_file):
256         return TestInput(test_file, needs_servers=(test_file.startswith('http')))
257
258     def get_shards(self, num_workers, fully_parallel, test_list=None):
259         port = TestPort(MockSystemHost())
260         self.sharder = Sharder(port.split_test)
261         test_list = test_list or self.test_list
262         return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel)
263
264     def assert_shards(self, actual_shards, expected_shard_names):
265         self.assertEqual(len(actual_shards), len(expected_shard_names))
266         for i, shard in enumerate(actual_shards):
267             expected_shard_name, expected_test_names = expected_shard_names[i]
268             self.assertEqual(shard.name, expected_shard_name)
269             self.assertEqual([test_input.test_name for test_input in shard.test_inputs],
270                               expected_test_names)
271
272     def test_shard_by_dir(self):
273         result = self.get_shards(num_workers=2, fully_parallel=False)
274
275         self.assert_shards(result,
276             [('animations', ['animations/keyframes.html']),
277              ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html',
278                                       'dom/html/level2/html/HTMLAnchorElement06.html']),
279              ('fast/css', ['fast/css/display-none-inline-style-change-crash.html']),
280              ('http/tests/security', ['http/tests/security/view-source-no-refresh.html']),
281              ('http/tests/websocket/tests', ['http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html']),
282              ('http/tests/xmlhttprequest', ['http/tests/xmlhttprequest/supported-xml-content-types.html']),
283              ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])])
284
285     def test_shard_every_file(self):
286         result = self.get_shards(num_workers=2, fully_parallel=True)
287         self.assert_shards(result,
288             [('.', ['http/tests/websocket/tests/unicode.htm']),
289              ('.', ['animations/keyframes.html']),
290              ('.', ['http/tests/security/view-source-no-refresh.html']),
291              ('.', ['http/tests/websocket/tests/websocket-protocol-ignored.html']),
292              ('.', ['fast/css/display-none-inline-style-change-crash.html']),
293              ('.', ['http/tests/xmlhttprequest/supported-xml-content-types.html']),
294              ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']),
295              ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']),
296              ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])