clean up NRWT logging/metering, be less verbose
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / controllers / manager_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """Unit tests for manager.py."""
32
33 import StringIO
34 import sys
35 import unittest
36
37 from webkitpy.common.system.filesystem_mock import MockFileSystem
38 from webkitpy.common.system import outputcapture
39 from webkitpy.thirdparty.mock import Mock
40 from webkitpy import layout_tests
41 from webkitpy.layout_tests.port import port_testcase
42
43 from webkitpy import layout_tests
44 from webkitpy.layout_tests import run_webkit_tests
45 from webkitpy.layout_tests.controllers import manager
46 from webkitpy.layout_tests.controllers.manager import interpret_test_failures,  Manager, natural_sort_key, test_key, TestRunInterruptedException, TestShard
47 from webkitpy.layout_tests.models import result_summary
48 from webkitpy.layout_tests.models import test_expectations
49 from webkitpy.layout_tests.models import test_failures
50 from webkitpy.layout_tests.models import test_results
51 from webkitpy.layout_tests.models.result_summary import ResultSummary
52 from webkitpy.layout_tests.models.test_expectations import TestExpectations
53 from webkitpy.layout_tests.models.test_results import TestResult
54 from webkitpy.layout_tests.views import printing
55 from webkitpy.tool.mocktool import MockOptions
56 from webkitpy.common.system.executive_mock import MockExecutive
57 from webkitpy.common.host_mock import MockHost
58
59
60 class ManagerWrapper(Manager):
61     def _get_test_input_for_file(self, test_file):
62         return test_file
63
64
65 class ShardingTests(unittest.TestCase):
66     test_list = [
67         "http/tests/websocket/tests/unicode.htm",
68         "animations/keyframes.html",
69         "http/tests/security/view-source-no-refresh.html",
70         "http/tests/websocket/tests/websocket-protocol-ignored.html",
71         "fast/css/display-none-inline-style-change-crash.html",
72         "http/tests/xmlhttprequest/supported-xml-content-types.html",
73         "dom/html/level2/html/HTMLAnchorElement03.html",
74         "ietestcenter/Javascript/11.1.5_4-4-c-1.html",
75         "dom/html/level2/html/HTMLAnchorElement06.html",
76     ]
77
78     def get_shards(self, num_workers, fully_parallel, test_list=None, max_locked_shards=None):
79         test_list = test_list or self.test_list
80         host = MockHost()
81         port = host.port_factory.get(port_name='test')
82         port._filesystem = MockFileSystem()
83         options = MockOptions(max_locked_shards=max_locked_shards)
84         self.manager = ManagerWrapper(port=port, options=options, printer=Mock())
85         return self.manager._shard_tests(test_list, num_workers, fully_parallel)
86
87     def test_shard_by_dir(self):
88         locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False)
89
90         # Note that although there are tests in multiple dirs that need locks,
91         # they are crammed into a single shard in order to reduce the # of
92         # workers hitting the server at once.
93         self.assertEquals(locked,
94             [TestShard('locked_shard_1',
95               ['http/tests/security/view-source-no-refresh.html',
96                'http/tests/websocket/tests/unicode.htm',
97                'http/tests/websocket/tests/websocket-protocol-ignored.html',
98                'http/tests/xmlhttprequest/supported-xml-content-types.html'])])
99         self.assertEquals(unlocked,
100             [TestShard('animations',
101                        ['animations/keyframes.html']),
102              TestShard('dom/html/level2/html',
103                        ['dom/html/level2/html/HTMLAnchorElement03.html',
104                         'dom/html/level2/html/HTMLAnchorElement06.html']),
105              TestShard('fast/css',
106                        ['fast/css/display-none-inline-style-change-crash.html']),
107              TestShard('ietestcenter/Javascript',
108                        ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])])
109
110     def test_shard_every_file(self):
111         locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True)
112         self.assertEquals(locked,
113             [TestShard('.', ['http/tests/websocket/tests/unicode.htm']),
114              TestShard('.', ['http/tests/security/view-source-no-refresh.html']),
115              TestShard('.', ['http/tests/websocket/tests/websocket-protocol-ignored.html']),
116              TestShard('.', ['http/tests/xmlhttprequest/supported-xml-content-types.html'])])
117         self.assertEquals(unlocked,
118             [TestShard('.', ['animations/keyframes.html']),
119              TestShard('.', ['fast/css/display-none-inline-style-change-crash.html']),
120              TestShard('.', ['dom/html/level2/html/HTMLAnchorElement03.html']),
121              TestShard('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']),
122              TestShard('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])
123
124     def test_shard_in_two(self):
125         locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False)
126         self.assertEquals(locked,
127             [TestShard('locked_tests',
128                        ['http/tests/websocket/tests/unicode.htm',
129                         'http/tests/security/view-source-no-refresh.html',
130                         'http/tests/websocket/tests/websocket-protocol-ignored.html',
131                         'http/tests/xmlhttprequest/supported-xml-content-types.html'])])
132         self.assertEquals(unlocked,
133             [TestShard('unlocked_tests',
134                        ['animations/keyframes.html',
135                         'fast/css/display-none-inline-style-change-crash.html',
136                         'dom/html/level2/html/HTMLAnchorElement03.html',
137                         'ietestcenter/Javascript/11.1.5_4-4-c-1.html',
138                         'dom/html/level2/html/HTMLAnchorElement06.html'])])
139
140     def test_shard_in_two_has_no_locked_shards(self):
141         locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
142              test_list=['animations/keyframe.html'])
143         self.assertEquals(len(locked), 0)
144         self.assertEquals(len(unlocked), 1)
145
146     def test_shard_in_two_has_no_unlocked_shards(self):
147         locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
148              test_list=['http/tests/webcoket/tests/unicode.htm'])
149         self.assertEquals(len(locked), 1)
150         self.assertEquals(len(unlocked), 0)
151
152     def test_multiple_locked_shards(self):
153         locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2)
154         self.assertEqual(locked,
155             [TestShard('locked_shard_1',
156                        ['http/tests/security/view-source-no-refresh.html',
157                         'http/tests/websocket/tests/unicode.htm',
158                         'http/tests/websocket/tests/websocket-protocol-ignored.html']),
159              TestShard('locked_shard_2',
160                         ['http/tests/xmlhttprequest/supported-xml-content-types.html'])])
161
162         locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False)
163         self.assertEquals(locked,
164             [TestShard('locked_shard_1',
165                        ['http/tests/security/view-source-no-refresh.html',
166                         'http/tests/websocket/tests/unicode.htm',
167                         'http/tests/websocket/tests/websocket-protocol-ignored.html',
168                         'http/tests/xmlhttprequest/supported-xml-content-types.html'])])
169
170
171 class ManagerTest(unittest.TestCase):
172     def get_options(self):
173         return MockOptions(pixel_tests=False, new_baseline=False, time_out_ms=6000, slow_time_out_ms=30000, worker_model='inline')
174
175     def get_printer(self):
176         class FakePrinter(object):
177             def __init__(self):
178                 self.output = []
179
180             def print_config(self, msg):
181                 self.output.append(msg)
182
183         return FakePrinter()
184
185     def test_fallback_path_in_config(self):
186         options = self.get_options()
187         host = MockHost()
188         port = host.port_factory.get('test-mac-leopard', options=options)
189         printer = self.get_printer()
190         manager = Manager(port, options, printer)
191         manager.print_config()
192         self.assertTrue('Baseline search path: test-mac-leopard -> test-mac-snowleopard -> generic' in printer.output)
193
194     def test_http_locking(tester):
195         class LockCheckingManager(Manager):
196             def __init__(self, port, options, printer):
197                 super(LockCheckingManager, self).__init__(port, options, printer)
198                 self._finished_list_called = False
199
200             def handle_finished_list(self, source, list_name, num_tests, elapsed_time):
201                 if not self._finished_list_called:
202                     tester.assertEquals(list_name, 'locked_tests')
203                     tester.assertTrue(self._remaining_locked_shards)
204                     tester.assertTrue(self._has_http_lock)
205
206                 super(LockCheckingManager, self).handle_finished_list(source, list_name, num_tests, elapsed_time)
207
208                 if not self._finished_list_called:
209                     tester.assertEquals(self._remaining_locked_shards, [])
210                     tester.assertFalse(self._has_http_lock)
211                     self._finished_list_called = True
212
213         options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', 'http/tests/passes', 'passes'])
214         host = MockHost()
215         port = host.port_factory.get(port_name=options.platform, options=options)
216         run_webkit_tests._set_up_derived_options(port, options)
217         printer = printing.Printer(port, options, StringIO.StringIO(), StringIO.StringIO())
218         manager = LockCheckingManager(port, options, printer)
219         manager.collect_tests(args)
220         manager.parse_expectations()
221         num_unexpected_results = manager.run()
222         printer.cleanup()
223         tester.assertEquals(num_unexpected_results, 0)
224
225     def test_interrupt_if_at_failure_limits(self):
226         port = Mock()  # FIXME: This should be a tighter mock.
227         port.TEST_PATH_SEPARATOR = '/'
228         port._filesystem = MockFileSystem()
229         manager = Manager(port=port, options=MockOptions(), printer=Mock())
230
231         manager._options = MockOptions(exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None)
232         result_summary = ResultSummary(expectations=Mock(), test_files=[])
233         result_summary.unexpected_failures = 100
234         result_summary.unexpected_crashes = 50
235         result_summary.unexpected_timeouts = 50
236         # No exception when the exit_after* options are None.
237         manager._interrupt_if_at_failure_limits(result_summary)
238
239         # No exception when we haven't hit the limit yet.
240         manager._options.exit_after_n_failures = 101
241         manager._options.exit_after_n_crashes_or_timeouts = 101
242         manager._interrupt_if_at_failure_limits(result_summary)
243
244         # Interrupt if we've exceeded either limit:
245         manager._options.exit_after_n_crashes_or_timeouts = 10
246         self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary)
247
248         manager._options.exit_after_n_crashes_or_timeouts = None
249         manager._options.exit_after_n_failures = 10
250         exception = self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary)
251
252     def test_update_summary_with_result(self):
253         host = MockHost()
254         port = host.port_factory.get('test-win-xp')
255         test = 'failures/expected/reftest.html'
256         expectations = TestExpectations(port, tests=[test],
257              expectations='WONTFIX : failures/expected/reftest.html = IMAGE',
258              test_config=port.test_configuration())
259         # Reftests expected to be image mismatch should be respected when pixel_tests=False.
260         manager = Manager(port=port, options=MockOptions(pixel_tests=False, exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None), printer=Mock())
261         manager._expectations = expectations
262         result_summary = ResultSummary(expectations=expectations, test_files=[test])
263         result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()])
264         manager._update_summary_with_result(result_summary, result)
265         self.assertEquals(1, result_summary.expected)
266         self.assertEquals(0, result_summary.unexpected)
267
268     def test_needs_servers(self):
269         def get_manager_with_tests(test_names):
270             port = Mock()  # FIXME: Use a tighter mock.
271             port.TEST_PATH_SEPARATOR = '/'
272             manager = Manager(port, options=MockOptions(http=True), printer=Mock())
273             manager._test_files = set(test_names)
274             manager._test_files_list = test_names
275             return manager
276
277         manager = get_manager_with_tests(['fast/html'])
278         self.assertFalse(manager.needs_servers())
279
280         manager = get_manager_with_tests(['http/tests/misc'])
281         self.assertTrue(manager.needs_servers())
282
283     def integration_test_needs_servers(self):
284         def get_manager_with_tests(test_names):
285             host = MockHost()
286             port = host.port_factory.get()
287             manager = Manager(port, options=MockOptions(test_list=None, http=True), printer=Mock())
288             manager.collect_tests(test_names)
289             return manager
290
291         manager = get_manager_with_tests(['fast/html'])
292         self.assertFalse(manager.needs_servers())
293
294         manager = get_manager_with_tests(['http/tests/mime'])
295         self.assertTrue(manager.needs_servers())
296
297         if sys.platform == 'win32':
298             manager = get_manager_with_tests(['fast\\html'])
299             self.assertFalse(manager.needs_servers())
300
301             manager = get_manager_with_tests(['http\\tests\\mime'])
302             self.assertTrue(manager.needs_servers())
303
304
305 class NaturalCompareTest(unittest.TestCase):
306     def assert_cmp(self, x, y, result):
307         self.assertEquals(cmp(natural_sort_key(x), natural_sort_key(y)), result)
308
309     def test_natural_compare(self):
310         self.assert_cmp('a', 'a', 0)
311         self.assert_cmp('ab', 'a', 1)
312         self.assert_cmp('a', 'ab', -1)
313         self.assert_cmp('', '', 0)
314         self.assert_cmp('', 'ab', -1)
315         self.assert_cmp('1', '2', -1)
316         self.assert_cmp('2', '1', 1)
317         self.assert_cmp('1', '10', -1)
318         self.assert_cmp('2', '10', -1)
319         self.assert_cmp('foo_1.html', 'foo_2.html', -1)
320         self.assert_cmp('foo_1.1.html', 'foo_2.html', -1)
321         self.assert_cmp('foo_1.html', 'foo_10.html', -1)
322         self.assert_cmp('foo_2.html', 'foo_10.html', -1)
323         self.assert_cmp('foo_23.html', 'foo_10.html', 1)
324         self.assert_cmp('foo_23.html', 'foo_100.html', -1)
325
326
327 class KeyCompareTest(unittest.TestCase):
328     def setUp(self):
329         host = MockHost()
330         self.port = host.port_factory.get('test')
331
332     def assert_cmp(self, x, y, result):
333         self.assertEquals(cmp(test_key(self.port, x), test_key(self.port, y)), result)
334
335     def test_test_key(self):
336         self.assert_cmp('/a', '/a', 0)
337         self.assert_cmp('/a', '/b', -1)
338         self.assert_cmp('/a2', '/a10', -1)
339         self.assert_cmp('/a2/foo', '/a10/foo', -1)
340         self.assert_cmp('/a/foo11', '/a/foo2', 1)
341         self.assert_cmp('/ab', '/a/a/b', -1)
342         self.assert_cmp('/a/a/b', '/ab', 1)
343         self.assert_cmp('/foo-bar/baz', '/foo/baz', -1)
344
345
346 class ResultSummaryTest(unittest.TestCase):
347
348     def setUp(self):
349         host = MockHost()
350         self.port = host.port_factory.get(port_name='test')
351
352     def test_interpret_test_failures(self):
353         test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
354             [test_failures.FailureReftestMismatch(self.port.abspath_for_test('foo/reftest-expected.html'))])
355         self.assertTrue('is_reftest' in test_dict)
356         self.assertFalse('is_mismatch_reftest' in test_dict)
357
358         test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
359             [test_failures.FailureReftestMismatch(self.port.abspath_for_test('foo/common.html'))])
360         self.assertTrue('is_reftest' in test_dict)
361         self.assertFalse('is_mismatch_reftest' in test_dict)
362         self.assertEqual(test_dict['ref_file'], 'foo/common.html')
363
364         test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
365             [test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.html'))])
366         self.assertFalse('is_reftest' in test_dict)
367         self.assertTrue(test_dict['is_mismatch_reftest'])
368
369         test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
370             [test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/common.html'))])
371         self.assertFalse('is_reftest' in test_dict)
372         self.assertTrue(test_dict['is_mismatch_reftest'])
373         self.assertEqual(test_dict['ref_file'], 'foo/common.html')
374
375     def get_result(self, test_name, result_type=test_expectations.PASS, run_time=0):
376         failures = []
377         if result_type == test_expectations.TIMEOUT:
378             failures = [test_failures.FailureTimeout()]
379         elif result_type == test_expectations.CRASH:
380             failures = [test_failures.FailureCrash()]
381         return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)
382
383     def get_result_summary(self, port, test_names, expectations_str):
384         expectations = test_expectations.TestExpectations(port, test_names, expectations_str, port.test_configuration(), is_lint_mode=False)
385         return test_names, result_summary.ResultSummary(expectations, test_names), expectations
386
387     # FIXME: Use this to test more of summarize_results. This was moved from printing_unittest.py.
388     def summarized_results(self, port, expected, passing, flaky, extra_tests=[], extra_expectations=None):
389         tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html', 'failures/expected/wontfix.html']
390         if extra_tests:
391             tests.extend(extra_tests)
392
393         expectations = ''
394         if extra_expectations:
395             expectations += extra_expectations
396
397         paths, rs, exp = self.get_result_summary(port, tests, expectations)
398         if expected:
399             rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected)
400             rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected)
401             rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected)
402         elif passing:
403             rs.add(self.get_result('passes/text.html'), expected)
404             rs.add(self.get_result('failures/expected/timeout.html'), expected)
405             rs.add(self.get_result('failures/expected/crash.html'), expected)
406         else:
407             rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected)
408             rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected)
409             rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected)
410
411         for test in extra_tests:
412             rs.add(self.get_result(test, test_expectations.CRASH), expected)
413
414         retry = rs
415         if flaky:
416             paths, retry, exp = self.get_result_summary(port, tests, expectations)
417             retry.add(self.get_result('passes/text.html'), True)
418             retry.add(self.get_result('failures/expected/timeout.html'), True)
419             retry.add(self.get_result('failures/expected/crash.html'), True)
420         unexpected_results = manager.summarize_results(port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False)
421         expected_results = manager.summarize_results(port, exp, rs, retry, test_timings={}, only_unexpected=False, interrupted=False)
422         return expected_results, unexpected_results
423
424     def test_no_svn_revision(self):
425         host = MockHost()
426         port = host.port_factory.get('test')
427         expected_results, unexpected_results = self.summarized_results(port, expected=False, passing=False, flaky=False)
428         self.assertTrue('revision' not in unexpected_results)
429
430     def test_svn_revision(self):
431         host = MockHost()
432         port = host.port_factory.get('test')
433         port._options.builder_name = 'dummy builder'
434         expected_results, unexpected_results = self.summarized_results(port, expected=False, passing=False, flaky=False)
435         self.assertTrue('revision' in unexpected_results)
436
437     def test_summarized_results_wontfix(self):
438         host = MockHost()
439         port = host.port_factory.get('test')
440         port._options.builder_name = 'dummy builder'
441         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), "failures/expected/wontfix.html"), "Dummy test contents")
442         expected_results, unexpected_results = self.summarized_results(port, expected=False, passing=False, flaky=False, extra_tests=['failures/expected/wontfix.html'], extra_expectations='BUGX WONTFIX : failures/expected/wontfix.html = FAIL\n')
443         self.assertTrue(expected_results['tests']['failures']['expected']['wontfix.html']['wontfix'])
444
445 if __name__ == '__main__':
446     port_testcase.main()