Tools/Scripts/webkitpy/layout_tests/controllers/manager_unittest.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Unit tests for manager.py."""
  32
  33 import StringIO
  34 import sys
  35 import unittest
  36
  37 from webkitpy.common.system.filesystem_mock import MockFileSystem
  38 from webkitpy.common.system import outputcapture
  39 from webkitpy.thirdparty.mock import Mock
  40 from webkitpy import layout_tests
  41 from webkitpy.layout_tests.port import port_testcase
  42
  43 from webkitpy import layout_tests
  44 from webkitpy.layout_tests import run_webkit_tests
  45 from webkitpy.layout_tests.controllers import manager
  46 from webkitpy.layout_tests.controllers.manager import interpret_test_failures,  Manager, natural_sort_key, test_key, TestRunInterruptedException, TestShard
  47 from webkitpy.layout_tests.models import result_summary
  48 from webkitpy.layout_tests.models import test_expectations
  49 from webkitpy.layout_tests.models import test_failures
  50 from webkitpy.layout_tests.models import test_results
  51 from webkitpy.layout_tests.models.result_summary import ResultSummary
  52 from webkitpy.layout_tests.models.test_expectations import TestExpectations
  53 from webkitpy.layout_tests.models.test_results import TestResult
  54 from webkitpy.layout_tests.views import printing
  55 from webkitpy.tool.mocktool import MockOptions
  56 from webkitpy.common.system.executive_mock import MockExecutive
  57 from webkitpy.common.host_mock import MockHost
  58
  59
  60 class ManagerWrapper(Manager):
  61     def _get_test_input_for_file(self, test_file):
  62         return test_file
  63
  64
  65 class ShardingTests(unittest.TestCase):
  66     test_list = [
  67         "http/tests/websocket/tests/unicode.htm",
  68         "animations/keyframes.html",
  69         "http/tests/security/view-source-no-refresh.html",
  70         "http/tests/websocket/tests/websocket-protocol-ignored.html",
  71         "fast/css/display-none-inline-style-change-crash.html",
  72         "http/tests/xmlhttprequest/supported-xml-content-types.html",
  73         "dom/html/level2/html/HTMLAnchorElement03.html",
  74         "ietestcenter/Javascript/11.1.5_4-4-c-1.html",
  75         "dom/html/level2/html/HTMLAnchorElement06.html",
  76     ]
  77
  78     def get_shards(self, num_workers, fully_parallel, test_list=None, max_locked_shards=None):
  79         test_list = test_list or self.test_list
  80         host = MockHost()
  81         port = host.port_factory.get(port_name='test')
  82         port._filesystem = MockFileSystem()
  83         options = MockOptions(max_locked_shards=max_locked_shards)
  84         self.manager = ManagerWrapper(port=port, options=options, printer=Mock())
  85         return self.manager._shard_tests(test_list, num_workers, fully_parallel)
  86
  87     def test_shard_by_dir(self):
  88         locked, unlocked = self.get_shards(num_workers=2, fully_parallel=False)
  89
  90         # Note that although there are tests in multiple dirs that need locks,
  91         # they are crammed into a single shard in order to reduce the # of
  92         # workers hitting the server at once.
  93         self.assertEquals(locked,
  94             [TestShard('locked_shard_1',
  95               ['http/tests/security/view-source-no-refresh.html',
  96                'http/tests/websocket/tests/unicode.htm',
  97                'http/tests/websocket/tests/websocket-protocol-ignored.html',
  98                'http/tests/xmlhttprequest/supported-xml-content-types.html'])])
  99         self.assertEquals(unlocked,
 100             [TestShard('animations',
 101                        ['animations/keyframes.html']),
 102              TestShard('dom/html/level2/html',
 103                        ['dom/html/level2/html/HTMLAnchorElement03.html',
 104                         'dom/html/level2/html/HTMLAnchorElement06.html']),
 105              TestShard('fast/css',
 106                        ['fast/css/display-none-inline-style-change-crash.html']),
 107              TestShard('ietestcenter/Javascript',
 108                        ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])])
 109
 110     def test_shard_every_file(self):
 111         locked, unlocked = self.get_shards(num_workers=2, fully_parallel=True)
 112         self.assertEquals(locked,
 113             [TestShard('.', ['http/tests/websocket/tests/unicode.htm']),
 114              TestShard('.', ['http/tests/security/view-source-no-refresh.html']),
 115              TestShard('.', ['http/tests/websocket/tests/websocket-protocol-ignored.html']),
 116              TestShard('.', ['http/tests/xmlhttprequest/supported-xml-content-types.html'])])
 117         self.assertEquals(unlocked,
 118             [TestShard('.', ['animations/keyframes.html']),
 119              TestShard('.', ['fast/css/display-none-inline-style-change-crash.html']),
 120              TestShard('.', ['dom/html/level2/html/HTMLAnchorElement03.html']),
 121              TestShard('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']),
 122              TestShard('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])
 123
 124     def test_shard_in_two(self):
 125         locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False)
 126         self.assertEquals(locked,
 127             [TestShard('locked_tests',
 128                        ['http/tests/websocket/tests/unicode.htm',
 129                         'http/tests/security/view-source-no-refresh.html',
 130                         'http/tests/websocket/tests/websocket-protocol-ignored.html',
 131                         'http/tests/xmlhttprequest/supported-xml-content-types.html'])])
 132         self.assertEquals(unlocked,
 133             [TestShard('unlocked_tests',
 134                        ['animations/keyframes.html',
 135                         'fast/css/display-none-inline-style-change-crash.html',
 136                         'dom/html/level2/html/HTMLAnchorElement03.html',
 137                         'ietestcenter/Javascript/11.1.5_4-4-c-1.html',
 138                         'dom/html/level2/html/HTMLAnchorElement06.html'])])
 139
 140     def test_shard_in_two_has_no_locked_shards(self):
 141         locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
 142              test_list=['animations/keyframe.html'])
 143         self.assertEquals(len(locked), 0)
 144         self.assertEquals(len(unlocked), 1)
 145
 146     def test_shard_in_two_has_no_unlocked_shards(self):
 147         locked, unlocked = self.get_shards(num_workers=1, fully_parallel=False,
 148              test_list=['http/tests/webcoket/tests/unicode.htm'])
 149         self.assertEquals(len(locked), 1)
 150         self.assertEquals(len(unlocked), 0)
 151
 152     def test_multiple_locked_shards(self):
 153         locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False, max_locked_shards=2)
 154         self.assertEqual(locked,
 155             [TestShard('locked_shard_1',
 156                        ['http/tests/security/view-source-no-refresh.html',
 157                         'http/tests/websocket/tests/unicode.htm',
 158                         'http/tests/websocket/tests/websocket-protocol-ignored.html']),
 159              TestShard('locked_shard_2',
 160                         ['http/tests/xmlhttprequest/supported-xml-content-types.html'])])
 161
 162         locked, unlocked = self.get_shards(num_workers=4, fully_parallel=False)
 163         self.assertEquals(locked,
 164             [TestShard('locked_shard_1',
 165                        ['http/tests/security/view-source-no-refresh.html',
 166                         'http/tests/websocket/tests/unicode.htm',
 167                         'http/tests/websocket/tests/websocket-protocol-ignored.html',
 168                         'http/tests/xmlhttprequest/supported-xml-content-types.html'])])
 169
 170
 171 class ManagerTest(unittest.TestCase):
 172     def get_options(self):
 173         return MockOptions(pixel_tests=False, new_baseline=False, time_out_ms=6000, slow_time_out_ms=30000, worker_model='inline')
 174
 175     def get_printer(self):
 176         class FakePrinter(object):
 177             def __init__(self):
 178                 self.output = []
 179
 180             def print_config(self, msg):
 181                 self.output.append(msg)
 182
 183         return FakePrinter()
 184
 185     def test_fallback_path_in_config(self):
 186         options = self.get_options()
 187         host = MockHost()
 188         port = host.port_factory.get('test-mac-leopard', options=options)
 189         printer = self.get_printer()
 190         manager = Manager(port, options, printer)
 191         manager.print_config()
 192         self.assertTrue('Baseline search path: test-mac-leopard -> test-mac-snowleopard -> generic' in printer.output)
 193
 194     def test_http_locking(tester):
 195         class LockCheckingManager(Manager):
 196             def __init__(self, port, options, printer):
 197                 super(LockCheckingManager, self).__init__(port, options, printer)
 198                 self._finished_list_called = False
 199
 200             def handle_finished_list(self, source, list_name, num_tests, elapsed_time):
 201                 if not self._finished_list_called:
 202                     tester.assertEquals(list_name, 'locked_tests')
 203                     tester.assertTrue(self._remaining_locked_shards)
 204                     tester.assertTrue(self._has_http_lock)
 205
 206                 super(LockCheckingManager, self).handle_finished_list(source, list_name, num_tests, elapsed_time)
 207
 208                 if not self._finished_list_called:
 209                     tester.assertEquals(self._remaining_locked_shards, [])
 210                     tester.assertFalse(self._has_http_lock)
 211                     self._finished_list_called = True
 212
 213         options, args = run_webkit_tests.parse_args(['--platform=test', '--print=nothing', 'http/tests/passes', 'passes'])
 214         host = MockHost()
 215         port = host.port_factory.get(port_name=options.platform, options=options)
 216         run_webkit_tests._set_up_derived_options(port, options)
 217         printer = printing.Printer(port, options, StringIO.StringIO(), StringIO.StringIO())
 218         manager = LockCheckingManager(port, options, printer)
 219         manager.collect_tests(args)
 220         manager.parse_expectations()
 221         num_unexpected_results = manager.run()
 222         printer.cleanup()
 223         tester.assertEquals(num_unexpected_results, 0)
 224
 225     def test_interrupt_if_at_failure_limits(self):
 226         port = Mock()  # FIXME: This should be a tighter mock.
 227         port.TEST_PATH_SEPARATOR = '/'
 228         port._filesystem = MockFileSystem()
 229         manager = Manager(port=port, options=MockOptions(), printer=Mock())
 230
 231         manager._options = MockOptions(exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None)
 232         result_summary = ResultSummary(expectations=Mock(), test_files=[])
 233         result_summary.unexpected_failures = 100
 234         result_summary.unexpected_crashes = 50
 235         result_summary.unexpected_timeouts = 50
 236         # No exception when the exit_after* options are None.
 237         manager._interrupt_if_at_failure_limits(result_summary)
 238
 239         # No exception when we haven't hit the limit yet.
 240         manager._options.exit_after_n_failures = 101
 241         manager._options.exit_after_n_crashes_or_timeouts = 101
 242         manager._interrupt_if_at_failure_limits(result_summary)
 243
 244         # Interrupt if we've exceeded either limit:
 245         manager._options.exit_after_n_crashes_or_timeouts = 10
 246         self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary)
 247
 248         manager._options.exit_after_n_crashes_or_timeouts = None
 249         manager._options.exit_after_n_failures = 10
 250         exception = self.assertRaises(TestRunInterruptedException, manager._interrupt_if_at_failure_limits, result_summary)
 251
 252     def test_update_summary_with_result(self):
 253         host = MockHost()
 254         port = host.port_factory.get('test-win-xp')
 255         test = 'failures/expected/reftest.html'
 256         expectations = TestExpectations(port, tests=[test],
 257              expectations='WONTFIX : failures/expected/reftest.html = IMAGE',
 258              test_config=port.test_configuration())
 259         # Reftests expected to be image mismatch should be respected when pixel_tests=False.
 260         manager = Manager(port=port, options=MockOptions(pixel_tests=False, exit_after_n_failures=None, exit_after_n_crashes_or_timeouts=None), printer=Mock())
 261         manager._expectations = expectations
 262         result_summary = ResultSummary(expectations=expectations, test_files=[test])
 263         result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()])
 264         manager._update_summary_with_result(result_summary, result)
 265         self.assertEquals(1, result_summary.expected)
 266         self.assertEquals(0, result_summary.unexpected)
 267
 268     def test_needs_servers(self):
 269         def get_manager_with_tests(test_names):
 270             port = Mock()  # FIXME: Use a tighter mock.
 271             port.TEST_PATH_SEPARATOR = '/'
 272             manager = Manager(port, options=MockOptions(http=True), printer=Mock())
 273             manager._test_files = set(test_names)
 274             manager._test_files_list = test_names
 275             return manager
 276
 277         manager = get_manager_with_tests(['fast/html'])
 278         self.assertFalse(manager.needs_servers())
 279
 280         manager = get_manager_with_tests(['http/tests/misc'])
 281         self.assertTrue(manager.needs_servers())
 282
 283     def integration_test_needs_servers(self):
 284         def get_manager_with_tests(test_names):
 285             host = MockHost()
 286             port = host.port_factory.get()
 287             manager = Manager(port, options=MockOptions(test_list=None, http=True), printer=Mock())
 288             manager.collect_tests(test_names)
 289             return manager
 290
 291         manager = get_manager_with_tests(['fast/html'])
 292         self.assertFalse(manager.needs_servers())
 293
 294         manager = get_manager_with_tests(['http/tests/mime'])
 295         self.assertTrue(manager.needs_servers())
 296
 297         if sys.platform == 'win32':
 298             manager = get_manager_with_tests(['fast\\html'])
 299             self.assertFalse(manager.needs_servers())
 300
 301             manager = get_manager_with_tests(['http\\tests\\mime'])
 302             self.assertTrue(manager.needs_servers())
 303
 304
 305 class NaturalCompareTest(unittest.TestCase):
 306     def assert_cmp(self, x, y, result):
 307         self.assertEquals(cmp(natural_sort_key(x), natural_sort_key(y)), result)
 308
 309     def test_natural_compare(self):
 310         self.assert_cmp('a', 'a', 0)
 311         self.assert_cmp('ab', 'a', 1)
 312         self.assert_cmp('a', 'ab', -1)
 313         self.assert_cmp('', '', 0)
 314         self.assert_cmp('', 'ab', -1)
 315         self.assert_cmp('1', '2', -1)
 316         self.assert_cmp('2', '1', 1)
 317         self.assert_cmp('1', '10', -1)
 318         self.assert_cmp('2', '10', -1)
 319         self.assert_cmp('foo_1.html', 'foo_2.html', -1)
 320         self.assert_cmp('foo_1.1.html', 'foo_2.html', -1)
 321         self.assert_cmp('foo_1.html', 'foo_10.html', -1)
 322         self.assert_cmp('foo_2.html', 'foo_10.html', -1)
 323         self.assert_cmp('foo_23.html', 'foo_10.html', 1)
 324         self.assert_cmp('foo_23.html', 'foo_100.html', -1)
 325
 326
 327 class KeyCompareTest(unittest.TestCase):
 328     def setUp(self):
 329         host = MockHost()
 330         self.port = host.port_factory.get('test')
 331
 332     def assert_cmp(self, x, y, result):
 333         self.assertEquals(cmp(test_key(self.port, x), test_key(self.port, y)), result)
 334
 335     def test_test_key(self):
 336         self.assert_cmp('/a', '/a', 0)
 337         self.assert_cmp('/a', '/b', -1)
 338         self.assert_cmp('/a2', '/a10', -1)
 339         self.assert_cmp('/a2/foo', '/a10/foo', -1)
 340         self.assert_cmp('/a/foo11', '/a/foo2', 1)
 341         self.assert_cmp('/ab', '/a/a/b', -1)
 342         self.assert_cmp('/a/a/b', '/ab', 1)
 343         self.assert_cmp('/foo-bar/baz', '/foo/baz', -1)
 344
 345
 346 class ResultSummaryTest(unittest.TestCase):
 347
 348     def setUp(self):
 349         host = MockHost()
 350         self.port = host.port_factory.get(port_name='test')
 351
 352     def test_interpret_test_failures(self):
 353         test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
 354             [test_failures.FailureReftestMismatch(self.port.abspath_for_test('foo/reftest-expected.html'))])
 355         self.assertTrue('is_reftest' in test_dict)
 356         self.assertFalse('is_mismatch_reftest' in test_dict)
 357
 358         test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
 359             [test_failures.FailureReftestMismatch(self.port.abspath_for_test('foo/common.html'))])
 360         self.assertTrue('is_reftest' in test_dict)
 361         self.assertFalse('is_mismatch_reftest' in test_dict)
 362         self.assertEqual(test_dict['ref_file'], 'foo/common.html')
 363
 364         test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
 365             [test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.html'))])
 366         self.assertFalse('is_reftest' in test_dict)
 367         self.assertTrue(test_dict['is_mismatch_reftest'])
 368
 369         test_dict = interpret_test_failures(self.port, 'foo/reftest.html',
 370             [test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/common.html'))])
 371         self.assertFalse('is_reftest' in test_dict)
 372         self.assertTrue(test_dict['is_mismatch_reftest'])
 373         self.assertEqual(test_dict['ref_file'], 'foo/common.html')
 374
 375     def get_result(self, test_name, result_type=test_expectations.PASS, run_time=0):
 376         failures = []
 377         if result_type == test_expectations.TIMEOUT:
 378             failures = [test_failures.FailureTimeout()]
 379         elif result_type == test_expectations.CRASH:
 380             failures = [test_failures.FailureCrash()]
 381         return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)
 382
 383     def get_result_summary(self, port, test_names, expectations_str):
 384         expectations = test_expectations.TestExpectations(port, test_names, expectations_str, port.test_configuration(), is_lint_mode=False)
 385         return test_names, result_summary.ResultSummary(expectations, test_names), expectations
 386
 387     # FIXME: Use this to test more of summarize_results. This was moved from printing_unittest.py.
 388     def summarized_results(self, port, expected, passing, flaky, extra_tests=[], extra_expectations=None):
 389         tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html', 'failures/expected/wontfix.html']
 390         if extra_tests:
 391             tests.extend(extra_tests)
 392
 393         expectations = ''
 394         if extra_expectations:
 395             expectations += extra_expectations
 396
 397         paths, rs, exp = self.get_result_summary(port, tests, expectations)
 398         if expected:
 399             rs.add(self.get_result('passes/text.html', test_expectations.PASS), expected)
 400             rs.add(self.get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected)
 401             rs.add(self.get_result('failures/expected/crash.html', test_expectations.CRASH), expected)
 402         elif passing:
 403             rs.add(self.get_result('passes/text.html'), expected)
 404             rs.add(self.get_result('failures/expected/timeout.html'), expected)
 405             rs.add(self.get_result('failures/expected/crash.html'), expected)
 406         else:
 407             rs.add(self.get_result('passes/text.html', test_expectations.TIMEOUT), expected)
 408             rs.add(self.get_result('failures/expected/timeout.html', test_expectations.CRASH), expected)
 409             rs.add(self.get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected)
 410
 411         for test in extra_tests:
 412             rs.add(self.get_result(test, test_expectations.CRASH), expected)
 413
 414         retry = rs
 415         if flaky:
 416             paths, retry, exp = self.get_result_summary(port, tests, expectations)
 417             retry.add(self.get_result('passes/text.html'), True)
 418             retry.add(self.get_result('failures/expected/timeout.html'), True)
 419             retry.add(self.get_result('failures/expected/crash.html'), True)
 420         unexpected_results = manager.summarize_results(port, exp, rs, retry, test_timings={}, only_unexpected=True, interrupted=False)
 421         expected_results = manager.summarize_results(port, exp, rs, retry, test_timings={}, only_unexpected=False, interrupted=False)
 422         return expected_results, unexpected_results
 423
 424     def test_no_svn_revision(self):
 425         host = MockHost()
 426         port = host.port_factory.get('test')
 427         expected_results, unexpected_results = self.summarized_results(port, expected=False, passing=False, flaky=False)
 428         self.assertTrue('revision' not in unexpected_results)
 429
 430     def test_svn_revision(self):
 431         host = MockHost()
 432         port = host.port_factory.get('test')
 433         port._options.builder_name = 'dummy builder'
 434         expected_results, unexpected_results = self.summarized_results(port, expected=False, passing=False, flaky=False)
 435         self.assertTrue('revision' in unexpected_results)
 436
 437     def test_summarized_results_wontfix(self):
 438         host = MockHost()
 439         port = host.port_factory.get('test')
 440         port._options.builder_name = 'dummy builder'
 441         port._filesystem.write_text_file(port._filesystem.join(port.layout_tests_dir(), "failures/expected/wontfix.html"), "Dummy test contents")
 442         expected_results, unexpected_results = self.summarized_results(port, expected=False, passing=False, flaky=False, extra_tests=['failures/expected/wontfix.html'], extra_expectations='BUGX WONTFIX : failures/expected/wontfix.html = FAIL\n')
 443         self.assertTrue(expected_results['tests']['failures']['expected']['wontfix.html']['wontfix'])
 444
 445 if __name__ == '__main__':
 446     port_testcase.main()