Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner_unittest.py

   1 # Copyright (C) 2012 Google Inc. All rights reserved.
   2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 import unittest
  31
  32 from webkitpy.common.host_mock import MockHost
  33 from webkitpy.common.system.systemhost_mock import MockSystemHost
  34 from webkitpy.layout_tests import run_webkit_tests
  35 from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner, Sharder, TestRunInterruptedException
  36 from webkitpy.layout_tests.models import test_expectations
  37 from webkitpy.layout_tests.models import test_failures
  38 from webkitpy.layout_tests.models.test_run_results import TestRunResults
  39 from webkitpy.layout_tests.models.test_input import TestInput
  40 from webkitpy.layout_tests.models.test_results import TestResult
  41 from webkitpy.port.test import TestPort
  42
  43
  44 TestExpectations = test_expectations.TestExpectations
  45
  46
  47 class FakePrinter(object):
  48     num_started = 0
  49     num_tests = 0
  50
  51     def print_expected(self, run_results, get_tests_with_result_type):
  52         pass
  53
  54     def print_workers_and_shards(self, num_workers, num_shards):
  55         pass
  56
  57     def print_started_test(self, test_name):
  58         pass
  59
  60     def print_finished_test(self, result, expected, exp_str, got_str):
  61         pass
  62
  63     def write(self, msg):
  64         pass
  65
  66     def write_update(self, msg):
  67         pass
  68
  69     def flush(self):
  70         pass
  71
  72
  73 class LayoutTestRunnerTests(unittest.TestCase):
  74     def _runner(self, port=None):
  75         # FIXME: we shouldn't have to use run_webkit_tests.py to get the options we need.
  76         options = run_webkit_tests.parse_args(['--platform', 'test-mac-snowleopard'])[0]
  77         options.child_processes = '1'
  78
  79         host = MockHost()
  80         port = port or host.port_factory.get(options.platform, options=options)
  81         return LayoutTestRunner(options, port, FakePrinter(), port.results_directory(), lambda test_name: False)
  82
  83     def _run_tests(self, runner, tests):
  84         test_inputs = [TestInput(test, 6000) for test in tests]
  85         expectations = TestExpectations(runner._port, tests)
  86         expectations.parse_all_expectations()
  87         runner.run_tests(expectations, test_inputs, set(),
  88             num_workers=1, needs_http=any('http' in test for test in tests), needs_websockets=any(['websocket' in test for test in tests]), needs_web_platform_test_server=any(['imported/w3c' in test for test in tests]), retrying=False)
  89
  90     def test_interrupt_if_at_failure_limits(self):
  91         runner = self._runner()
  92         runner._options.exit_after_n_failures = None
  93         runner._options.exit_after_n_crashes_or_times = None
  94         test_names = ['passes/text.html', 'passes/image.html']
  95         runner._test_inputs = [TestInput(test_name, 6000) for test_name in test_names]
  96
  97         expectations = TestExpectations(runner._port, test_names)
  98         expectations.parse_all_expectations()
  99         run_results = TestRunResults(expectations, len(test_names))
 100         run_results.unexpected_failures = 100
 101         run_results.unexpected_crashes = 50
 102         run_results.unexpected_timeouts = 50
 103         # No exception when the exit_after* options are None.
 104         runner._interrupt_if_at_failure_limits(run_results)
 105
 106         # No exception when we haven't hit the limit yet.
 107         runner._options.exit_after_n_failures = 101
 108         runner._options.exit_after_n_crashes_or_timeouts = 101
 109         runner._interrupt_if_at_failure_limits(run_results)
 110
 111         # Interrupt if we've exceeded either limit:
 112         runner._options.exit_after_n_crashes_or_timeouts = 10
 113         self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results)
 114         self.assertEqual(run_results.results_by_name['passes/text.html'].type, test_expectations.SKIP)
 115         self.assertEqual(run_results.results_by_name['passes/image.html'].type, test_expectations.SKIP)
 116
 117         runner._options.exit_after_n_crashes_or_timeouts = None
 118         runner._options.exit_after_n_failures = 10
 119         exception = self.assertRaises(TestRunInterruptedException, runner._interrupt_if_at_failure_limits, run_results)
 120
 121     def test_update_summary_with_result(self):
 122         # Reftests expected to be image mismatch should be respected when pixel_tests=False.
 123         runner = self._runner()
 124         runner._options.pixel_tests = False
 125         runner._options.world_leaks = False
 126         test = 'failures/expected/reftest.html'
 127         leak_test = 'failures/expected/leak.html'
 128         expectations = TestExpectations(runner._port, tests=[test, leak_test])
 129         expectations.parse_all_expectations()
 130         runner._expectations = expectations
 131
 132         run_results = TestRunResults(expectations, 1)
 133         result = TestResult(test_name=test, failures=[test_failures.FailureReftestMismatchDidNotOccur()], reftest_type=['!='])
 134         runner._update_summary_with_result(run_results, result)
 135         self.assertEqual(1, run_results.expected)
 136         self.assertEqual(0, run_results.unexpected)
 137
 138         run_results = TestRunResults(expectations, 1)
 139         result = TestResult(test_name=test, failures=[], reftest_type=['=='])
 140         runner._update_summary_with_result(run_results, result)
 141         self.assertEqual(0, run_results.expected)
 142         self.assertEqual(1, run_results.unexpected)
 143
 144         run_results = TestRunResults(expectations, 1)
 145         result = TestResult(test_name=leak_test, failures=[])
 146         runner._update_summary_with_result(run_results, result)
 147         self.assertEqual(1, run_results.expected)
 148         self.assertEqual(0, run_results.unexpected)
 149
 150     def test_servers_started(self):
 151
 152         def start_http_server():
 153             self.http_started = True
 154
 155         def start_websocket_server():
 156             self.websocket_started = True
 157
 158         def start_web_platform_test_server():
 159             self.web_platform_test_server_started = True
 160
 161         def stop_http_server():
 162             self.http_stopped = True
 163
 164         def stop_websocket_server():
 165             self.websocket_stopped = True
 166
 167         def stop_web_platform_test_server():
 168             self.web_platform_test_server_stopped = True
 169
 170         def is_http_server_running():
 171             return self.http_started and not self.http_stopped
 172
 173         def is_websocket_server_running():
 174             return self.websocket_started and not self.websocket_stopped
 175
 176         def is_wpt_server_running():
 177             return self.websocket_started and not self.web_platform_test_server_stopped
 178
 179         host = MockHost()
 180         port = host.port_factory.get('test-mac-leopard')
 181         port.start_http_server = start_http_server
 182         port.start_websocket_server = start_websocket_server
 183         port.start_web_platform_test_server = start_web_platform_test_server
 184         port.stop_http_server = stop_http_server
 185         port.stop_websocket_server = stop_websocket_server
 186         port.stop_web_platform_test_server = stop_web_platform_test_server
 187         port.is_http_server_running = is_http_server_running
 188         port.is_websocket_server_running = is_websocket_server_running
 189         port.is_wpt_server_running = is_wpt_server_running
 190
 191         self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
 192         self.web_platform_test_server_started = self.web_platform_test_server_stopped = False
 193         runner = self._runner(port=port)
 194         runner._needs_http = True
 195         runner._needs_websockets = False
 196         runner._needs_web_platform_test_server = False
 197         runner.start_servers()
 198         self.assertEqual(self.http_started, True)
 199         self.assertEqual(self.websocket_started, False)
 200         self.assertEqual(self.web_platform_test_server_started, False)
 201         runner.stop_servers()
 202         self.assertEqual(self.http_stopped, True)
 203         self.assertEqual(self.websocket_stopped, False)
 204         self.assertEqual(self.web_platform_test_server_stopped, False)
 205
 206         self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
 207         self.web_platform_test_server_started = self.web_platform_test_server_stopped = False
 208         runner._needs_http = True
 209         runner._needs_websockets = True
 210         runner._needs_web_platform_test_server = False
 211         runner.start_servers()
 212         self.assertEqual(self.http_started, True)
 213         self.assertEqual(self.websocket_started, True)
 214         self.assertEqual(self.web_platform_test_server_started, False)
 215         runner.stop_servers()
 216         self.assertEqual(self.http_stopped, True)
 217         self.assertEqual(self.websocket_stopped, True)
 218         self.assertEqual(self.web_platform_test_server_stopped, False)
 219
 220         self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
 221         self.web_platform_test_server_started = self.web_platform_test_server_stopped = False
 222         runner._needs_http = False
 223         runner._needs_websockets = False
 224         runner._needs_web_platform_test_server = True
 225         runner.start_servers()
 226         self.assertEqual(self.http_started, False)
 227         self.assertEqual(self.websocket_started, False)
 228         self.assertEqual(self.web_platform_test_server_started, True)
 229         runner.stop_servers()
 230         self.assertEqual(self.http_stopped, False)
 231         self.assertEqual(self.websocket_stopped, False)
 232         self.assertEqual(self.web_platform_test_server_stopped, True)
 233
 234         self.http_started = self.http_stopped = self.websocket_started = self.websocket_stopped = False
 235         self.web_platform_test_server_started = self.web_platform_test_server_stopped = False
 236         runner._needs_http = False
 237         runner._needs_websockets = False
 238         runner._needs_web_platform_test_server = False
 239         runner.start_servers()
 240         self.assertEqual(self.http_started, False)
 241         self.assertEqual(self.websocket_started, False)
 242         self.assertEqual(self.web_platform_test_server_started, False)
 243         runner.stop_servers()
 244         self.assertEqual(self.http_stopped, False)
 245         self.assertEqual(self.websocket_stopped, False)
 246         self.assertEqual(self.web_platform_test_server_stopped, False)
 247
 248
 249 class SharderTests(unittest.TestCase):
 250
 251     test_list = [
 252         "http/tests/websocket/tests/unicode.htm",
 253         "animations/keyframes.html",
 254         "http/tests/security/view-source-no-refresh.html",
 255         "http/tests/websocket/tests/websocket-protocol-ignored.html",
 256         "fast/css/display-none-inline-style-change-crash.html",
 257         "http/tests/xmlhttprequest/supported-xml-content-types.html",
 258         "dom/html/level2/html/HTMLAnchorElement03.html",
 259         "ietestcenter/Javascript/11.1.5_4-4-c-1.html",
 260         "dom/html/level2/html/HTMLAnchorElement06.html",
 261     ]
 262
 263     def get_test_input(self, test_file):
 264         return TestInput(test_file, needs_servers=(test_file.startswith('http')))
 265
 266     def get_shards(self, num_workers, fully_parallel, test_list=None):
 267         port = TestPort(MockSystemHost())
 268         self.sharder = Sharder(port.split_test)
 269         test_list = test_list or self.test_list
 270         return self.sharder.shard_tests([self.get_test_input(test) for test in test_list], num_workers, fully_parallel)
 271
 272     def assert_shards(self, actual_shards, expected_shard_names):
 273         self.assertEqual(len(actual_shards), len(expected_shard_names))
 274         for i, shard in enumerate(actual_shards):
 275             expected_shard_name, expected_test_names = expected_shard_names[i]
 276             self.assertEqual(shard.name, expected_shard_name)
 277             self.assertEqual([test_input.test_name for test_input in shard.test_inputs],
 278                               expected_test_names)
 279
 280     def test_shard_by_dir(self):
 281         result = self.get_shards(num_workers=2, fully_parallel=False)
 282
 283         self.assert_shards(result,
 284             [('animations', ['animations/keyframes.html']),
 285              ('dom/html/level2/html', ['dom/html/level2/html/HTMLAnchorElement03.html',
 286                                       'dom/html/level2/html/HTMLAnchorElement06.html']),
 287              ('fast/css', ['fast/css/display-none-inline-style-change-crash.html']),
 288              ('http/tests/security', ['http/tests/security/view-source-no-refresh.html']),
 289              ('http/tests/websocket/tests', ['http/tests/websocket/tests/unicode.htm', 'http/tests/websocket/tests/websocket-protocol-ignored.html']),
 290              ('http/tests/xmlhttprequest', ['http/tests/xmlhttprequest/supported-xml-content-types.html']),
 291              ('ietestcenter/Javascript', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html'])])
 292
 293     def test_shard_every_file(self):
 294         result = self.get_shards(num_workers=2, fully_parallel=True)
 295         self.assert_shards(result,
 296             [('.', ['http/tests/websocket/tests/unicode.htm']),
 297              ('.', ['animations/keyframes.html']),
 298              ('.', ['http/tests/security/view-source-no-refresh.html']),
 299              ('.', ['http/tests/websocket/tests/websocket-protocol-ignored.html']),
 300              ('.', ['fast/css/display-none-inline-style-change-crash.html']),
 301              ('.', ['http/tests/xmlhttprequest/supported-xml-content-types.html']),
 302              ('.', ['dom/html/level2/html/HTMLAnchorElement03.html']),
 303              ('.', ['ietestcenter/Javascript/11.1.5_4-4-c-1.html']),
 304              ('.', ['dom/html/level2/html/HTMLAnchorElement06.html'])])