WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py

   1 #!/usr/bin/python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """Unit tests for run_webkit_tests."""
  32
  33 import codecs
  34 import itertools
  35 import logging
  36 import os
  37 import Queue
  38 import shutil
  39 import sys
  40 import tempfile
  41 import thread
  42 import time
  43 import threading
  44 import unittest
  45
  46 from webkitpy.common import array_stream
  47 from webkitpy.common.system import outputcapture
  48 from webkitpy.common.system import user
  49 from webkitpy.layout_tests import port
  50 from webkitpy.layout_tests.port import test
  51 from webkitpy.layout_tests import run_webkit_tests
  52 from webkitpy.layout_tests.layout_package import dump_render_tree_thread
  53 from webkitpy.layout_tests.port.test import TestPort, TestDriver
  54 from webkitpy.python24.versioning import compare_version
  55 from webkitpy.test.skip import skip_if
  56
  57 from webkitpy.thirdparty.mock import Mock
  58
  59
  60 class MockUser():
  61     def __init__(self):
  62         self.url = None
  63
  64     def open_url(self, url):
  65         self.url = url
  66
  67
  68 def passing_run(extra_args=None, port_obj=None, record_results=False,
  69                 tests_included=False):
  70     extra_args = extra_args or []
  71     args = ['--print', 'nothing']
  72     if not '--platform' in extra_args:
  73         args.extend(['--platform', 'test'])
  74     if not record_results:
  75         args.append('--no-record-results')
  76     if not '--child-processes' in extra_args:
  77         args.extend(['--worker-model', 'inline'])
  78     args.extend(extra_args)
  79     if not tests_included:
  80         # We use the glob to test that globbing works.
  81         args.extend(['passes',
  82                      'http/tests',
  83                      'websocket/tests',
  84                      'failures/expected/*'])
  85     options, parsed_args = run_webkit_tests.parse_args(args)
  86     if not port_obj:
  87         port_obj = port.get(port_name=options.platform, options=options,
  88                             user=MockUser())
  89     res = run_webkit_tests.run(port_obj, options, parsed_args)
  90     return res == 0
  91
  92
  93 def logging_run(extra_args=None, port_obj=None, tests_included=False):
  94     extra_args = extra_args or []
  95     args = ['--no-record-results']
  96     if not '--platform' in extra_args:
  97         args.extend(['--platform', 'test'])
  98     if not '--child-processes' in extra_args:
  99         args.extend(['--worker-model', 'inline'])
 100     args.extend(extra_args)
 101     if not tests_included:
 102         args.extend(['passes',
 103                      'http/tests',
 104                      'websocket/tests',
 105                      'failures/expected/*'])
 106
 107     oc = outputcapture.OutputCapture()
 108     try:
 109         oc.capture_output()
 110         options, parsed_args = run_webkit_tests.parse_args(args)
 111         user = MockUser()
 112         if not port_obj:
 113             port_obj = port.get(port_name=options.platform, options=options,
 114                                 user=user)
 115         buildbot_output = array_stream.ArrayStream()
 116         regular_output = array_stream.ArrayStream()
 117         res = run_webkit_tests.run(port_obj, options, parsed_args,
 118                                    buildbot_output=buildbot_output,
 119                                    regular_output=regular_output)
 120     finally:
 121         oc.restore_output()
 122     return (res, buildbot_output, regular_output, user)
 123
 124
 125 def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False):
 126     extra_args = extra_args or []
 127     args = [
 128         '--print', 'nothing',
 129         '--platform', 'test',
 130         '--no-record-results',
 131         '--worker-model', 'inline']
 132     args.extend(extra_args)
 133     if not tests_included:
 134         # Not including http tests since they get run out of order (that
 135         # behavior has its own test, see test_get_test_file_queue)
 136         args.extend(['passes', 'failures'])
 137     options, parsed_args = run_webkit_tests.parse_args(args)
 138     user = MockUser()
 139
 140     test_batches = []
 141
 142     class RecordingTestDriver(TestDriver):
 143         def __init__(self, port, worker_number):
 144             TestDriver.__init__(self, port, worker_number)
 145             self._current_test_batch = None
 146
 147         def poll(self):
 148             # So that we don't create a new driver for every test
 149             return None
 150
 151         def stop(self):
 152             self._current_test_batch = None
 153
 154         def run_test(self, test_input):
 155             if self._current_test_batch is None:
 156                 self._current_test_batch = []
 157                 test_batches.append(self._current_test_batch)
 158             test_name = self._port.relative_test_filename(test_input.filename)
 159             self._current_test_batch.append(test_name)
 160             return TestDriver.run_test(self, test_input)
 161
 162     class RecordingTestPort(TestPort):
 163         def create_driver(self, worker_number):
 164             return RecordingTestDriver(self, worker_number)
 165
 166     recording_port = RecordingTestPort(options=options, user=user)
 167     logging_run(extra_args=args, port_obj=recording_port, tests_included=True)
 168
 169     if flatten_batches:
 170         return list(itertools.chain(*test_batches))
 171
 172     return test_batches
 173
 174 class MainTest(unittest.TestCase):
 175     def test_accelerated_compositing(self):
 176         # This just tests that we recognize the command line args
 177         self.assertTrue(passing_run(['--accelerated-compositing']))
 178         self.assertTrue(passing_run(['--no-accelerated-compositing']))
 179
 180     def test_accelerated_2d_canvas(self):
 181         # This just tests that we recognize the command line args
 182         self.assertTrue(passing_run(['--accelerated-2d-canvas']))
 183         self.assertTrue(passing_run(['--no-accelerated-2d-canvas']))
 184
 185     def test_basic(self):
 186         self.assertTrue(passing_run())
 187
 188     def test_batch_size(self):
 189         batch_tests_run = get_tests_run(['--batch-size', '2'])
 190         self.assertEquals(len(batch_tests_run), 9)
 191         for batch in batch_tests_run:
 192             self.assertTrue(len(batch) <= 2, '%s had too many tests' % ', '.join(batch))
 193
 194     def test_child_process_1(self):
 195         (res, buildbot_output, regular_output, user) = logging_run(
 196              ['--print', 'config', '--child-processes', '1'])
 197         self.assertTrue('Running one DumpRenderTree\n'
 198                         in regular_output.get())
 199
 200     def test_child_processes_2(self):
 201         (res, buildbot_output, regular_output, user) = logging_run(
 202              ['--print', 'config', '--child-processes', '2'])
 203         self.assertTrue('Running 2 DumpRenderTrees in parallel\n'
 204                         in regular_output.get())
 205
 206     def test_dryrun(self):
 207         batch_tests_run = get_tests_run(['--dry-run'])
 208         self.assertEqual(batch_tests_run, [])
 209
 210         batch_tests_run = get_tests_run(['-n'])
 211         self.assertEqual(batch_tests_run, [])
 212
 213     def test_exception_raised(self):
 214         self.assertRaises(ValueError, logging_run,
 215             ['failures/expected/exception.html'], tests_included=True)
 216
 217     def test_full_results_html(self):
 218         # FIXME: verify html?
 219         self.assertTrue(passing_run(['--full-results-html']))
 220
 221     def test_help_printing(self):
 222         res, out, err, user = logging_run(['--help-printing'])
 223         self.assertEqual(res, 0)
 224         self.assertTrue(out.empty())
 225         self.assertFalse(err.empty())
 226
 227     def test_hung_thread(self):
 228         res, out, err, user = logging_run(['--run-singly', '--time-out-ms=50',
 229                                           'failures/expected/hang.html'],
 230                                           tests_included=True)
 231         self.assertEqual(res, 0)
 232         self.assertFalse(out.empty())
 233         self.assertFalse(err.empty())
 234
 235     def test_keyboard_interrupt(self):
 236         # Note that this also tests running a test marked as SKIP if
 237         # you specify it explicitly.
 238         self.assertRaises(KeyboardInterrupt, logging_run,
 239             ['failures/expected/keyboard.html'], tests_included=True)
 240
 241     def test_last_results(self):
 242         passing_run(['--clobber-old-results'], record_results=True)
 243         (res, buildbot_output, regular_output, user) = logging_run(
 244             ['--print-last-failures'])
 245         self.assertEqual(regular_output.get(), ['\n\n'])
 246         self.assertEqual(buildbot_output.get(), [])
 247
 248     def test_lint_test_files(self):
 249         # FIXME:  add errors?
 250         res, out, err, user = logging_run(['--lint-test-files'],
 251                                           tests_included=True)
 252         self.assertEqual(res, 0)
 253         self.assertTrue(out.empty())
 254         self.assertTrue(any(['lint succeeded' in msg for msg in err.get()]))
 255
 256     def test_no_tests_found(self):
 257         res, out, err, user = logging_run(['resources'], tests_included=True)
 258         self.assertEqual(res, -1)
 259         self.assertTrue(out.empty())
 260         self.assertTrue('No tests to run.\n' in err.get())
 261
 262     def test_no_tests_found_2(self):
 263         res, out, err, user = logging_run(['foo'], tests_included=True)
 264         self.assertEqual(res, -1)
 265         self.assertTrue(out.empty())
 266         self.assertTrue('No tests to run.\n' in err.get())
 267
 268     def test_randomize_order(self):
 269         # FIXME: verify order was shuffled
 270         self.assertTrue(passing_run(['--randomize-order']))
 271
 272     def test_run_chunk(self):
 273         # Test that we actually select the right chunk
 274         all_tests_run = get_tests_run(flatten_batches=True)
 275         chunk_tests_run = get_tests_run(['--run-chunk', '1:4'], flatten_batches=True)
 276         self.assertEquals(all_tests_run[4:8], chunk_tests_run)
 277
 278         # Test that we wrap around if the number of tests is not evenly divisible by the chunk size
 279         tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
 280         chunk_tests_run = get_tests_run(['--run-chunk', '1:3'] + tests_to_run, tests_included=True, flatten_batches=True)
 281         self.assertEquals(['passes/text.html', 'passes/error.html', 'passes/image.html'], chunk_tests_run)
 282
 283     def test_run_force(self):
 284         # This raises an exception because we run
 285         # failures/expected/exception.html, which is normally SKIPped.
 286         self.assertRaises(ValueError, logging_run, ['--force'])
 287
 288     def test_run_part(self):
 289         # Test that we actually select the right part
 290         tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
 291         tests_run = get_tests_run(['--run-part', '1:2'] + tests_to_run, tests_included=True, flatten_batches=True)
 292         self.assertEquals(['passes/error.html', 'passes/image.html'], tests_run)
 293
 294         # Test that we wrap around if the number of tests is not evenly divisible by the chunk size
 295         # (here we end up with 3 parts, each with 2 tests, and we only have 4 tests total, so the
 296         # last part repeats the first two tests).
 297         chunk_tests_run = get_tests_run(['--run-part', '3:3'] + tests_to_run, tests_included=True, flatten_batches=True)
 298         self.assertEquals(['passes/error.html', 'passes/image.html'], chunk_tests_run)
 299
 300     def test_run_singly(self):
 301         batch_tests_run = get_tests_run(['--run-singly'])
 302         self.assertEqual(len(batch_tests_run), 14)
 303         for batch in batch_tests_run:
 304             self.assertEquals(len(batch), 1, '%s had too many tests' % ', '.join(batch))
 305
 306     def test_single_file(self):
 307         tests_run = get_tests_run(['passes/text.html'], tests_included=True, flatten_batches=True)
 308         self.assertEquals(['passes/text.html'], tests_run)
 309
 310     def test_test_list(self):
 311         filename = tempfile.mktemp()
 312         tmpfile = file(filename, mode='w+')
 313         tmpfile.write('passes/text.html')
 314         tmpfile.close()
 315         tests_run = get_tests_run(['--test-list=%s' % filename], tests_included=True, flatten_batches=True)
 316         self.assertEquals(['passes/text.html'], tests_run)
 317         os.remove(filename)
 318         res, out, err, user = logging_run(['--test-list=%s' % filename],
 319                                           tests_included=True)
 320         self.assertEqual(res, -1)
 321         self.assertFalse(err.empty())
 322
 323     def test_unexpected_failures(self):
 324         # Run tests including the unexpected failures.
 325         self._url_opened = None
 326         res, out, err, user = logging_run(tests_included=True)
 327         self.assertEqual(res, 1)
 328         self.assertFalse(out.empty())
 329         self.assertFalse(err.empty())
 330         self.assertEqual(user.url, '/tmp/layout-test-results/results.html')
 331
 332     def test_results_directory_absolute(self):
 333         # We run a configuration that should fail, to generate output, then
 334         # look for what the output results url was.
 335
 336         tmpdir = tempfile.mkdtemp()
 337         res, out, err, user = logging_run(['--results-directory=' + tmpdir],
 338                                           tests_included=True)
 339         self.assertEqual(user.url, os.path.join(tmpdir, 'results.html'))
 340         shutil.rmtree(tmpdir, ignore_errors=True)
 341
 342     def test_results_directory_default(self):
 343         # We run a configuration that should fail, to generate output, then
 344         # look for what the output results url was.
 345
 346         # This is the default location.
 347         res, out, err, user = logging_run(tests_included=True)
 348         self.assertEqual(user.url, '/tmp/layout-test-results/results.html')
 349
 350     def test_results_directory_relative(self):
 351         # We run a configuration that should fail, to generate output, then
 352         # look for what the output results url was.
 353
 354         res, out, err, user = logging_run(['--results-directory=foo'],
 355                                           tests_included=True)
 356         self.assertEqual(user.url, '/tmp/foo/results.html')
 357
 358     def test_tolerance(self):
 359         class ImageDiffTestPort(TestPort):
 360             def diff_image(self, expected_contents, actual_contents,
 361                    diff_filename=None):
 362                 self.tolerance_used_for_diff_image = self._options.tolerance
 363                 return True
 364
 365         def get_port_for_run(args):
 366             options, parsed_args = run_webkit_tests.parse_args(args)
 367             test_port = ImageDiffTestPort(options=options, user=MockUser())
 368             passing_run(args, port_obj=test_port, tests_included=True)
 369             return test_port
 370
 371         base_args = ['--pixel-tests', 'failures/expected/*']
 372
 373         # If we pass in an explicit tolerance argument, then that will be used.
 374         test_port = get_port_for_run(base_args + ['--tolerance', '.1'])
 375         self.assertEqual(0.1, test_port.tolerance_used_for_diff_image)
 376         test_port = get_port_for_run(base_args + ['--tolerance', '0'])
 377         self.assertEqual(0, test_port.tolerance_used_for_diff_image)
 378
 379         # Otherwise the port's default tolerance behavior (including ignoring it)
 380         # should be used.
 381         test_port = get_port_for_run(base_args)
 382         self.assertEqual(None, test_port.tolerance_used_for_diff_image)
 383
 384     def test_worker_model__inline(self):
 385         self.assertTrue(passing_run(['--worker-model', 'inline']))
 386
 387     def test_worker_model__threads(self):
 388         self.assertTrue(passing_run(['--worker-model', 'threads']))
 389
 390     def test_worker_model__processes(self):
 391         self.assertRaises(ValueError, logging_run,
 392                           ['--worker-model', 'processes'])
 393
 394     def test_worker_model__unknown(self):
 395         self.assertRaises(ValueError, logging_run,
 396                           ['--worker-model', 'unknown'])
 397
 398     def test_worker_model__port_override(self):
 399         class OverridePort(test.TestPort):
 400             def default_worker_model(self):
 401                 return 'INVALID'
 402
 403         options, parsed_args = run_webkit_tests.parse_args(
 404             ['--print', 'nothing', '--platform', 'test', '--noshow-results'])
 405         port_obj = OverridePort(options=options)
 406         self.assertRaises(ValueError, run_webkit_tests.run, port_obj,
 407                           options, parsed_args)
 408
 409
 410 MainTest = skip_if(MainTest, sys.platform == 'cygwin' and compare_version(sys, '2.6')[0] < 0, 'new-run-webkit-tests tests hang on Cygwin Python 2.5.2')
 411
 412
 413
 414 def _mocked_open(original_open, file_list):
 415     def _wrapper(name, mode, encoding):
 416         if name.find("-expected.") != -1 and mode.find("w") != -1:
 417             # we don't want to actually write new baselines, so stub these out
 418             name.replace('\\', '/')
 419             file_list.append(name)
 420             return original_open(os.devnull, mode, encoding)
 421         return original_open(name, mode, encoding)
 422     return _wrapper
 423
 424
 425 class RebaselineTest(unittest.TestCase):
 426     def assertBaselines(self, file_list, file):
 427         "assert that the file_list contains the baselines."""
 428         for ext in [".txt", ".png", ".checksum"]:
 429             baseline = file + "-expected" + ext
 430             self.assertTrue(any(f.find(baseline) != -1 for f in file_list))
 431
 432     # FIXME: Add tests to ensure that we're *not* writing baselines when we're not
 433     # supposed to be.
 434
 435     def disabled_test_reset_results(self):
 436         # FIXME: This test is disabled until we can rewrite it to use a
 437         # mock filesystem.
 438         #
 439         # Test that we update expectations in place. If the expectation
 440         # is missing, update the expected generic location.
 441         file_list = []
 442         passing_run(['--pixel-tests',
 443                         '--reset-results',
 444                         'passes/image.html',
 445                         'failures/expected/missing_image.html'],
 446                         tests_included=True)
 447         self.assertEqual(len(file_list), 6)
 448         self.assertBaselines(file_list,
 449             "data/passes/image")
 450         self.assertBaselines(file_list,
 451             "data/failures/expected/missing_image")
 452
 453     def disabled_test_new_baseline(self):
 454         # FIXME: This test is disabled until we can rewrite it to use a
 455         # mock filesystem.
 456         #
 457         # Test that we update the platform expectations. If the expectation
 458         # is mssing, then create a new expectation in the platform dir.
 459         file_list = []
 460         original_open = codecs.open
 461         try:
 462             # Test that we update the platform expectations. If the expectation
 463             # is mssing, then create a new expectation in the platform dir.
 464             file_list = []
 465             codecs.open = _mocked_open(original_open, file_list)
 466             passing_run(['--pixel-tests',
 467                          '--new-baseline',
 468                          'passes/image.html',
 469                          'failures/expected/missing_image.html'],
 470                         tests_included=True)
 471             self.assertEqual(len(file_list), 6)
 472             self.assertBaselines(file_list,
 473                 "data/platform/test/passes/image")
 474             self.assertBaselines(file_list,
 475                 "data/platform/test/failures/expected/missing_image")
 476         finally:
 477             codecs.open = original_open
 478
 479
 480 class TestRunnerWrapper(run_webkit_tests.TestRunner):
 481     def _get_test_input_for_file(self, test_file):
 482         return test_file
 483
 484
 485 class TestRunnerTest(unittest.TestCase):
 486     def test_results_html(self):
 487         mock_port = Mock()
 488         mock_port.relative_test_filename = lambda name: name
 489         mock_port.filename_to_uri = lambda name: name
 490
 491         runner = run_webkit_tests.TestRunner(port=mock_port, options=Mock(),
 492                                              printer=Mock())
 493         expected_html = u"""<html>
 494   <head>
 495     <title>Layout Test Results (time)</title>
 496   </head>
 497   <body>
 498     <h2>Title (time)</h2>
 499         <p><a href='test_path'>test_path</a><br />
 500 </p>
 501 </body></html>
 502 """
 503         html = runner._results_html(["test_path"], {}, "Title", override_time="time")
 504         self.assertEqual(html, expected_html)
 505
 506     def test_shard_tests(self):
 507         # Test that _shard_tests in run_webkit_tests.TestRunner really
 508         # put the http tests first in the queue.
 509         runner = TestRunnerWrapper(port=Mock(), options=Mock(),
 510                                    printer=Mock())
 511
 512         test_list = [
 513           "LayoutTests/websocket/tests/unicode.htm",
 514           "LayoutTests/animations/keyframes.html",
 515           "LayoutTests/http/tests/security/view-source-no-refresh.html",
 516           "LayoutTests/websocket/tests/websocket-protocol-ignored.html",
 517           "LayoutTests/fast/css/display-none-inline-style-change-crash.html",
 518           "LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html",
 519           "LayoutTests/dom/html/level2/html/HTMLAnchorElement03.html",
 520           "LayoutTests/ietestcenter/Javascript/11.1.5_4-4-c-1.html",
 521           "LayoutTests/dom/html/level2/html/HTMLAnchorElement06.html",
 522         ]
 523
 524         expected_tests_to_http_lock = set([
 525           'LayoutTests/websocket/tests/unicode.htm',
 526           'LayoutTests/http/tests/security/view-source-no-refresh.html',
 527           'LayoutTests/websocket/tests/websocket-protocol-ignored.html',
 528           'LayoutTests/http/tests/xmlhttprequest/supported-xml-content-types.html',
 529         ])
 530
 531         # FIXME: Ideally the HTTP tests don't have to all be in one shard.
 532         single_thread_results = runner._shard_tests(test_list, False)
 533         multi_thread_results = runner._shard_tests(test_list, True)
 534
 535         self.assertEqual("tests_to_http_lock", single_thread_results[0][0])
 536         self.assertEqual(expected_tests_to_http_lock, set(single_thread_results[0][1]))
 537         self.assertEqual("tests_to_http_lock", multi_thread_results[0][0])
 538         self.assertEqual(expected_tests_to_http_lock, set(multi_thread_results[0][1]))
 539
 540
 541 class DryrunTest(unittest.TestCase):
 542     # FIXME: it's hard to know which platforms are safe to test; the
 543     # chromium platforms require a chromium checkout, and the mac platform
 544     # requires fcntl, so it can't be tested on win32, etc. There is
 545     # probably a better way of handling this.
 546     def test_darwin(self):
 547         if sys.platform != "darwin":
 548             return
 549
 550         self.assertTrue(passing_run(['--platform', 'test']))
 551         self.assertTrue(passing_run(['--platform', 'dryrun',
 552                                      'fast/html']))
 553         self.assertTrue(passing_run(['--platform', 'dryrun-mac',
 554                                      'fast/html']))
 555
 556     def test_test(self):
 557         self.assertTrue(passing_run(['--platform', 'dryrun-test',
 558                                            '--pixel-tests']))
 559
 560
 561 if __name__ == '__main__':
 562     unittest.main()