Tools/Scripts/webkitpy/port/test.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the Google name nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 import time
  30
  31 from webkitpy.port import Port, Driver, DriverOutput
  32 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
  33 from webkitpy.common.system.crashlogs import CrashLogs
  34 from webkitpy.common.version_name_map import PUBLIC_TABLE, VersionNameMap
  35
  36
  37 # This sets basic expectations for a test. Each individual expectation
  38 # can be overridden by a keyword argument in TestList.add().
  39 class TestInstance(object):
  40     def __init__(self, name):
  41         self.name = name
  42         self.base = name[(name.rfind("/") + 1):name.rfind(".")]
  43         self.crash = False
  44         self.web_process_crash = False
  45         self.exception = False
  46         self.hang = False
  47         self.keyboard = False
  48         self.error = ''
  49         self.timeout = False
  50         self.is_reftest = False
  51
  52         # The values of each field are treated as raw byte strings. They
  53         # will be converted to unicode strings where appropriate using
  54         # FileSystem.read_text_file().
  55         self.actual_text = self.base + '-txt'
  56         self.actual_checksum = self.base + '-checksum'
  57
  58         # We add the '\x8a' for the image file to prevent the value from
  59         # being treated as UTF-8 (the character is invalid)
  60         self.actual_image = self.base + '\x8a' + '-png' + 'tEXtchecksum\x00' + self.actual_checksum
  61
  62         self.expected_text = self.actual_text
  63         self.expected_image = self.actual_image
  64
  65         self.actual_audio = None
  66         self.expected_audio = None
  67
  68
  69 # This is an in-memory list of tests, what we want them to produce, and
  70 # what we want to claim are the expected results.
  71 class TestList(object):
  72     def __init__(self):
  73         self.tests = {}
  74
  75     def add(self, name, **kwargs):
  76         test = TestInstance(name)
  77         for key, value in kwargs.items():
  78             test.__dict__[key] = value
  79         self.tests[name] = test
  80
  81     def add_reftest(self, name, reference_name, same_image, **kwargs):
  82         self.add(name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
  83         if same_image:
  84             self.add(reference_name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
  85         else:
  86             self.add(reference_name, actual_checksum='yyy', actual_image='YYY', is_reftest=True)
  87
  88         if kwargs:
  89             test = self.tests[name]
  90             for key, value in kwargs.items():
  91                 test.__dict__[key] = value
  92
  93     def keys(self):
  94         return self.tests.keys()
  95
  96     def __contains__(self, item):
  97         return item in self.tests
  98
  99     def __getitem__(self, item):
 100         return self.tests[item]
 101
 102 #
 103 # These numbers may need to be updated whenever we add or delete tests.
 104 #
 105 TOTAL_TESTS = 77
 106 TOTAL_SKIPS = 9
 107 TOTAL_RETRIES = 15
 108
 109 UNEXPECTED_PASSES = 7
 110 UNEXPECTED_FAILURES = 18
 111
 112
 113 def unit_test_list():
 114     silent_audio = "RIFF2\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x22\x56\x00\x00\x44\xAC\x00\x00\x02\x00\x10\x00data\x0E\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
 115     silent_audio_with_single_bit_difference = "RIFF2\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x22\x56\x00\x00\x44\xAC\x00\x00\x02\x00\x10\x00data\x0E\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
 116     audio2 = "RIFF2\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x22\x56\x00\x00\x44\xAC\x00\x00\x02\x00\x10\x00data\x0E\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
 117     tests = TestList()
 118     tests.add('failures/expected/crash.html', crash=True)
 119     tests.add('failures/expected/exception.html', exception=True)
 120     tests.add('failures/expected/timeout.html', timeout=True)
 121     tests.add('failures/expected/hang.html', hang=True)
 122     tests.add('failures/expected/missing_text.html', expected_text=None)
 123     tests.add('failures/expected/leak.html', leak=True)
 124     tests.add('failures/expected/flaky-leak.html', leak=True)
 125     tests.add('failures/expected/image.html',
 126               actual_image='image_fail-pngtEXtchecksum\x00checksum_fail',
 127               expected_image='image-pngtEXtchecksum\x00checksum-png')
 128     tests.add('failures/expected/pixel-fail.html',
 129               actual_image='image_fail-pngtEXtchecksum\x00checksum_fail',
 130               expected_image='image-pngtEXtchecksum\x00checksum-png')
 131     tests.add('failures/expected/image_checksum.html',
 132               actual_checksum='image_checksum_fail-checksum',
 133               actual_image='image_checksum_fail-png')
 134     tests.add('failures/expected/audio.html',
 135               actual_audio=silent_audio, expected_audio=audio2,
 136               actual_text=None, expected_text=None,
 137               actual_image=None, expected_image=None,
 138               actual_checksum=None)
 139     tests.add('failures/expected/keyboard.html', keyboard=True)
 140     tests.add('failures/expected/missing_check.html',
 141               expected_image='missing_check-png')
 142     tests.add('failures/expected/missing_image.html', expected_image=None)
 143     tests.add('failures/expected/missing_audio.html', expected_audio=None,
 144               actual_text=None, expected_text=None,
 145               actual_image=None, expected_image=None,
 146               actual_checksum=None)
 147     tests.add('failures/expected/missing_text.html', expected_text=None)
 148     tests.add('failures/expected/newlines_leading.html',
 149               expected_text="\nfoo\n", actual_text="foo\n")
 150     tests.add('failures/expected/newlines_trailing.html',
 151               expected_text="foo\n\n", actual_text="foo\n")
 152     tests.add('failures/expected/newlines_with_excess_CR.html',
 153               expected_text="foo\r\r\r\n", actual_text="foo\n")
 154     tests.add('failures/expected/text.html', actual_text='text_fail-png')
 155     tests.add('failures/expected/skip_text.html', actual_text='text diff')
 156     tests.add('failures/flaky/text.html')
 157     tests.add('failures/unexpected/missing_text.html', expected_text=None)
 158     tests.add('failures/unexpected/missing_check.html', expected_image='missing-check-png')
 159     tests.add('failures/unexpected/missing_image.html', expected_image=None)
 160     tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="""layer at (0,0) size 800x600
 161   RenderView at (0,0) size 800x600
 162 layer at (0,0) size 800x34
 163   RenderBlock {HTML} at (0,0) size 800x34
 164     RenderBody {BODY} at (8,8) size 784x18
 165       RenderText {#text} at (0,0) size 133x18
 166         text run at (0,0) width 133: "This is an image test!"
 167 """, expected_text=None)
 168     tests.add('failures/unexpected/crash.html', crash=True)
 169     tests.add('failures/unexpected/crash-with-stderr.html', crash=True,
 170               error="mock-std-error-output")
 171     tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_process_crash=True,
 172               error="mock-std-error-output")
 173     tests.add('failures/unexpected/pass.html')
 174     tests.add('failures/unexpected/text-checksum.html',
 175               actual_text='text-checksum_fail-txt',
 176               actual_checksum='text-checksum_fail-checksum')
 177     tests.add('failures/unexpected/text-image-checksum.html',
 178               actual_text='text-image-checksum_fail-txt',
 179               actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum_fail',
 180               actual_checksum='text-image-checksum_fail-checksum')
 181     tests.add('failures/unexpected/text-image-missing.html',
 182               actual_text='text-image-checksum_fail-txt',
 183               expected_image=None)
 184     tests.add('failures/unexpected/checksum-with-matching-image.html',
 185               actual_checksum='text-image-checksum_fail-checksum')
 186     tests.add('failures/unexpected/skip_pass.html')
 187     tests.add('failures/unexpected/text.html', actual_text='text_fail-txt')
 188     tests.add('failures/unexpected/timeout.html', timeout=True)
 189     tests.add('failures/unexpected/leak.html', leak=True)
 190     tests.add('http/tests/passes/text.html')
 191     tests.add('http/tests/passes/image.html')
 192     tests.add('http/tests/ssl/text.html')
 193     tests.add('passes/args.html')
 194     tests.add('passes/error.html', error='stuff going to stderr')
 195     tests.add('passes/image.html')
 196     tests.add('passes/audio.html',
 197               actual_audio=silent_audio, expected_audio=silent_audio,
 198               actual_text=None, expected_text=None,
 199               actual_image=None, expected_image=None,
 200               actual_checksum=None)
 201     tests.add('passes/audio-tolerance.html',
 202               actual_audio=silent_audio_with_single_bit_difference, expected_audio=silent_audio,
 203               actual_text=None, expected_text=None,
 204               actual_image=None, expected_image=None,
 205               actual_checksum=None)
 206     tests.add('passes/platform_image.html')
 207     tests.add('passes/checksum_in_image.html',
 208               expected_image='tEXtchecksum\x00checksum_in_image-checksum')
 209     tests.add('passes/skipped/skip.html')
 210
 211     # Note that here the checksums don't match but the images do, so this test passes "unexpectedly".
 212     # See https://bugs.webkit.org/show_bug.cgi?id=69444 .
 213     tests.add('failures/unexpected/checksum.html', actual_checksum='checksum_fail-checksum')
 214
 215     # Text output files contain "\r\n" on Windows.  This may be
 216     # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
 217     tests.add('passes/text.html',
 218               expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n')
 219
 220     # For reftests.
 221     tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True)
 222     tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False)
 223     tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg', same_image=True)
 224     tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html', same_image=True)
 225     tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-mismatch.svg', same_image=False)
 226     tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False)
 227     tests.add_reftest('failures/expected/leaky-reftest.html', 'failures/expected/leaky-reftest-expected.html', same_image=False, leak=True)
 228     tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True)
 229     tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
 230     tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
 231     tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True)
 232     tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checksum=None, actual_image=None, is_reftest=True)
 233     # FIXME: Add a reftest which crashes.
 234     tests.add('reftests/foo/test.html')
 235     tests.add('reftests/foo/test-ref.html')
 236
 237     tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc', actual_image='abc')
 238     tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc', actual_image='abc')
 239     tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='abc', actual_image='abc')
 240     tests.add('reftests/foo/multiple-mismatch-failure.html', actual_checksum='abc', actual_image='abc')
 241     tests.add('reftests/foo/multiple-both-success.html', actual_checksum='abc', actual_image='abc')
 242     tests.add('reftests/foo/multiple-both-failure.html', actual_checksum='abc', actual_image='abc')
 243
 244     tests.add('reftests/foo/matching-ref.html', actual_checksum='abc', actual_image='abc')
 245     tests.add('reftests/foo/mismatching-ref.html', actual_checksum='def', actual_image='def')
 246     tests.add('reftests/foo/second-mismatching-ref.html', actual_checksum='ghi', actual_image='ghi')
 247
 248     # The following files shouldn't be treated as reftests
 249     tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.html', same_image=True)
 250     tests.add('reftests/foo/reference/bar/common.html')
 251     tests.add('reftests/foo/reftest/bar/shared.html')
 252
 253     tests.add('websocket/tests/passes/text.html')
 254
 255     # For testing test are properly included from platform directories.
 256     tests.add('platform/test-mac-leopard/http/test.html')
 257     tests.add('platform/test-win-7sp0/http/test.html')
 258
 259     # For --no-http tests, test that platform specific HTTP tests are properly skipped.
 260     tests.add('platform/test-snow-leopard/http/test.html')
 261     tests.add('platform/test-snow-leopard/websocket/test.html')
 262
 263     # For testing if perf tests are running in a locked shard.
 264     tests.add('perf/foo/test.html')
 265     tests.add('perf/foo/test-ref.html')
 266
 267     # For testing --pixel-test-directories.
 268     tests.add('failures/unexpected/pixeldir/image_in_pixeldir.html',
 269         actual_image='image_in_pixeldir-pngtEXtchecksum\x00checksum_fail',
 270         expected_image='image_in_pixeldir-pngtEXtchecksum\x00checksum-png')
 271     tests.add('failures/unexpected/image_not_in_pixeldir.html',
 272         actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fail',
 273         expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-png')
 274
 275     return tests
 276
 277
 278 # Here we use a non-standard location for the layout tests, to ensure that
 279 # this works. The path contains a '.' in the name because we've seen bugs
 280 # related to this before.
 281
 282 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests'
 283 PERF_TEST_DIR = '/test.checkout/PerformanceTests'
 284
 285
 286 # Here we synthesize an in-memory filesystem from the test list
 287 # in order to fully control the test output and to demonstrate that
 288 # we don't need a real filesystem to run the tests.
 289 def add_unit_tests_to_mock_filesystem(filesystem):
 290     # Add the test_expectations file.
 291     filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/platform/test')
 292     if not filesystem.exists(LAYOUT_TEST_DIR + '/platform/test/TestExpectations'):
 293         filesystem.write_text_file(LAYOUT_TEST_DIR + '/platform/test/TestExpectations', """
 294 Bug(test) failures/expected/crash.html [ Crash ]
 295 Bug(test) failures/expected/leak.html [ Leak ]
 296 Bug(test) failures/expected/flaky-leak.html [ Failure Leak ]
 297 Bug(test) failures/expected/leaky-reftest.html [ ImageOnlyFailure Leak ]
 298 Bug(test) failures/expected/image.html [ ImageOnlyFailure ]
 299 Bug(test) failures/expected/pixel-fail.html [ ImageOnlyFailure ]
 300 Bug(test) failures/expected/audio.html [ Failure ]
 301 Bug(test) failures/expected/image_checksum.html [ ImageOnlyFailure ]
 302 Bug(test) failures/expected/mismatch.html [ ImageOnlyFailure ]
 303 Bug(test) failures/expected/missing_check.html [ Missing Pass ]
 304 Bug(test) failures/expected/missing_image.html [ Missing Pass ]
 305 Bug(test) failures/expected/missing_audio.html [ Missing Pass ]
 306 Bug(test) failures/expected/missing_text.html [ Missing Pass ]
 307 Bug(test) failures/expected/newlines_leading.html [ Failure ]
 308 Bug(test) failures/expected/newlines_trailing.html [ Failure ]
 309 Bug(test) failures/expected/newlines_with_excess_CR.html [ Failure ]
 310 Bug(test) failures/expected/reftest.html [ ImageOnlyFailure ]
 311 Bug(test) failures/expected/text.html [ Failure ]
 312 Bug(test) failures/expected/timeout.html [ Timeout ]
 313 Bug(test) failures/expected/hang.html [ WontFix ]
 314 Bug(test) failures/expected/keyboard.html [ WontFix ]
 315 Bug(test) failures/expected/exception.html [ WontFix ]
 316 Bug(test) failures/unexpected/pass.html [ Failure ]
 317 Bug(test) passes/skipped/skip.html [ Skip ]
 318 """)
 319
 320     filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/reftests/foo')
 321     filesystem.write_text_file(LAYOUT_TEST_DIR + '/reftests/foo/reftest.list', """
 322 == test.html test-ref.html
 323
 324 == multiple-match-success.html mismatching-ref.html
 325 == multiple-match-success.html matching-ref.html
 326 == multiple-match-failure.html mismatching-ref.html
 327 == multiple-match-failure.html second-mismatching-ref.html
 328 != multiple-mismatch-success.html mismatching-ref.html
 329 != multiple-mismatch-success.html second-mismatching-ref.html
 330 != multiple-mismatch-failure.html mismatching-ref.html
 331 != multiple-mismatch-failure.html matching-ref.html
 332 == multiple-both-success.html matching-ref.html
 333 == multiple-both-success.html mismatching-ref.html
 334 != multiple-both-success.html second-mismatching-ref.html
 335 == multiple-both-failure.html matching-ref.html
 336 != multiple-both-failure.html second-mismatching-ref.html
 337 != multiple-both-failure.html matching-ref.html
 338 """)
 339
 340     # FIXME: This test was only being ignored because of missing a leading '/'.
 341     # Fixing the typo causes several tests to assert, so disabling the test entirely.
 342     # Add in a file should be ignored by port.find_test_files().
 343     #files[LAYOUT_TEST_DIR + '/userscripts/resources/iframe.html'] = 'iframe'
 344
 345     def add_file(test, suffix, contents):
 346         dirname = filesystem.join(LAYOUT_TEST_DIR, test.name[0:test.name.rfind('/')])
 347         base = test.base
 348         filesystem.maybe_make_directory(dirname)
 349         filesystem.write_binary_file(filesystem.join(dirname, base + suffix), contents)
 350
 351     # Add each test and the expected output, if any.
 352     test_list = unit_test_list()
 353     for test in test_list.tests.values():
 354         add_file(test, test.name[test.name.rfind('.'):], '')
 355         if test.is_reftest:
 356             continue
 357         if test.actual_audio:
 358             add_file(test, '-expected.wav', test.expected_audio)
 359             continue
 360         add_file(test, '-expected.txt', test.expected_text)
 361         add_file(test, '-expected.png', test.expected_image)
 362
 363     # Clear the list of written files so that we can watch what happens during testing.
 364     filesystem.clear_written_files()
 365
 366
 367 class TestPort(Port):
 368     port_name = 'test'
 369     default_port_name = 'test-mac-leopard'
 370
 371     """Test implementation of the Port interface."""
 372     ALL_BASELINE_VARIANTS = (
 373         'test-linux-x86_64',
 374         'test-mac-snowleopard', 'test-mac-leopard',
 375         'test-win-vista', 'test-win-7sp0', 'test-win-xp',
 376     )
 377
 378     @classmethod
 379     def determine_full_port_name(cls, host, options, port_name):
 380         if port_name == 'test':
 381             return TestPort.default_port_name
 382         return port_name
 383
 384     def __init__(self, host, port_name=None, **kwargs):
 385         Port.__init__(self, host, port_name or TestPort.default_port_name, **kwargs)
 386         self._tests = unit_test_list()
 387         self._flakes = set()
 388         self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/TestExpectations'
 389         self._results_directory = None
 390
 391         self._operating_system = self._name.split('-')[1]
 392         if self._operating_system == 'linux':
 393             self._os_version = None
 394         else:
 395             self._os_version = VersionNameMap.map(self.host.platform).from_name(self._name.split('-')[2])[1]
 396
 397     def version_name(self):
 398         if self._os_version is None:
 399             return None
 400         return VersionNameMap.map(self.host.platform).to_name(self._os_version, platform=self._operating_system, table=PUBLIC_TABLE)
 401
 402     def default_pixel_tests(self):
 403         return True
 404
 405     def _path_to_driver(self):
 406         # This routine shouldn't normally be called, but it is called by
 407         # the mock_drt Driver. We return something, but make sure it's useless.
 408         return 'MOCK _path_to_driver'
 409
 410     def baseline_search_path(self):
 411         search_paths = {
 412             'test-mac-snowleopard': ['test-mac-snowleopard'],
 413             'test-mac-leopard': ['test-mac-leopard', 'test-mac-snowleopard'],
 414             'test-win-7sp0': ['test-win-7sp0'],
 415             'test-win-vista': ['test-win-vista', 'test-win-7sp0'],
 416             'test-win-xp': ['test-win-xp', 'test-win-vista', 'test-win-7sp0'],
 417             'test-linux-x86_64': ['test-linux'],
 418         }
 419         return [self._webkit_baseline_path(d) for d in search_paths[self.name()]]
 420
 421     def default_child_processes(self):
 422         return 1
 423
 424     def worker_startup_delay_secs(self):
 425         return 0
 426
 427     def check_build(self):
 428         return True
 429
 430     def check_sys_deps(self):
 431         return True
 432
 433     def default_configuration(self):
 434         return 'Release'
 435
 436     def diff_image(self, expected_contents, actual_contents, tolerance=None):
 437         diffed = actual_contents != expected_contents
 438         if not actual_contents and not expected_contents:
 439             return (None, 0, None)
 440         if not actual_contents or not expected_contents:
 441             return (True, 0, None)
 442         if 'ref' in expected_contents:
 443             assert tolerance == 0
 444         if diffed:
 445             return ("< %s\n---\n> %s\n" % (expected_contents, actual_contents), 1, None)
 446         return (None, 0, None)
 447
 448     def layout_tests_dir(self):
 449         return LAYOUT_TEST_DIR
 450
 451     def perf_tests_dir(self):
 452         return PERF_TEST_DIR
 453
 454     def webkit_base(self):
 455         return '/test.checkout'
 456
 457     def _skipped_tests_for_unsupported_features(self, test_list):
 458         return set(['failures/expected/skip_text.html',
 459                     'failures/unexpected/skip_pass.html'])
 460
 461     def name(self):
 462         return self._name
 463
 464     def operating_system(self):
 465         return self._operating_system
 466
 467     def default_results_directory(self):
 468         return '/tmp/layout-test-results'
 469
 470     def setup_test_run(self, device_class=None):
 471         pass
 472
 473     def _driver_class(self):
 474         return TestDriver
 475
 476     def path_to_crash_logs(self):
 477         return self.results_directory()
 478
 479     def start_http_server(self, additional_dirs=None):
 480         pass
 481
 482     def start_websocket_server(self):
 483         pass
 484
 485     def stop_http_server(self):
 486         pass
 487
 488     def stop_websocket_server(self):
 489         pass
 490
 491     def _path_to_lighttpd(self):
 492         return "/usr/sbin/lighttpd"
 493
 494     def _path_to_lighttpd_modules(self):
 495         return "/usr/lib/lighttpd"
 496
 497     def _path_to_lighttpd_php(self):
 498         return "/usr/bin/php-cgi"
 499
 500     def _path_to_apache(self):
 501         return "/usr/sbin/httpd"
 502
 503     def _path_to_apache_config_file(self):
 504         return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', 'httpd.conf')
 505
 506     def path_to_test_expectations_file(self):
 507         return self._expectations_path
 508
 509     def all_test_configurations(self):
 510         """Returns a sequence of the TestConfigurations the port supports."""
 511         # By default, we assume we want to test every graphics type in
 512         # every configuration on every system.
 513         test_configurations = []
 514         for version, architecture in self._all_systems():
 515             for build_type in self._all_build_types():
 516                 test_configurations.append(TestConfiguration(
 517                     version=version,
 518                     architecture=architecture,
 519                     build_type=build_type))
 520         return test_configurations
 521
 522     def _all_systems(self):
 523         return (('leopard', 'x86'),
 524                 ('snowleopard', 'x86'),
 525                 ('xp', 'x86'),
 526                 ('vista', 'x86'),
 527                 ('7sp0', 'x86'),
 528                 ('lucid', 'x86'),
 529                 ('lucid', 'x86_64'))
 530
 531     def _all_build_types(self):
 532         return ('debug', 'release')
 533
 534     def configuration_specifier_macros(self):
 535         """To avoid surprises when introducing new macros, these are intentionally fixed in time."""
 536         return {'mac': ['leopard', 'snowleopard'], 'win': ['xp', 'vista', '7sp0'], 'linux': ['lucid']}
 537
 538     def all_baseline_variants(self):
 539         return self.ALL_BASELINE_VARIANTS
 540
 541
 542 class TestDriver(Driver):
 543     """Test/Dummy implementation of the DumpRenderTree interface."""
 544     next_pid = 1
 545
 546     def __init__(self, *args, **kwargs):
 547         super(TestDriver, self).__init__(*args, **kwargs)
 548         self.started = False
 549         self.pid = 0
 550
 551     def cmd_line(self, pixel_tests, per_test_args):
 552         pixel_tests_flag = '-p' if pixel_tests else ''
 553         return [self._port._path_to_driver()] + [pixel_tests_flag] + self._port.get_option('additional_drt_flag', []) + per_test_args
 554
 555     def run_test(self, test_input, stop_when_done):
 556         if not self.started:
 557             self.started = True
 558             self.pid = TestDriver.next_pid
 559             TestDriver.next_pid += 1
 560
 561         start_time = time.time()
 562         test_name = test_input.test_name
 563         test_args = test_input.args or []
 564         test = self._port._tests[test_name]
 565         if test.keyboard:
 566             raise KeyboardInterrupt
 567         if test.exception:
 568             raise ValueError('exception from ' + test_name)
 569         if test.hang:
 570             time.sleep((float(test_input.timeout) * 4) / 1000.0 + 1.0)  # The 1.0 comes from thread_padding_sec in layout_test_runnery.
 571
 572         audio = None
 573         actual_text = test.actual_text
 574
 575         if 'flaky' in test_name and not test_name in self._port._flakes:
 576             self._port._flakes.add(test_name)
 577             actual_text = 'flaky text failure'
 578
 579         if actual_text and test_args and test_name == 'passes/args.html':
 580             actual_text = actual_text + ' ' + ' '.join(test_args)
 581
 582         if test.actual_audio:
 583             audio = test.actual_audio
 584         crashed_process_name = None
 585         crashed_pid = None
 586         if test.crash:
 587             crashed_process_name = self._port.driver_name()
 588             crashed_pid = 1
 589         elif test.web_process_crash:
 590             crashed_process_name = 'WebProcess'
 591             crashed_pid = 2
 592
 593         crash_log = ''
 594         if crashed_process_name:
 595             crash_logs = CrashLogs(self._port.host, self._port.path_to_crash_logs())
 596             crash_log = crash_logs.find_newest_log(crashed_process_name, None) or ''
 597
 598         if stop_when_done:
 599             self.stop()
 600
 601         if test.actual_checksum == test_input.image_hash:
 602             image = None
 603         else:
 604             image = test.actual_image
 605         return DriverOutput(actual_text, image, test.actual_checksum, audio,
 606             crash=test.crash or test.web_process_crash, crashed_process_name=crashed_process_name,
 607             crashed_pid=crashed_pid, crash_log=crash_log,
 608             test_time=time.time() - start_time, timeout=test.timeout, error=test.error, pid=self.pid)
 609
 610     def do_post_tests_work(self):
 611         if not self._port.get_option('world_leaks'):
 612             return None
 613
 614         test_world_leaks_output = """TEST: file:///test.checkout/LayoutTests/failures/expected/leak.html
 615 ABANDONED DOCUMENT: file:///test.checkout/LayoutTests/failures/expected/leak.html
 616 TEST: file:///test.checkout/LayoutTests/failures/unexpected/leak.html
 617 ABANDONED DOCUMENT: file:///test.checkout/LayoutTests/failures/expected/flaky-leak.html
 618 TEST: file:///test.checkout/LayoutTests/failures/unexpected/flaky-leak.html
 619 ABANDONED DOCUMENT: file:///test.checkout/LayoutTests/failures/expected/leak.html
 620 TEST: file:///test.checkout/LayoutTests/failures/unexpected/leak.html
 621 ABANDONED DOCUMENT: file:///test.checkout/LayoutTests/failures/expected/leak-subframe.html
 622 TEST: file:///test.checkout/LayoutTests/failures/expected/leaky-reftest.html
 623 ABANDONED DOCUMENT: file:///test.checkout/LayoutTests/failures/expected/leaky-reftest.html"""
 624         return self._parse_world_leaks_output(test_world_leaks_output)
 625
 626     def stop(self):
 627         self.started = False