Remove virtual test support from webkitpy
[WebKit-https.git] / Tools / Scripts / webkitpy / port / test.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the Google name nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import base64
30 import sys
31 import time
32
33 from webkitpy.port import Port, Driver, DriverOutput
34 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
35 from webkitpy.common.system.filesystem_mock import MockFileSystem
36 from webkitpy.common.system.crashlogs import CrashLogs
37
38
39 # This sets basic expectations for a test. Each individual expectation
40 # can be overridden by a keyword argument in TestList.add().
41 class TestInstance(object):
42     def __init__(self, name):
43         self.name = name
44         self.base = name[(name.rfind("/") + 1):name.rfind(".")]
45         self.crash = False
46         self.web_process_crash = False
47         self.exception = False
48         self.hang = False
49         self.keyboard = False
50         self.error = ''
51         self.timeout = False
52         self.is_reftest = False
53
54         # The values of each field are treated as raw byte strings. They
55         # will be converted to unicode strings where appropriate using
56         # FileSystem.read_text_file().
57         self.actual_text = self.base + '-txt'
58         self.actual_checksum = self.base + '-checksum'
59
60         # We add the '\x8a' for the image file to prevent the value from
61         # being treated as UTF-8 (the character is invalid)
62         self.actual_image = self.base + '\x8a' + '-png' + 'tEXtchecksum\x00' + self.actual_checksum
63
64         self.expected_text = self.actual_text
65         self.expected_image = self.actual_image
66
67         self.actual_audio = None
68         self.expected_audio = None
69
70
71 # This is an in-memory list of tests, what we want them to produce, and
72 # what we want to claim are the expected results.
73 class TestList(object):
74     def __init__(self):
75         self.tests = {}
76
77     def add(self, name, **kwargs):
78         test = TestInstance(name)
79         for key, value in kwargs.items():
80             test.__dict__[key] = value
81         self.tests[name] = test
82
83     def add_reftest(self, name, reference_name, same_image):
84         self.add(name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
85         if same_image:
86             self.add(reference_name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
87         else:
88             self.add(reference_name, actual_checksum='yyy', actual_image='YYY', is_reftest=True)
89
90     def keys(self):
91         return self.tests.keys()
92
93     def __contains__(self, item):
94         return item in self.tests
95
96     def __getitem__(self, item):
97         return self.tests[item]
98
99 #
100 # These numbers may need to be updated whenever we add or delete tests.
101 #
102 TOTAL_TESTS = 71
103 TOTAL_SKIPS = 28
104 TOTAL_RETRIES = 14
105
106 UNEXPECTED_PASSES = 7
107 UNEXPECTED_FAILURES = 17
108
109 def unit_test_list():
110     tests = TestList()
111     tests.add('failures/expected/crash.html', crash=True)
112     tests.add('failures/expected/exception.html', exception=True)
113     tests.add('failures/expected/timeout.html', timeout=True)
114     tests.add('failures/expected/hang.html', hang=True)
115     tests.add('failures/expected/missing_text.html', expected_text=None)
116     tests.add('failures/expected/image.html',
117               actual_image='image_fail-pngtEXtchecksum\x00checksum_fail',
118               expected_image='image-pngtEXtchecksum\x00checksum-png')
119     tests.add('failures/expected/image_checksum.html',
120               actual_checksum='image_checksum_fail-checksum',
121               actual_image='image_checksum_fail-png')
122     tests.add('failures/expected/audio.html',
123               actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='audio-wav',
124               actual_text=None, expected_text=None,
125               actual_image=None, expected_image=None,
126               actual_checksum=None)
127     tests.add('failures/expected/keyboard.html', keyboard=True)
128     tests.add('failures/expected/missing_check.html',
129               expected_image='missing_check-png')
130     tests.add('failures/expected/missing_image.html', expected_image=None)
131     tests.add('failures/expected/missing_audio.html', expected_audio=None,
132               actual_text=None, expected_text=None,
133               actual_image=None, expected_image=None,
134               actual_checksum=None)
135     tests.add('failures/expected/missing_text.html', expected_text=None)
136     tests.add('failures/expected/newlines_leading.html',
137               expected_text="\nfoo\n", actual_text="foo\n")
138     tests.add('failures/expected/newlines_trailing.html',
139               expected_text="foo\n\n", actual_text="foo\n")
140     tests.add('failures/expected/newlines_with_excess_CR.html',
141               expected_text="foo\r\r\r\n", actual_text="foo\n")
142     tests.add('failures/expected/text.html', actual_text='text_fail-png')
143     tests.add('failures/expected/skip_text.html', actual_text='text diff')
144     tests.add('failures/flaky/text.html')
145     tests.add('failures/unexpected/missing_text.html', expected_text=None)
146     tests.add('failures/unexpected/missing_check.html', expected_image='missing-check-png')
147     tests.add('failures/unexpected/missing_image.html', expected_image=None)
148     tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="""layer at (0,0) size 800x600
149   RenderView at (0,0) size 800x600
150 layer at (0,0) size 800x34
151   RenderBlock {HTML} at (0,0) size 800x34
152     RenderBody {BODY} at (8,8) size 784x18
153       RenderText {#text} at (0,0) size 133x18
154         text run at (0,0) width 133: "This is an image test!"
155 """, expected_text=None)
156     tests.add('failures/unexpected/crash.html', crash=True)
157     tests.add('failures/unexpected/crash-with-stderr.html', crash=True,
158               error="mock-std-error-output")
159     tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_process_crash=True,
160               error="mock-std-error-output")
161     tests.add('failures/unexpected/pass.html')
162     tests.add('failures/unexpected/text-checksum.html',
163               actual_text='text-checksum_fail-txt',
164               actual_checksum='text-checksum_fail-checksum')
165     tests.add('failures/unexpected/text-image-checksum.html',
166               actual_text='text-image-checksum_fail-txt',
167               actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum_fail',
168               actual_checksum='text-image-checksum_fail-checksum')
169     tests.add('failures/unexpected/text-image-missing.html',
170               actual_text='text-image-checksum_fail-txt',
171               expected_image=None)
172     tests.add('failures/unexpected/checksum-with-matching-image.html',
173               actual_checksum='text-image-checksum_fail-checksum')
174     tests.add('failures/unexpected/skip_pass.html')
175     tests.add('failures/unexpected/text.html', actual_text='text_fail-txt')
176     tests.add('failures/unexpected/timeout.html', timeout=True)
177     tests.add('http/tests/passes/text.html')
178     tests.add('http/tests/passes/image.html')
179     tests.add('http/tests/ssl/text.html')
180     tests.add('passes/args.html')
181     tests.add('passes/error.html', error='stuff going to stderr')
182     tests.add('passes/image.html')
183     tests.add('passes/audio.html',
184               actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-wav',
185               actual_text=None, expected_text=None,
186               actual_image=None, expected_image=None,
187               actual_checksum=None)
188     tests.add('passes/platform_image.html')
189     tests.add('passes/checksum_in_image.html',
190               expected_image='tEXtchecksum\x00checksum_in_image-checksum')
191     tests.add('passes/skipped/skip.html')
192
193     # Note that here the checksums don't match but the images do, so this test passes "unexpectedly".
194     # See https://bugs.webkit.org/show_bug.cgi?id=69444 .
195     tests.add('failures/unexpected/checksum.html', actual_checksum='checksum_fail-checksum')
196
197     # Text output files contain "\r\n" on Windows.  This may be
198     # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
199     tests.add('passes/text.html',
200               expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n')
201
202     # For reftests.
203     tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True)
204     tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False)
205     tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg', same_image=True)
206     tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html', same_image=True)
207     tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-mismatch.svg', same_image=False)
208     tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False)
209     tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True)
210     tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
211     tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
212     tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True)
213     tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checksum=None, actual_image=None, is_reftest=True)
214     # FIXME: Add a reftest which crashes.
215     tests.add('reftests/foo/test.html')
216     tests.add('reftests/foo/test-ref.html')
217
218     tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc', actual_image='abc')
219     tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc', actual_image='abc')
220     tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='abc', actual_image='abc')
221     tests.add('reftests/foo/multiple-mismatch-failure.html', actual_checksum='abc', actual_image='abc')
222     tests.add('reftests/foo/multiple-both-success.html', actual_checksum='abc', actual_image='abc')
223     tests.add('reftests/foo/multiple-both-failure.html', actual_checksum='abc', actual_image='abc')
224
225     tests.add('reftests/foo/matching-ref.html', actual_checksum='abc', actual_image='abc')
226     tests.add('reftests/foo/mismatching-ref.html', actual_checksum='def', actual_image='def')
227     tests.add('reftests/foo/second-mismatching-ref.html', actual_checksum='ghi', actual_image='ghi')
228
229     # The following files shouldn't be treated as reftests
230     tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.html', same_image=True)
231     tests.add('reftests/foo/reference/bar/common.html')
232     tests.add('reftests/foo/reftest/bar/shared.html')
233
234     tests.add('websocket/tests/passes/text.html')
235
236     # For testing test are properly included from platform directories.
237     tests.add('platform/test-mac-leopard/http/test.html')
238     tests.add('platform/test-win-win7/http/test.html')
239
240     # For --no-http tests, test that platform specific HTTP tests are properly skipped.
241     tests.add('platform/test-snow-leopard/http/test.html')
242     tests.add('platform/test-snow-leopard/websocket/test.html')
243
244     # For testing if perf tests are running in a locked shard.
245     tests.add('perf/foo/test.html')
246     tests.add('perf/foo/test-ref.html')
247
248     # For testing --pixel-test-directories.
249     tests.add('failures/unexpected/pixeldir/image_in_pixeldir.html',
250         actual_image='image_in_pixeldir-pngtEXtchecksum\x00checksum_fail',
251         expected_image='image_in_pixeldir-pngtEXtchecksum\x00checksum-png')
252     tests.add('failures/unexpected/image_not_in_pixeldir.html',
253         actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fail',
254         expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-png')
255
256     return tests
257
258
259 # Here we use a non-standard location for the layout tests, to ensure that
260 # this works. The path contains a '.' in the name because we've seen bugs
261 # related to this before.
262
263 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests'
264 PERF_TEST_DIR = '/test.checkout/PerformanceTests'
265
266
267 # Here we synthesize an in-memory filesystem from the test list
268 # in order to fully control the test output and to demonstrate that
269 # we don't need a real filesystem to run the tests.
270 def add_unit_tests_to_mock_filesystem(filesystem):
271     # Add the test_expectations file.
272     filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/platform/test')
273     if not filesystem.exists(LAYOUT_TEST_DIR + '/platform/test/TestExpectations'):
274         filesystem.write_text_file(LAYOUT_TEST_DIR + '/platform/test/TestExpectations', """
275 Bug(test) failures/expected/crash.html [ Crash ]
276 Bug(test) failures/expected/image.html [ ImageOnlyFailure ]
277 Bug(test) failures/expected/audio.html [ Failure ]
278 Bug(test) failures/expected/image_checksum.html [ ImageOnlyFailure ]
279 Bug(test) failures/expected/mismatch.html [ ImageOnlyFailure ]
280 Bug(test) failures/expected/missing_check.html [ Missing Pass ]
281 Bug(test) failures/expected/missing_image.html [ Missing Pass ]
282 Bug(test) failures/expected/missing_audio.html [ Missing Pass ]
283 Bug(test) failures/expected/missing_text.html [ Missing Pass ]
284 Bug(test) failures/expected/newlines_leading.html [ Failure ]
285 Bug(test) failures/expected/newlines_trailing.html [ Failure ]
286 Bug(test) failures/expected/newlines_with_excess_CR.html [ Failure ]
287 Bug(test) failures/expected/reftest.html [ ImageOnlyFailure ]
288 Bug(test) failures/expected/text.html [ Failure ]
289 Bug(test) failures/expected/timeout.html [ Timeout ]
290 Bug(test) failures/expected/hang.html [ WontFix ]
291 Bug(test) failures/expected/keyboard.html [ WontFix ]
292 Bug(test) failures/expected/exception.html [ WontFix ]
293 Bug(test) failures/unexpected/pass.html [ Failure ]
294 Bug(test) passes/skipped/skip.html [ Skip ]
295 """)
296
297     filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/reftests/foo')
298     filesystem.write_text_file(LAYOUT_TEST_DIR + '/reftests/foo/reftest.list', """
299 == test.html test-ref.html
300
301 == multiple-match-success.html mismatching-ref.html
302 == multiple-match-success.html matching-ref.html
303 == multiple-match-failure.html mismatching-ref.html
304 == multiple-match-failure.html second-mismatching-ref.html
305 != multiple-mismatch-success.html mismatching-ref.html
306 != multiple-mismatch-success.html second-mismatching-ref.html
307 != multiple-mismatch-failure.html mismatching-ref.html
308 != multiple-mismatch-failure.html matching-ref.html
309 == multiple-both-success.html matching-ref.html
310 == multiple-both-success.html mismatching-ref.html
311 != multiple-both-success.html second-mismatching-ref.html
312 == multiple-both-failure.html matching-ref.html
313 != multiple-both-failure.html second-mismatching-ref.html
314 != multiple-both-failure.html matching-ref.html
315 """)
316
317     # FIXME: This test was only being ignored because of missing a leading '/'.
318     # Fixing the typo causes several tests to assert, so disabling the test entirely.
319     # Add in a file should be ignored by port.find_test_files().
320     #files[LAYOUT_TEST_DIR + '/userscripts/resources/iframe.html'] = 'iframe'
321
322     def add_file(test, suffix, contents):
323         dirname = filesystem.join(LAYOUT_TEST_DIR, test.name[0:test.name.rfind('/')])
324         base = test.base
325         filesystem.maybe_make_directory(dirname)
326         filesystem.write_binary_file(filesystem.join(dirname, base + suffix), contents)
327
328     # Add each test and the expected output, if any.
329     test_list = unit_test_list()
330     for test in test_list.tests.values():
331         add_file(test, test.name[test.name.rfind('.'):], '')
332         if test.is_reftest:
333             continue
334         if test.actual_audio:
335             add_file(test, '-expected.wav', test.expected_audio)
336             continue
337         add_file(test, '-expected.txt', test.expected_text)
338         add_file(test, '-expected.png', test.expected_image)
339
340     # Clear the list of written files so that we can watch what happens during testing.
341     filesystem.clear_written_files()
342
343
344 class TestPort(Port):
345     port_name = 'test'
346     default_port_name = 'test-mac-leopard'
347
348     """Test implementation of the Port interface."""
349     ALL_BASELINE_VARIANTS = (
350         'test-linux-x86_64',
351         'test-mac-snowleopard', 'test-mac-leopard',
352         'test-win-vista', 'test-win-win7', 'test-win-xp',
353     )
354
355     @classmethod
356     def determine_full_port_name(cls, host, options, port_name):
357         if port_name == 'test':
358             return TestPort.default_port_name
359         return port_name
360
361     def __init__(self, host, port_name=None, **kwargs):
362         Port.__init__(self, host, port_name or TestPort.default_port_name, **kwargs)
363         self._tests = unit_test_list()
364         self._flakes = set()
365         self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/TestExpectations'
366         self._results_directory = None
367
368         self._operating_system = 'mac'
369         if self._name.startswith('test-win'):
370             self._operating_system = 'win'
371         elif self._name.startswith('test-linux'):
372             self._operating_system = 'linux'
373
374         version_map = {
375             'test-win-xp': 'xp',
376             'test-win-win7': 'win7',
377             'test-win-vista': 'vista',
378             'test-mac-leopard': 'leopard',
379             'test-mac-snowleopard': 'snowleopard',
380             'test-linux-x86_64': 'lucid',
381         }
382         self._version = version_map[self._name]
383
384     def default_pixel_tests(self):
385         return True
386
387     def _path_to_driver(self):
388         # This routine shouldn't normally be called, but it is called by
389         # the mock_drt Driver. We return something, but make sure it's useless.
390         return 'MOCK _path_to_driver'
391
392     def baseline_search_path(self):
393         search_paths = {
394             'test-mac-snowleopard': ['test-mac-snowleopard'],
395             'test-mac-leopard': ['test-mac-leopard', 'test-mac-snowleopard'],
396             'test-win-win7': ['test-win-win7'],
397             'test-win-vista': ['test-win-vista', 'test-win-win7'],
398             'test-win-xp': ['test-win-xp', 'test-win-vista', 'test-win-win7'],
399             'test-linux-x86_64': ['test-linux', 'test-win-win7'],
400         }
401         return [self._webkit_baseline_path(d) for d in search_paths[self.name()]]
402
403     def default_child_processes(self):
404         return 1
405
406     def worker_startup_delay_secs(self):
407         return 0
408
409     def check_build(self, needs_http):
410         return True
411
412     def check_sys_deps(self, needs_http):
413         return True
414
415     def default_configuration(self):
416         return 'Release'
417
418     def diff_image(self, expected_contents, actual_contents, tolerance=None):
419         diffed = actual_contents != expected_contents
420         if not actual_contents and not expected_contents:
421             return (None, 0, None)
422         if not actual_contents or not expected_contents:
423             return (True, 0, None)
424         if 'ref' in expected_contents:
425             assert tolerance == 0
426         if diffed:
427             return ("< %s\n---\n> %s\n" % (expected_contents, actual_contents), 1, None)
428         return (None, 0, None)
429
430     def layout_tests_dir(self):
431         return LAYOUT_TEST_DIR
432
433     def perf_tests_dir(self):
434         return PERF_TEST_DIR
435
436     def webkit_base(self):
437         return '/test.checkout'
438
439     def _skipped_tests_for_unsupported_features(self, test_list):
440         return set(['failures/expected/skip_text.html',
441                     'failures/unexpected/skip_pass.html'])
442
443     def name(self):
444         return self._name
445
446     def operating_system(self):
447         return self._operating_system
448
449     def default_results_directory(self):
450         return '/tmp/layout-test-results'
451
452     def setup_test_run(self):
453         pass
454
455     def _driver_class(self):
456         return TestDriver
457
458     def start_http_server(self, additional_dirs=None):
459         pass
460
461     def start_websocket_server(self):
462         pass
463
464     def stop_http_server(self):
465         pass
466
467     def stop_websocket_server(self):
468         pass
469
470     def _path_to_lighttpd(self):
471         return "/usr/sbin/lighttpd"
472
473     def _path_to_lighttpd_modules(self):
474         return "/usr/lib/lighttpd"
475
476     def _path_to_lighttpd_php(self):
477         return "/usr/bin/php-cgi"
478
479     def _path_to_apache(self):
480         return "/usr/sbin/httpd"
481
482     def _path_to_apache_config_file(self):
483         return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', 'httpd.conf')
484
485     def path_to_test_expectations_file(self):
486         return self._expectations_path
487
488     def all_test_configurations(self):
489         """Returns a sequence of the TestConfigurations the port supports."""
490         # By default, we assume we want to test every graphics type in
491         # every configuration on every system.
492         test_configurations = []
493         for version, architecture in self._all_systems():
494             for build_type in self._all_build_types():
495                 test_configurations.append(TestConfiguration(
496                     version=version,
497                     architecture=architecture,
498                     build_type=build_type))
499         return test_configurations
500
501     def _all_systems(self):
502         return (('leopard', 'x86'),
503                 ('snowleopard', 'x86'),
504                 ('xp', 'x86'),
505                 ('vista', 'x86'),
506                 ('win7', 'x86'),
507                 ('lucid', 'x86'),
508                 ('lucid', 'x86_64'))
509
510     def _all_build_types(self):
511         return ('debug', 'release')
512
513     def configuration_specifier_macros(self):
514         """To avoid surprises when introducing new macros, these are intentionally fixed in time."""
515         return {'mac': ['leopard', 'snowleopard'], 'win': ['xp', 'vista', 'win7'], 'linux': ['lucid']}
516
517     def all_baseline_variants(self):
518         return self.ALL_BASELINE_VARIANTS
519
520 class TestDriver(Driver):
521     """Test/Dummy implementation of the DumpRenderTree interface."""
522     next_pid = 1
523
524     def __init__(self, *args, **kwargs):
525         super(TestDriver, self).__init__(*args, **kwargs)
526         self.started = False
527         self.pid = 0
528
529     def cmd_line(self, pixel_tests, per_test_args):
530         pixel_tests_flag = '-p' if pixel_tests else ''
531         return [self._port._path_to_driver()] + [pixel_tests_flag] + self._port.get_option('additional_drt_flag', []) + per_test_args
532
533     def run_test(self, test_input, stop_when_done):
534         if not self.started:
535             self.started = True
536             self.pid = TestDriver.next_pid
537             TestDriver.next_pid += 1
538
539         start_time = time.time()
540         test_name = test_input.test_name
541         test_args = test_input.args or []
542         test = self._port._tests[test_name]
543         if test.keyboard:
544             raise KeyboardInterrupt
545         if test.exception:
546             raise ValueError('exception from ' + test_name)
547         if test.hang:
548             time.sleep((float(test_input.timeout) * 4) / 1000.0 + 1.0)  # The 1.0 comes from thread_padding_sec in layout_test_runnery.
549
550         audio = None
551         actual_text = test.actual_text
552
553         if 'flaky' in test_name and not test_name in self._port._flakes:
554             self._port._flakes.add(test_name)
555             actual_text = 'flaky text failure'
556
557         if actual_text and test_args and test_name == 'passes/args.html':
558             actual_text = actual_text + ' ' + ' '.join(test_args)
559
560         if test.actual_audio:
561             audio = base64.b64decode(test.actual_audio)
562         crashed_process_name = None
563         crashed_pid = None
564         if test.crash:
565             crashed_process_name = self._port.driver_name()
566             crashed_pid = 1
567         elif test.web_process_crash:
568             crashed_process_name = 'WebProcess'
569             crashed_pid = 2
570
571         crash_log = ''
572         if crashed_process_name:
573             crash_logs = CrashLogs(self._port.host)
574             crash_log = crash_logs.find_newest_log(crashed_process_name, None) or ''
575
576         if stop_when_done:
577             self.stop()
578
579         if test.actual_checksum == test_input.image_hash:
580             image = None
581         else:
582             image = test.actual_image
583         return DriverOutput(actual_text, image, test.actual_checksum, audio,
584             crash=test.crash or test.web_process_crash, crashed_process_name=crashed_process_name,
585             crashed_pid=crashed_pid, crash_log=crash_log,
586             test_time=time.time() - start_time, timeout=test.timeout, error=test.error, pid=self.pid)
587
588     def stop(self):
589         self.started = False