Remove wdiff
[WebKit-https.git] / Tools / Scripts / webkitpy / port / test.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the Google name nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import base64
30 import sys
31 import time
32
33 from webkitpy.port import Port, Driver, DriverOutput
34 from webkitpy.port.base import VirtualTestSuite
35 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
36 from webkitpy.common.system.filesystem_mock import MockFileSystem
37 from webkitpy.common.system.crashlogs import CrashLogs
38
39
40 # This sets basic expectations for a test. Each individual expectation
41 # can be overridden by a keyword argument in TestList.add().
42 class TestInstance(object):
43     def __init__(self, name):
44         self.name = name
45         self.base = name[(name.rfind("/") + 1):name.rfind(".")]
46         self.crash = False
47         self.web_process_crash = False
48         self.exception = False
49         self.hang = False
50         self.keyboard = False
51         self.error = ''
52         self.timeout = False
53         self.is_reftest = False
54
55         # The values of each field are treated as raw byte strings. They
56         # will be converted to unicode strings where appropriate using
57         # FileSystem.read_text_file().
58         self.actual_text = self.base + '-txt'
59         self.actual_checksum = self.base + '-checksum'
60
61         # We add the '\x8a' for the image file to prevent the value from
62         # being treated as UTF-8 (the character is invalid)
63         self.actual_image = self.base + '\x8a' + '-png' + 'tEXtchecksum\x00' + self.actual_checksum
64
65         self.expected_text = self.actual_text
66         self.expected_image = self.actual_image
67
68         self.actual_audio = None
69         self.expected_audio = None
70
71
72 # This is an in-memory list of tests, what we want them to produce, and
73 # what we want to claim are the expected results.
74 class TestList(object):
75     def __init__(self):
76         self.tests = {}
77
78     def add(self, name, **kwargs):
79         test = TestInstance(name)
80         for key, value in kwargs.items():
81             test.__dict__[key] = value
82         self.tests[name] = test
83
84     def add_reftest(self, name, reference_name, same_image):
85         self.add(name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
86         if same_image:
87             self.add(reference_name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
88         else:
89             self.add(reference_name, actual_checksum='yyy', actual_image='YYY', is_reftest=True)
90
91     def keys(self):
92         return self.tests.keys()
93
94     def __contains__(self, item):
95         return item in self.tests
96
97     def __getitem__(self, item):
98         return self.tests[item]
99
100 #
101 # These numbers may need to be updated whenever we add or delete tests.
102 #
103 TOTAL_TESTS = 106
104 TOTAL_SKIPS = 28
105 TOTAL_RETRIES = 14
106
107 UNEXPECTED_PASSES = 6
108 UNEXPECTED_FAILURES = 17
109
110 def unit_test_list():
111     tests = TestList()
112     tests.add('failures/expected/crash.html', crash=True)
113     tests.add('failures/expected/exception.html', exception=True)
114     tests.add('failures/expected/timeout.html', timeout=True)
115     tests.add('failures/expected/hang.html', hang=True)
116     tests.add('failures/expected/missing_text.html', expected_text=None)
117     tests.add('failures/expected/image.html',
118               actual_image='image_fail-pngtEXtchecksum\x00checksum_fail',
119               expected_image='image-pngtEXtchecksum\x00checksum-png')
120     tests.add('failures/expected/image_checksum.html',
121               actual_checksum='image_checksum_fail-checksum',
122               actual_image='image_checksum_fail-png')
123     tests.add('failures/expected/audio.html',
124               actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='audio-wav',
125               actual_text=None, expected_text=None,
126               actual_image=None, expected_image=None,
127               actual_checksum=None)
128     tests.add('failures/expected/keyboard.html', keyboard=True)
129     tests.add('failures/expected/missing_check.html',
130               expected_image='missing_check-png')
131     tests.add('failures/expected/missing_image.html', expected_image=None)
132     tests.add('failures/expected/missing_audio.html', expected_audio=None,
133               actual_text=None, expected_text=None,
134               actual_image=None, expected_image=None,
135               actual_checksum=None)
136     tests.add('failures/expected/missing_text.html', expected_text=None)
137     tests.add('failures/expected/newlines_leading.html',
138               expected_text="\nfoo\n", actual_text="foo\n")
139     tests.add('failures/expected/newlines_trailing.html',
140               expected_text="foo\n\n", actual_text="foo\n")
141     tests.add('failures/expected/newlines_with_excess_CR.html',
142               expected_text="foo\r\r\r\n", actual_text="foo\n")
143     tests.add('failures/expected/text.html', actual_text='text_fail-png')
144     tests.add('failures/expected/skip_text.html', actual_text='text diff')
145     tests.add('failures/flaky/text.html')
146     tests.add('failures/unexpected/missing_text.html', expected_text=None)
147     tests.add('failures/unexpected/missing_check.html', expected_image='missing-check-png')
148     tests.add('failures/unexpected/missing_image.html', expected_image=None)
149     tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="""layer at (0,0) size 800x600
150   RenderView at (0,0) size 800x600
151 layer at (0,0) size 800x34
152   RenderBlock {HTML} at (0,0) size 800x34
153     RenderBody {BODY} at (8,8) size 784x18
154       RenderText {#text} at (0,0) size 133x18
155         text run at (0,0) width 133: "This is an image test!"
156 """, expected_text=None)
157     tests.add('failures/unexpected/crash.html', crash=True)
158     tests.add('failures/unexpected/crash-with-stderr.html', crash=True,
159               error="mock-std-error-output")
160     tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_process_crash=True,
161               error="mock-std-error-output")
162     tests.add('failures/unexpected/pass.html')
163     tests.add('failures/unexpected/text-checksum.html',
164               actual_text='text-checksum_fail-txt',
165               actual_checksum='text-checksum_fail-checksum')
166     tests.add('failures/unexpected/text-image-checksum.html',
167               actual_text='text-image-checksum_fail-txt',
168               actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum_fail',
169               actual_checksum='text-image-checksum_fail-checksum')
170     tests.add('failures/unexpected/checksum-with-matching-image.html',
171               actual_checksum='text-image-checksum_fail-checksum')
172     tests.add('failures/unexpected/skip_pass.html')
173     tests.add('failures/unexpected/text.html', actual_text='text_fail-txt')
174     tests.add('failures/unexpected/timeout.html', timeout=True)
175     tests.add('http/tests/passes/text.html')
176     tests.add('http/tests/passes/image.html')
177     tests.add('http/tests/ssl/text.html')
178     tests.add('passes/args.html')
179     tests.add('passes/error.html', error='stuff going to stderr')
180     tests.add('passes/image.html')
181     tests.add('passes/audio.html',
182               actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-wav',
183               actual_text=None, expected_text=None,
184               actual_image=None, expected_image=None,
185               actual_checksum=None)
186     tests.add('passes/platform_image.html')
187     tests.add('passes/checksum_in_image.html',
188               expected_image='tEXtchecksum\x00checksum_in_image-checksum')
189     tests.add('passes/skipped/skip.html')
190
191     # Note that here the checksums don't match but the images do, so this test passes "unexpectedly".
192     # See https://bugs.webkit.org/show_bug.cgi?id=69444 .
193     tests.add('failures/unexpected/checksum.html', actual_checksum='checksum_fail-checksum')
194
195     # Text output files contain "\r\n" on Windows.  This may be
196     # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
197     tests.add('passes/text.html',
198               expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n')
199
200     # For reftests.
201     tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True)
202     tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False)
203     tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg', same_image=True)
204     tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html', same_image=True)
205     tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-mismatch.svg', same_image=False)
206     tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False)
207     tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True)
208     tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
209     tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
210     tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True)
211     tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checksum=None, actual_image=None, is_reftest=True)
212     # FIXME: Add a reftest which crashes.
213     tests.add('reftests/foo/test.html')
214     tests.add('reftests/foo/test-ref.html')
215
216     tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc', actual_image='abc')
217     tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc', actual_image='abc')
218     tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='abc', actual_image='abc')
219     tests.add('reftests/foo/multiple-mismatch-failure.html', actual_checksum='abc', actual_image='abc')
220     tests.add('reftests/foo/multiple-both-success.html', actual_checksum='abc', actual_image='abc')
221     tests.add('reftests/foo/multiple-both-failure.html', actual_checksum='abc', actual_image='abc')
222
223     tests.add('reftests/foo/matching-ref.html', actual_checksum='abc', actual_image='abc')
224     tests.add('reftests/foo/mismatching-ref.html', actual_checksum='def', actual_image='def')
225     tests.add('reftests/foo/second-mismatching-ref.html', actual_checksum='ghi', actual_image='ghi')
226
227     # The following files shouldn't be treated as reftests
228     tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.html', same_image=True)
229     tests.add('reftests/foo/reference/bar/common.html')
230     tests.add('reftests/foo/reftest/bar/shared.html')
231
232     tests.add('websocket/tests/passes/text.html')
233
234     # For testing test are properly included from platform directories.
235     tests.add('platform/test-mac-leopard/http/test.html')
236     tests.add('platform/test-win-win7/http/test.html')
237
238     # For --no-http tests, test that platform specific HTTP tests are properly skipped.
239     tests.add('platform/test-snow-leopard/http/test.html')
240     tests.add('platform/test-snow-leopard/websocket/test.html')
241
242     # For testing if perf tests are running in a locked shard.
243     tests.add('perf/foo/test.html')
244     tests.add('perf/foo/test-ref.html')
245
246     # For testing --pixel-test-directories.
247     tests.add('failures/unexpected/pixeldir/image_in_pixeldir.html',
248         actual_image='image_in_pixeldir-pngtEXtchecksum\x00checksum_fail',
249         expected_image='image_in_pixeldir-pngtEXtchecksum\x00checksum-png')
250     tests.add('failures/unexpected/image_not_in_pixeldir.html',
251         actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fail',
252         expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-png')
253
254     # For testing that virtual test suites don't expand names containing themselves
255     # See webkit.org/b/97925 and base_unittest.PortTest.test_tests().
256     tests.add('passes/test-virtual-passes.html')
257     tests.add('passes/passes/test-virtual-passes.html')
258
259     return tests
260
261
262 # Here we use a non-standard location for the layout tests, to ensure that
263 # this works. The path contains a '.' in the name because we've seen bugs
264 # related to this before.
265
266 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests'
267 PERF_TEST_DIR = '/test.checkout/PerformanceTests'
268
269
270 # Here we synthesize an in-memory filesystem from the test list
271 # in order to fully control the test output and to demonstrate that
272 # we don't need a real filesystem to run the tests.
273 def add_unit_tests_to_mock_filesystem(filesystem):
274     # Add the test_expectations file.
275     filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/platform/test')
276     if not filesystem.exists(LAYOUT_TEST_DIR + '/platform/test/TestExpectations'):
277         filesystem.write_text_file(LAYOUT_TEST_DIR + '/platform/test/TestExpectations', """
278 Bug(test) failures/expected/crash.html [ Crash ]
279 Bug(test) failures/expected/image.html [ ImageOnlyFailure ]
280 Bug(test) failures/expected/audio.html [ Failure ]
281 Bug(test) failures/expected/image_checksum.html [ ImageOnlyFailure ]
282 Bug(test) failures/expected/mismatch.html [ ImageOnlyFailure ]
283 Bug(test) failures/expected/missing_check.html [ Missing Pass ]
284 Bug(test) failures/expected/missing_image.html [ Missing Pass ]
285 Bug(test) failures/expected/missing_audio.html [ Missing Pass ]
286 Bug(test) failures/expected/missing_text.html [ Missing Pass ]
287 Bug(test) failures/expected/newlines_leading.html [ Failure ]
288 Bug(test) failures/expected/newlines_trailing.html [ Failure ]
289 Bug(test) failures/expected/newlines_with_excess_CR.html [ Failure ]
290 Bug(test) failures/expected/reftest.html [ ImageOnlyFailure ]
291 Bug(test) failures/expected/text.html [ Failure ]
292 Bug(test) failures/expected/timeout.html [ Timeout ]
293 Bug(test) failures/expected/hang.html [ WontFix ]
294 Bug(test) failures/expected/keyboard.html [ WontFix ]
295 Bug(test) failures/expected/exception.html [ WontFix ]
296 Bug(test) failures/unexpected/pass.html [ Failure ]
297 Bug(test) passes/skipped/skip.html [ Skip ]
298 """)
299
300     filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/reftests/foo')
301     filesystem.write_text_file(LAYOUT_TEST_DIR + '/reftests/foo/reftest.list', """
302 == test.html test-ref.html
303
304 == multiple-match-success.html mismatching-ref.html
305 == multiple-match-success.html matching-ref.html
306 == multiple-match-failure.html mismatching-ref.html
307 == multiple-match-failure.html second-mismatching-ref.html
308 != multiple-mismatch-success.html mismatching-ref.html
309 != multiple-mismatch-success.html second-mismatching-ref.html
310 != multiple-mismatch-failure.html mismatching-ref.html
311 != multiple-mismatch-failure.html matching-ref.html
312 == multiple-both-success.html matching-ref.html
313 == multiple-both-success.html mismatching-ref.html
314 != multiple-both-success.html second-mismatching-ref.html
315 == multiple-both-failure.html matching-ref.html
316 != multiple-both-failure.html second-mismatching-ref.html
317 != multiple-both-failure.html matching-ref.html
318 """)
319
320     # FIXME: This test was only being ignored because of missing a leading '/'.
321     # Fixing the typo causes several tests to assert, so disabling the test entirely.
322     # Add in a file should be ignored by port.find_test_files().
323     #files[LAYOUT_TEST_DIR + '/userscripts/resources/iframe.html'] = 'iframe'
324
325     def add_file(test, suffix, contents):
326         dirname = filesystem.join(LAYOUT_TEST_DIR, test.name[0:test.name.rfind('/')])
327         base = test.base
328         filesystem.maybe_make_directory(dirname)
329         filesystem.write_binary_file(filesystem.join(dirname, base + suffix), contents)
330
331     # Add each test and the expected output, if any.
332     test_list = unit_test_list()
333     for test in test_list.tests.values():
334         add_file(test, test.name[test.name.rfind('.'):], '')
335         if test.is_reftest:
336             continue
337         if test.actual_audio:
338             add_file(test, '-expected.wav', test.expected_audio)
339             continue
340         add_file(test, '-expected.txt', test.expected_text)
341         add_file(test, '-expected.png', test.expected_image)
342
343     filesystem.write_text_file(filesystem.join(LAYOUT_TEST_DIR, 'virtual', 'passes', 'args-expected.txt'), 'args-txt --virtual-arg')
344     # Clear the list of written files so that we can watch what happens during testing.
345     filesystem.clear_written_files()
346
347
348 class TestPort(Port):
349     port_name = 'test'
350     default_port_name = 'test-mac-leopard'
351
352     """Test implementation of the Port interface."""
353     ALL_BASELINE_VARIANTS = (
354         'test-linux-x86_64',
355         'test-mac-snowleopard', 'test-mac-leopard',
356         'test-win-vista', 'test-win-win7', 'test-win-xp',
357     )
358
359     @classmethod
360     def determine_full_port_name(cls, host, options, port_name):
361         if port_name == 'test':
362             return TestPort.default_port_name
363         return port_name
364
365     def __init__(self, host, port_name=None, **kwargs):
366         Port.__init__(self, host, port_name or TestPort.default_port_name, **kwargs)
367         self._tests = unit_test_list()
368         self._flakes = set()
369         self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/TestExpectations'
370         self._results_directory = None
371
372         self._operating_system = 'mac'
373         if self._name.startswith('test-win'):
374             self._operating_system = 'win'
375         elif self._name.startswith('test-linux'):
376             self._operating_system = 'linux'
377
378         version_map = {
379             'test-win-xp': 'xp',
380             'test-win-win7': 'win7',
381             'test-win-vista': 'vista',
382             'test-mac-leopard': 'leopard',
383             'test-mac-snowleopard': 'snowleopard',
384             'test-linux-x86_64': 'lucid',
385         }
386         self._version = version_map[self._name]
387
388     def default_pixel_tests(self):
389         return True
390
391     def _path_to_driver(self):
392         # This routine shouldn't normally be called, but it is called by
393         # the mock_drt Driver. We return something, but make sure it's useless.
394         return 'MOCK _path_to_driver'
395
396     def baseline_search_path(self):
397         search_paths = {
398             'test-mac-snowleopard': ['test-mac-snowleopard'],
399             'test-mac-leopard': ['test-mac-leopard', 'test-mac-snowleopard'],
400             'test-win-win7': ['test-win-win7'],
401             'test-win-vista': ['test-win-vista', 'test-win-win7'],
402             'test-win-xp': ['test-win-xp', 'test-win-vista', 'test-win-win7'],
403             'test-linux-x86_64': ['test-linux', 'test-win-win7'],
404         }
405         return [self._webkit_baseline_path(d) for d in search_paths[self.name()]]
406
407     def default_child_processes(self):
408         return 1
409
410     def worker_startup_delay_secs(self):
411         return 0
412
413     def check_build(self, needs_http):
414         return True
415
416     def check_sys_deps(self, needs_http):
417         return True
418
419     def default_configuration(self):
420         return 'Release'
421
422     def diff_image(self, expected_contents, actual_contents, tolerance=None):
423         diffed = actual_contents != expected_contents
424         if not actual_contents and not expected_contents:
425             return (None, 0, None)
426         if not actual_contents or not expected_contents:
427             return (True, 0, None)
428         if 'ref' in expected_contents:
429             assert tolerance == 0
430         if diffed:
431             return ("< %s\n---\n> %s\n" % (expected_contents, actual_contents), 1, None)
432         return (None, 0, None)
433
434     def layout_tests_dir(self):
435         return LAYOUT_TEST_DIR
436
437     def perf_tests_dir(self):
438         return PERF_TEST_DIR
439
440     def webkit_base(self):
441         return '/test.checkout'
442
443     def _skipped_tests_for_unsupported_features(self, test_list):
444         return set(['failures/expected/skip_text.html',
445                     'failures/unexpected/skip_pass.html',
446                     'virtual/skipped'])
447
448     def name(self):
449         return self._name
450
451     def operating_system(self):
452         return self._operating_system
453
454     def default_results_directory(self):
455         return '/tmp/layout-test-results'
456
457     def setup_test_run(self):
458         pass
459
460     def _driver_class(self):
461         return TestDriver
462
463     def start_http_server(self, additional_dirs=None, number_of_servers=None):
464         pass
465
466     def start_websocket_server(self):
467         pass
468
469     def acquire_http_lock(self):
470         pass
471
472     def stop_http_server(self):
473         pass
474
475     def stop_websocket_server(self):
476         pass
477
478     def release_http_lock(self):
479         pass
480
481     def _path_to_lighttpd(self):
482         return "/usr/sbin/lighttpd"
483
484     def _path_to_lighttpd_modules(self):
485         return "/usr/lib/lighttpd"
486
487     def _path_to_lighttpd_php(self):
488         return "/usr/bin/php-cgi"
489
490     def _path_to_apache(self):
491         return "/usr/sbin/httpd"
492
493     def _path_to_apache_config_file(self):
494         return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', 'httpd.conf')
495
496     def path_to_test_expectations_file(self):
497         return self._expectations_path
498
499     def all_test_configurations(self):
500         """Returns a sequence of the TestConfigurations the port supports."""
501         # By default, we assume we want to test every graphics type in
502         # every configuration on every system.
503         test_configurations = []
504         for version, architecture in self._all_systems():
505             for build_type in self._all_build_types():
506                 test_configurations.append(TestConfiguration(
507                     version=version,
508                     architecture=architecture,
509                     build_type=build_type))
510         return test_configurations
511
512     def _all_systems(self):
513         return (('leopard', 'x86'),
514                 ('snowleopard', 'x86'),
515                 ('xp', 'x86'),
516                 ('vista', 'x86'),
517                 ('win7', 'x86'),
518                 ('lucid', 'x86'),
519                 ('lucid', 'x86_64'))
520
521     def _all_build_types(self):
522         return ('debug', 'release')
523
524     def configuration_specifier_macros(self):
525         """To avoid surprises when introducing new macros, these are intentionally fixed in time."""
526         return {'mac': ['leopard', 'snowleopard'], 'win': ['xp', 'vista', 'win7'], 'linux': ['lucid']}
527
528     def all_baseline_variants(self):
529         return self.ALL_BASELINE_VARIANTS
530
531     def virtual_test_suites(self):
532         return [
533             VirtualTestSuite('virtual/passes', 'passes', ['--virtual-arg']),
534             VirtualTestSuite('virtual/skipped', 'failures/expected', ['--virtual-arg2']),
535         ]
536
537
538 class TestDriver(Driver):
539     """Test/Dummy implementation of the DumpRenderTree interface."""
540     next_pid = 1
541
542     def __init__(self, *args, **kwargs):
543         super(TestDriver, self).__init__(*args, **kwargs)
544         self.started = False
545         self.pid = 0
546
547     def cmd_line(self, pixel_tests, per_test_args):
548         pixel_tests_flag = '-p' if pixel_tests else ''
549         return [self._port._path_to_driver()] + [pixel_tests_flag] + self._port.get_option('additional_drt_flag', []) + per_test_args
550
551     def run_test(self, test_input, stop_when_done):
552         if not self.started:
553             self.started = True
554             self.pid = TestDriver.next_pid
555             TestDriver.next_pid += 1
556
557         start_time = time.time()
558         test_name = test_input.test_name
559         test_args = test_input.args or []
560         test = self._port._tests[test_name]
561         if test.keyboard:
562             raise KeyboardInterrupt
563         if test.exception:
564             raise ValueError('exception from ' + test_name)
565         if test.hang:
566             time.sleep((float(test_input.timeout) * 4) / 1000.0 + 1.0)  # The 1.0 comes from thread_padding_sec in layout_test_runnery.
567
568         audio = None
569         actual_text = test.actual_text
570
571         if 'flaky' in test_name and not test_name in self._port._flakes:
572             self._port._flakes.add(test_name)
573             actual_text = 'flaky text failure'
574
575         if actual_text and test_args and test_name == 'passes/args.html':
576             actual_text = actual_text + ' ' + ' '.join(test_args)
577
578         if test.actual_audio:
579             audio = base64.b64decode(test.actual_audio)
580         crashed_process_name = None
581         crashed_pid = None
582         if test.crash:
583             crashed_process_name = self._port.driver_name()
584             crashed_pid = 1
585         elif test.web_process_crash:
586             crashed_process_name = 'WebProcess'
587             crashed_pid = 2
588
589         crash_log = ''
590         if crashed_process_name:
591             crash_logs = CrashLogs(self._port.host)
592             crash_log = crash_logs.find_newest_log(crashed_process_name, None) or ''
593
594         if stop_when_done:
595             self.stop()
596
597         if test.actual_checksum == test_input.image_hash:
598             image = None
599         else:
600             image = test.actual_image
601         return DriverOutput(actual_text, image, test.actual_checksum, audio,
602             crash=test.crash or test.web_process_crash, crashed_process_name=crashed_process_name,
603             crashed_pid=crashed_pid, crash_log=crash_log,
604             test_time=time.time() - start_time, timeout=test.timeout, error=test.error, pid=self.pid)
605
606     def stop(self):
607         self.started = False