76fea3ba182853f4d5badf54248a7a2d51a84557
[WebKit-https.git] / Tools / Scripts / webkitpy / port / test.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the Google name nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import base64
30 import sys
31 import time
32
33 from webkitpy.port import Port, Driver, DriverOutput
34 from webkitpy.port.base import VirtualTestSuite
35 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
36 from webkitpy.common.system.filesystem_mock import MockFileSystem
37 from webkitpy.common.system.crashlogs import CrashLogs
38
39
40 # This sets basic expectations for a test. Each individual expectation
41 # can be overridden by a keyword argument in TestList.add().
42 class TestInstance(object):
43     def __init__(self, name):
44         self.name = name
45         self.base = name[(name.rfind("/") + 1):name.rfind(".")]
46         self.crash = False
47         self.web_process_crash = False
48         self.exception = False
49         self.hang = False
50         self.keyboard = False
51         self.error = ''
52         self.timeout = False
53         self.is_reftest = False
54
55         # The values of each field are treated as raw byte strings. They
56         # will be converted to unicode strings where appropriate using
57         # FileSystem.read_text_file().
58         self.actual_text = self.base + '-txt'
59         self.actual_checksum = self.base + '-checksum'
60
61         # We add the '\x8a' for the image file to prevent the value from
62         # being treated as UTF-8 (the character is invalid)
63         self.actual_image = self.base + '\x8a' + '-png' + 'tEXtchecksum\x00' + self.actual_checksum
64
65         self.expected_text = self.actual_text
66         self.expected_image = self.actual_image
67
68         self.actual_audio = None
69         self.expected_audio = None
70
71
72 # This is an in-memory list of tests, what we want them to produce, and
73 # what we want to claim are the expected results.
74 class TestList(object):
75     def __init__(self):
76         self.tests = {}
77
78     def add(self, name, **kwargs):
79         test = TestInstance(name)
80         for key, value in kwargs.items():
81             test.__dict__[key] = value
82         self.tests[name] = test
83
84     def add_reftest(self, name, reference_name, same_image):
85         self.add(name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
86         if same_image:
87             self.add(reference_name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
88         else:
89             self.add(reference_name, actual_checksum='yyy', actual_image='YYY', is_reftest=True)
90
91     def keys(self):
92         return self.tests.keys()
93
94     def __contains__(self, item):
95         return item in self.tests
96
97     def __getitem__(self, item):
98         return self.tests[item]
99
100 #
101 # These numbers may need to be updated whenever we add or delete tests.
102 #
103 TOTAL_TESTS = 106
104 TOTAL_SKIPS = 28
105 TOTAL_RETRIES = 14
106
107 UNEXPECTED_PASSES = 6
108 UNEXPECTED_FAILURES = 17
109
110 def unit_test_list():
111     tests = TestList()
112     tests.add('failures/expected/crash.html', crash=True)
113     tests.add('failures/expected/exception.html', exception=True)
114     tests.add('failures/expected/timeout.html', timeout=True)
115     tests.add('failures/expected/hang.html', hang=True)
116     tests.add('failures/expected/missing_text.html', expected_text=None)
117     tests.add('failures/expected/image.html',
118               actual_image='image_fail-pngtEXtchecksum\x00checksum_fail',
119               expected_image='image-pngtEXtchecksum\x00checksum-png')
120     tests.add('failures/expected/image_checksum.html',
121               actual_checksum='image_checksum_fail-checksum',
122               actual_image='image_checksum_fail-png')
123     tests.add('failures/expected/audio.html',
124               actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='audio-wav',
125               actual_text=None, expected_text=None,
126               actual_image=None, expected_image=None,
127               actual_checksum=None)
128     tests.add('failures/expected/keyboard.html', keyboard=True)
129     tests.add('failures/expected/missing_check.html',
130               expected_image='missing_check-png')
131     tests.add('failures/expected/missing_image.html', expected_image=None)
132     tests.add('failures/expected/missing_audio.html', expected_audio=None,
133               actual_text=None, expected_text=None,
134               actual_image=None, expected_image=None,
135               actual_checksum=None)
136     tests.add('failures/expected/missing_text.html', expected_text=None)
137     tests.add('failures/expected/newlines_leading.html',
138               expected_text="\nfoo\n", actual_text="foo\n")
139     tests.add('failures/expected/newlines_trailing.html',
140               expected_text="foo\n\n", actual_text="foo\n")
141     tests.add('failures/expected/newlines_with_excess_CR.html',
142               expected_text="foo\r\r\r\n", actual_text="foo\n")
143     tests.add('failures/expected/text.html', actual_text='text_fail-png')
144     tests.add('failures/expected/skip_text.html', actual_text='text diff')
145     tests.add('failures/flaky/text.html')
146     tests.add('failures/unexpected/missing_text.html', expected_text=None)
147     tests.add('failures/unexpected/missing_check.html', expected_image='missing-check-png')
148     tests.add('failures/unexpected/missing_image.html', expected_image=None)
149     tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="""layer at (0,0) size 800x600
150   RenderView at (0,0) size 800x600
151 layer at (0,0) size 800x34
152   RenderBlock {HTML} at (0,0) size 800x34
153     RenderBody {BODY} at (8,8) size 784x18
154       RenderText {#text} at (0,0) size 133x18
155         text run at (0,0) width 133: "This is an image test!"
156 """, expected_text=None)
157     tests.add('failures/unexpected/crash.html', crash=True)
158     tests.add('failures/unexpected/crash-with-stderr.html', crash=True,
159               error="mock-std-error-output")
160     tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_process_crash=True,
161               error="mock-std-error-output")
162     tests.add('failures/unexpected/pass.html')
163     tests.add('failures/unexpected/text-checksum.html',
164               actual_text='text-checksum_fail-txt',
165               actual_checksum='text-checksum_fail-checksum')
166     tests.add('failures/unexpected/text-image-checksum.html',
167               actual_text='text-image-checksum_fail-txt',
168               actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum_fail',
169               actual_checksum='text-image-checksum_fail-checksum')
170     tests.add('failures/unexpected/checksum-with-matching-image.html',
171               actual_checksum='text-image-checksum_fail-checksum')
172     tests.add('failures/unexpected/skip_pass.html')
173     tests.add('failures/unexpected/text.html', actual_text='text_fail-txt')
174     tests.add('failures/unexpected/timeout.html', timeout=True)
175     tests.add('http/tests/passes/text.html')
176     tests.add('http/tests/passes/image.html')
177     tests.add('http/tests/ssl/text.html')
178     tests.add('passes/args.html')
179     tests.add('passes/error.html', error='stuff going to stderr')
180     tests.add('passes/image.html')
181     tests.add('passes/audio.html',
182               actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-wav',
183               actual_text=None, expected_text=None,
184               actual_image=None, expected_image=None,
185               actual_checksum=None)
186     tests.add('passes/platform_image.html')
187     tests.add('passes/checksum_in_image.html',
188               expected_image='tEXtchecksum\x00checksum_in_image-checksum')
189     tests.add('passes/skipped/skip.html')
190
191     # Note that here the checksums don't match but the images do, so this test passes "unexpectedly".
192     # See https://bugs.webkit.org/show_bug.cgi?id=69444 .
193     tests.add('failures/unexpected/checksum.html', actual_checksum='checksum_fail-checksum')
194
195     # Text output files contain "\r\n" on Windows.  This may be
196     # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
197     tests.add('passes/text.html',
198               expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n')
199
200     # For reftests.
201     tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True)
202     tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False)
203     tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg', same_image=True)
204     tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html', same_image=True)
205     tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-mismatch.svg', same_image=False)
206     tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False)
207     tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True)
208     tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
209     tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
210     tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True)
211     tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checksum=None, actual_image=None, is_reftest=True)
212     # FIXME: Add a reftest which crashes.
213     tests.add('reftests/foo/test.html')
214     tests.add('reftests/foo/test-ref.html')
215
216     tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc', actual_image='abc')
217     tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc', actual_image='abc')
218     tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='abc', actual_image='abc')
219     tests.add('reftests/foo/multiple-mismatch-failure.html', actual_checksum='abc', actual_image='abc')
220     tests.add('reftests/foo/multiple-both-success.html', actual_checksum='abc', actual_image='abc')
221     tests.add('reftests/foo/multiple-both-failure.html', actual_checksum='abc', actual_image='abc')
222
223     tests.add('reftests/foo/matching-ref.html', actual_checksum='abc', actual_image='abc')
224     tests.add('reftests/foo/mismatching-ref.html', actual_checksum='def', actual_image='def')
225     tests.add('reftests/foo/second-mismatching-ref.html', actual_checksum='ghi', actual_image='ghi')
226
227     # The following files shouldn't be treated as reftests
228     tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.html', same_image=True)
229     tests.add('reftests/foo/reference/bar/common.html')
230     tests.add('reftests/foo/reftest/bar/shared.html')
231
232     tests.add('websocket/tests/passes/text.html')
233
234     # For testing test are properly included from platform directories.
235     tests.add('platform/test-mac-leopard/http/test.html')
236     tests.add('platform/test-win-win7/http/test.html')
237
238     # For --no-http tests, test that platform specific HTTP tests are properly skipped.
239     tests.add('platform/test-snow-leopard/http/test.html')
240     tests.add('platform/test-snow-leopard/websocket/test.html')
241
242     # For testing if perf tests are running in a locked shard.
243     tests.add('perf/foo/test.html')
244     tests.add('perf/foo/test-ref.html')
245
246     # For testing --pixel-test-directories.
247     tests.add('failures/unexpected/pixeldir/image_in_pixeldir.html',
248         actual_image='image_in_pixeldir-pngtEXtchecksum\x00checksum_fail',
249         expected_image='image_in_pixeldir-pngtEXtchecksum\x00checksum-png')
250     tests.add('failures/unexpected/image_not_in_pixeldir.html',
251         actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fail',
252         expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-png')
253
254     # For testing that virtual test suites don't expand names containing themselves
255     # See webkit.org/b/97925 and base_unittest.PortTest.test_tests().
256     tests.add('passes/test-virtual-passes.html')
257     tests.add('passes/passes/test-virtual-passes.html')
258
259     return tests
260
261
262 # Here we use a non-standard location for the layout tests, to ensure that
263 # this works. The path contains a '.' in the name because we've seen bugs
264 # related to this before.
265
266 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests'
267 PERF_TEST_DIR = '/test.checkout/PerformanceTests'
268
269
270 # Here we synthesize an in-memory filesystem from the test list
271 # in order to fully control the test output and to demonstrate that
272 # we don't need a real filesystem to run the tests.
273 def add_unit_tests_to_mock_filesystem(filesystem):
274     # Add the test_expectations file.
275     filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/platform/test')
276     if not filesystem.exists(LAYOUT_TEST_DIR + '/platform/test/TestExpectations'):
277         filesystem.write_text_file(LAYOUT_TEST_DIR + '/platform/test/TestExpectations', """
278 Bug(test) failures/expected/crash.html [ Crash ]
279 Bug(test) failures/expected/image.html [ ImageOnlyFailure ]
280 Bug(test) failures/expected/audio.html [ Failure ]
281 Bug(test) failures/expected/image_checksum.html [ ImageOnlyFailure ]
282 Bug(test) failures/expected/mismatch.html [ ImageOnlyFailure ]
283 Bug(test) failures/expected/missing_check.html [ Missing Pass ]
284 Bug(test) failures/expected/missing_image.html [ Missing Pass ]
285 Bug(test) failures/expected/missing_audio.html [ Missing Pass ]
286 Bug(test) failures/expected/missing_text.html [ Missing Pass ]
287 Bug(test) failures/expected/newlines_leading.html [ Failure ]
288 Bug(test) failures/expected/newlines_trailing.html [ Failure ]
289 Bug(test) failures/expected/newlines_with_excess_CR.html [ Failure ]
290 Bug(test) failures/expected/reftest.html [ ImageOnlyFailure ]
291 Bug(test) failures/expected/text.html [ Failure ]
292 Bug(test) failures/expected/timeout.html [ Timeout ]
293 Bug(test) failures/expected/hang.html [ WontFix ]
294 Bug(test) failures/expected/keyboard.html [ WontFix ]
295 Bug(test) failures/expected/exception.html [ WontFix ]
296 Bug(test) failures/unexpected/pass.html [ Failure ]
297 Bug(test) passes/skipped/skip.html [ Skip ]
298 """)
299
300     filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/reftests/foo')
301     filesystem.write_text_file(LAYOUT_TEST_DIR + '/reftests/foo/reftest.list', """
302 == test.html test-ref.html
303
304 == multiple-match-success.html mismatching-ref.html
305 == multiple-match-success.html matching-ref.html
306 == multiple-match-failure.html mismatching-ref.html
307 == multiple-match-failure.html second-mismatching-ref.html
308 != multiple-mismatch-success.html mismatching-ref.html
309 != multiple-mismatch-success.html second-mismatching-ref.html
310 != multiple-mismatch-failure.html mismatching-ref.html
311 != multiple-mismatch-failure.html matching-ref.html
312 == multiple-both-success.html matching-ref.html
313 == multiple-both-success.html mismatching-ref.html
314 != multiple-both-success.html second-mismatching-ref.html
315 == multiple-both-failure.html matching-ref.html
316 != multiple-both-failure.html second-mismatching-ref.html
317 != multiple-both-failure.html matching-ref.html
318 """)
319
320     # FIXME: This test was only being ignored because of missing a leading '/'.
321     # Fixing the typo causes several tests to assert, so disabling the test entirely.
322     # Add in a file should be ignored by port.find_test_files().
323     #files[LAYOUT_TEST_DIR + '/userscripts/resources/iframe.html'] = 'iframe'
324
325     def add_file(test, suffix, contents):
326         dirname = filesystem.join(LAYOUT_TEST_DIR, test.name[0:test.name.rfind('/')])
327         base = test.base
328         filesystem.maybe_make_directory(dirname)
329         filesystem.write_binary_file(filesystem.join(dirname, base + suffix), contents)
330
331     # Add each test and the expected output, if any.
332     test_list = unit_test_list()
333     for test in test_list.tests.values():
334         add_file(test, test.name[test.name.rfind('.'):], '')
335         if test.is_reftest:
336             continue
337         if test.actual_audio:
338             add_file(test, '-expected.wav', test.expected_audio)
339             continue
340         add_file(test, '-expected.txt', test.expected_text)
341         add_file(test, '-expected.png', test.expected_image)
342
343     filesystem.write_text_file(filesystem.join(LAYOUT_TEST_DIR, 'virtual', 'passes', 'args-expected.txt'), 'args-txt --virtual-arg')
344     # Clear the list of written files so that we can watch what happens during testing.
345     filesystem.clear_written_files()
346
347
348 class TestPort(Port):
349     port_name = 'test'
350     default_port_name = 'test-mac-leopard'
351
352     """Test implementation of the Port interface."""
353     ALL_BASELINE_VARIANTS = (
354         'test-linux-x86_64',
355         'test-mac-snowleopard', 'test-mac-leopard',
356         'test-win-vista', 'test-win-win7', 'test-win-xp',
357     )
358
359     @classmethod
360     def determine_full_port_name(cls, host, options, port_name):
361         if port_name == 'test':
362             return TestPort.default_port_name
363         return port_name
364
365     def __init__(self, host, port_name=None, **kwargs):
366         Port.__init__(self, host, port_name or TestPort.default_port_name, **kwargs)
367         self._tests = unit_test_list()
368         self._flakes = set()
369         self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/TestExpectations'
370         self._results_directory = None
371
372         self._operating_system = 'mac'
373         if self._name.startswith('test-win'):
374             self._operating_system = 'win'
375         elif self._name.startswith('test-linux'):
376             self._operating_system = 'linux'
377
378         version_map = {
379             'test-win-xp': 'xp',
380             'test-win-win7': 'win7',
381             'test-win-vista': 'vista',
382             'test-mac-leopard': 'leopard',
383             'test-mac-snowleopard': 'snowleopard',
384             'test-linux-x86_64': 'lucid',
385         }
386         self._version = version_map[self._name]
387
388     def default_pixel_tests(self):
389         return True
390
391     def _path_to_driver(self):
392         # This routine shouldn't normally be called, but it is called by
393         # the mock_drt Driver. We return something, but make sure it's useless.
394         return 'MOCK _path_to_driver'
395
396     def baseline_search_path(self):
397         search_paths = {
398             'test-mac-snowleopard': ['test-mac-snowleopard'],
399             'test-mac-leopard': ['test-mac-leopard', 'test-mac-snowleopard'],
400             'test-win-win7': ['test-win-win7'],
401             'test-win-vista': ['test-win-vista', 'test-win-win7'],
402             'test-win-xp': ['test-win-xp', 'test-win-vista', 'test-win-win7'],
403             'test-linux-x86_64': ['test-linux', 'test-win-win7'],
404         }
405         return [self._webkit_baseline_path(d) for d in search_paths[self.name()]]
406
407     def default_child_processes(self):
408         return 1
409
410     def worker_startup_delay_secs(self):
411         return 0
412
413     def check_build(self, needs_http):
414         return True
415
416     def check_sys_deps(self, needs_http):
417         return True
418
419     def default_configuration(self):
420         return 'Release'
421
422     def diff_image(self, expected_contents, actual_contents, tolerance=None):
423         diffed = actual_contents != expected_contents
424         if not actual_contents and not expected_contents:
425             return (None, 0, None)
426         if not actual_contents or not expected_contents:
427             return (True, 0, None)
428         if 'ref' in expected_contents:
429             assert tolerance == 0
430         if diffed:
431             return ("< %s\n---\n> %s\n" % (expected_contents, actual_contents), 1, None)
432         return (None, 0, None)
433
434     def layout_tests_dir(self):
435         return LAYOUT_TEST_DIR
436
437     def perf_tests_dir(self):
438         return PERF_TEST_DIR
439
440     def webkit_base(self):
441         return '/test.checkout'
442
443     def _skipped_tests_for_unsupported_features(self, test_list):
444         return set(['failures/expected/skip_text.html',
445                     'failures/unexpected/skip_pass.html',
446                     'virtual/skipped'])
447
448     def name(self):
449         return self._name
450
451     def operating_system(self):
452         return self._operating_system
453
454     def _path_to_wdiff(self):
455         return None
456
457     def default_results_directory(self):
458         return '/tmp/layout-test-results'
459
460     def setup_test_run(self):
461         pass
462
463     def _driver_class(self):
464         return TestDriver
465
466     def start_http_server(self, additional_dirs=None, number_of_servers=None):
467         pass
468
469     def start_websocket_server(self):
470         pass
471
472     def acquire_http_lock(self):
473         pass
474
475     def stop_http_server(self):
476         pass
477
478     def stop_websocket_server(self):
479         pass
480
481     def release_http_lock(self):
482         pass
483
484     def _path_to_lighttpd(self):
485         return "/usr/sbin/lighttpd"
486
487     def _path_to_lighttpd_modules(self):
488         return "/usr/lib/lighttpd"
489
490     def _path_to_lighttpd_php(self):
491         return "/usr/bin/php-cgi"
492
493     def _path_to_apache(self):
494         return "/usr/sbin/httpd"
495
496     def _path_to_apache_config_file(self):
497         return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', 'httpd.conf')
498
499     def path_to_test_expectations_file(self):
500         return self._expectations_path
501
502     def all_test_configurations(self):
503         """Returns a sequence of the TestConfigurations the port supports."""
504         # By default, we assume we want to test every graphics type in
505         # every configuration on every system.
506         test_configurations = []
507         for version, architecture in self._all_systems():
508             for build_type in self._all_build_types():
509                 test_configurations.append(TestConfiguration(
510                     version=version,
511                     architecture=architecture,
512                     build_type=build_type))
513         return test_configurations
514
515     def _all_systems(self):
516         return (('leopard', 'x86'),
517                 ('snowleopard', 'x86'),
518                 ('xp', 'x86'),
519                 ('vista', 'x86'),
520                 ('win7', 'x86'),
521                 ('lucid', 'x86'),
522                 ('lucid', 'x86_64'))
523
524     def _all_build_types(self):
525         return ('debug', 'release')
526
527     def configuration_specifier_macros(self):
528         """To avoid surprises when introducing new macros, these are intentionally fixed in time."""
529         return {'mac': ['leopard', 'snowleopard'], 'win': ['xp', 'vista', 'win7'], 'linux': ['lucid']}
530
531     def all_baseline_variants(self):
532         return self.ALL_BASELINE_VARIANTS
533
534     def virtual_test_suites(self):
535         return [
536             VirtualTestSuite('virtual/passes', 'passes', ['--virtual-arg']),
537             VirtualTestSuite('virtual/skipped', 'failures/expected', ['--virtual-arg2']),
538         ]
539
540
541 class TestDriver(Driver):
542     """Test/Dummy implementation of the DumpRenderTree interface."""
543     next_pid = 1
544
545     def __init__(self, *args, **kwargs):
546         super(TestDriver, self).__init__(*args, **kwargs)
547         self.started = False
548         self.pid = 0
549
550     def cmd_line(self, pixel_tests, per_test_args):
551         pixel_tests_flag = '-p' if pixel_tests else ''
552         return [self._port._path_to_driver()] + [pixel_tests_flag] + self._port.get_option('additional_drt_flag', []) + per_test_args
553
554     def run_test(self, test_input, stop_when_done):
555         if not self.started:
556             self.started = True
557             self.pid = TestDriver.next_pid
558             TestDriver.next_pid += 1
559
560         start_time = time.time()
561         test_name = test_input.test_name
562         test_args = test_input.args or []
563         test = self._port._tests[test_name]
564         if test.keyboard:
565             raise KeyboardInterrupt
566         if test.exception:
567             raise ValueError('exception from ' + test_name)
568         if test.hang:
569             time.sleep((float(test_input.timeout) * 4) / 1000.0 + 1.0)  # The 1.0 comes from thread_padding_sec in layout_test_runnery.
570
571         audio = None
572         actual_text = test.actual_text
573
574         if 'flaky' in test_name and not test_name in self._port._flakes:
575             self._port._flakes.add(test_name)
576             actual_text = 'flaky text failure'
577
578         if actual_text and test_args and test_name == 'passes/args.html':
579             actual_text = actual_text + ' ' + ' '.join(test_args)
580
581         if test.actual_audio:
582             audio = base64.b64decode(test.actual_audio)
583         crashed_process_name = None
584         crashed_pid = None
585         if test.crash:
586             crashed_process_name = self._port.driver_name()
587             crashed_pid = 1
588         elif test.web_process_crash:
589             crashed_process_name = 'WebProcess'
590             crashed_pid = 2
591
592         crash_log = ''
593         if crashed_process_name:
594             crash_logs = CrashLogs(self._port.host)
595             crash_log = crash_logs.find_newest_log(crashed_process_name, None) or ''
596
597         if stop_when_done:
598             self.stop()
599
600         if test.actual_checksum == test_input.image_hash:
601             image = None
602         else:
603             image = test.actual_image
604         return DriverOutput(actual_text, image, test.actual_checksum, audio,
605             crash=test.crash or test.web_process_crash, crashed_process_name=crashed_process_name,
606             crashed_pid=crashed_pid, crash_log=crash_log,
607             test_time=time.time() - start_time, timeout=test.timeout, error=test.error, pid=self.pid)
608
609     def stop(self):
610         self.started = False