26f39c3fdf85c06d733f8dcd54c95f2a9d09a51f
[WebKit-https.git] / Tools / Scripts / webkitpy / port / test.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the Google name nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import base64
30 import sys
31 import time
32
33 from webkitpy.port import Port, Driver, DriverOutput
34 from webkitpy.port.base import VirtualTestSuite
35 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
36 from webkitpy.common.system.filesystem_mock import MockFileSystem
37 from webkitpy.common.system.crashlogs import CrashLogs
38
39
40 # This sets basic expectations for a test. Each individual expectation
41 # can be overridden by a keyword argument in TestList.add().
42 class TestInstance(object):
43     def __init__(self, name):
44         self.name = name
45         self.base = name[(name.rfind("/") + 1):name.rfind(".")]
46         self.crash = False
47         self.web_process_crash = False
48         self.exception = False
49         self.hang = False
50         self.keyboard = False
51         self.error = ''
52         self.timeout = False
53         self.is_reftest = False
54
55         # The values of each field are treated as raw byte strings. They
56         # will be converted to unicode strings where appropriate using
57         # FileSystem.read_text_file().
58         self.actual_text = self.base + '-txt'
59         self.actual_checksum = self.base + '-checksum'
60
61         # We add the '\x8a' for the image file to prevent the value from
62         # being treated as UTF-8 (the character is invalid)
63         self.actual_image = self.base + '\x8a' + '-png' + 'tEXtchecksum\x00' + self.actual_checksum
64
65         self.expected_text = self.actual_text
66         self.expected_image = self.actual_image
67
68         self.actual_audio = None
69         self.expected_audio = None
70
71
72 # This is an in-memory list of tests, what we want them to produce, and
73 # what we want to claim are the expected results.
74 class TestList(object):
75     def __init__(self):
76         self.tests = {}
77
78     def add(self, name, **kwargs):
79         test = TestInstance(name)
80         for key, value in kwargs.items():
81             test.__dict__[key] = value
82         self.tests[name] = test
83
84     def add_reftest(self, name, reference_name, same_image):
85         self.add(name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
86         if same_image:
87             self.add(reference_name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
88         else:
89             self.add(reference_name, actual_checksum='yyy', actual_image='YYY', is_reftest=True)
90
91     def keys(self):
92         return self.tests.keys()
93
94     def __contains__(self, item):
95         return item in self.tests
96
97     def __getitem__(self, item):
98         return self.tests[item]
99
100 #
101 # These numbers may need to be updated whenever we add or delete tests.
102 #
103 TOTAL_TESTS = 107
104 TOTAL_SKIPS = 28
105 TOTAL_RETRIES = 14
106
107 UNEXPECTED_PASSES = 7
108 UNEXPECTED_FAILURES = 17
109
110 def unit_test_list():
111     tests = TestList()
112     tests.add('failures/expected/crash.html', crash=True)
113     tests.add('failures/expected/exception.html', exception=True)
114     tests.add('failures/expected/timeout.html', timeout=True)
115     tests.add('failures/expected/hang.html', hang=True)
116     tests.add('failures/expected/missing_text.html', expected_text=None)
117     tests.add('failures/expected/image.html',
118               actual_image='image_fail-pngtEXtchecksum\x00checksum_fail',
119               expected_image='image-pngtEXtchecksum\x00checksum-png')
120     tests.add('failures/expected/image_checksum.html',
121               actual_checksum='image_checksum_fail-checksum',
122               actual_image='image_checksum_fail-png')
123     tests.add('failures/expected/audio.html',
124               actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='audio-wav',
125               actual_text=None, expected_text=None,
126               actual_image=None, expected_image=None,
127               actual_checksum=None)
128     tests.add('failures/expected/keyboard.html', keyboard=True)
129     tests.add('failures/expected/missing_check.html',
130               expected_image='missing_check-png')
131     tests.add('failures/expected/missing_image.html', expected_image=None)
132     tests.add('failures/expected/missing_audio.html', expected_audio=None,
133               actual_text=None, expected_text=None,
134               actual_image=None, expected_image=None,
135               actual_checksum=None)
136     tests.add('failures/expected/missing_text.html', expected_text=None)
137     tests.add('failures/expected/newlines_leading.html',
138               expected_text="\nfoo\n", actual_text="foo\n")
139     tests.add('failures/expected/newlines_trailing.html',
140               expected_text="foo\n\n", actual_text="foo\n")
141     tests.add('failures/expected/newlines_with_excess_CR.html',
142               expected_text="foo\r\r\r\n", actual_text="foo\n")
143     tests.add('failures/expected/text.html', actual_text='text_fail-png')
144     tests.add('failures/expected/skip_text.html', actual_text='text diff')
145     tests.add('failures/flaky/text.html')
146     tests.add('failures/unexpected/missing_text.html', expected_text=None)
147     tests.add('failures/unexpected/missing_check.html', expected_image='missing-check-png')
148     tests.add('failures/unexpected/missing_image.html', expected_image=None)
149     tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="""layer at (0,0) size 800x600
150   RenderView at (0,0) size 800x600
151 layer at (0,0) size 800x34
152   RenderBlock {HTML} at (0,0) size 800x34
153     RenderBody {BODY} at (8,8) size 784x18
154       RenderText {#text} at (0,0) size 133x18
155         text run at (0,0) width 133: "This is an image test!"
156 """, expected_text=None)
157     tests.add('failures/unexpected/crash.html', crash=True)
158     tests.add('failures/unexpected/crash-with-stderr.html', crash=True,
159               error="mock-std-error-output")
160     tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_process_crash=True,
161               error="mock-std-error-output")
162     tests.add('failures/unexpected/pass.html')
163     tests.add('failures/unexpected/text-checksum.html',
164               actual_text='text-checksum_fail-txt',
165               actual_checksum='text-checksum_fail-checksum')
166     tests.add('failures/unexpected/text-image-checksum.html',
167               actual_text='text-image-checksum_fail-txt',
168               actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum_fail',
169               actual_checksum='text-image-checksum_fail-checksum')
170     tests.add('failures/unexpected/text-image-missing.html',
171               actual_text='text-image-checksum_fail-txt',
172               expected_image=None)
173     tests.add('failures/unexpected/checksum-with-matching-image.html',
174               actual_checksum='text-image-checksum_fail-checksum')
175     tests.add('failures/unexpected/skip_pass.html')
176     tests.add('failures/unexpected/text.html', actual_text='text_fail-txt')
177     tests.add('failures/unexpected/timeout.html', timeout=True)
178     tests.add('http/tests/passes/text.html')
179     tests.add('http/tests/passes/image.html')
180     tests.add('http/tests/ssl/text.html')
181     tests.add('passes/args.html')
182     tests.add('passes/error.html', error='stuff going to stderr')
183     tests.add('passes/image.html')
184     tests.add('passes/audio.html',
185               actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-wav',
186               actual_text=None, expected_text=None,
187               actual_image=None, expected_image=None,
188               actual_checksum=None)
189     tests.add('passes/platform_image.html')
190     tests.add('passes/checksum_in_image.html',
191               expected_image='tEXtchecksum\x00checksum_in_image-checksum')
192     tests.add('passes/skipped/skip.html')
193
194     # Note that here the checksums don't match but the images do, so this test passes "unexpectedly".
195     # See https://bugs.webkit.org/show_bug.cgi?id=69444 .
196     tests.add('failures/unexpected/checksum.html', actual_checksum='checksum_fail-checksum')
197
198     # Text output files contain "\r\n" on Windows.  This may be
199     # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
200     tests.add('passes/text.html',
201               expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n')
202
203     # For reftests.
204     tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True)
205     tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False)
206     tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg', same_image=True)
207     tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html', same_image=True)
208     tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-mismatch.svg', same_image=False)
209     tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False)
210     tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True)
211     tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
212     tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
213     tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True)
214     tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checksum=None, actual_image=None, is_reftest=True)
215     # FIXME: Add a reftest which crashes.
216     tests.add('reftests/foo/test.html')
217     tests.add('reftests/foo/test-ref.html')
218
219     tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc', actual_image='abc')
220     tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc', actual_image='abc')
221     tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='abc', actual_image='abc')
222     tests.add('reftests/foo/multiple-mismatch-failure.html', actual_checksum='abc', actual_image='abc')
223     tests.add('reftests/foo/multiple-both-success.html', actual_checksum='abc', actual_image='abc')
224     tests.add('reftests/foo/multiple-both-failure.html', actual_checksum='abc', actual_image='abc')
225
226     tests.add('reftests/foo/matching-ref.html', actual_checksum='abc', actual_image='abc')
227     tests.add('reftests/foo/mismatching-ref.html', actual_checksum='def', actual_image='def')
228     tests.add('reftests/foo/second-mismatching-ref.html', actual_checksum='ghi', actual_image='ghi')
229
230     # The following files shouldn't be treated as reftests
231     tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.html', same_image=True)
232     tests.add('reftests/foo/reference/bar/common.html')
233     tests.add('reftests/foo/reftest/bar/shared.html')
234
235     tests.add('websocket/tests/passes/text.html')
236
237     # For testing test are properly included from platform directories.
238     tests.add('platform/test-mac-leopard/http/test.html')
239     tests.add('platform/test-win-win7/http/test.html')
240
241     # For --no-http tests, test that platform specific HTTP tests are properly skipped.
242     tests.add('platform/test-snow-leopard/http/test.html')
243     tests.add('platform/test-snow-leopard/websocket/test.html')
244
245     # For testing if perf tests are running in a locked shard.
246     tests.add('perf/foo/test.html')
247     tests.add('perf/foo/test-ref.html')
248
249     # For testing --pixel-test-directories.
250     tests.add('failures/unexpected/pixeldir/image_in_pixeldir.html',
251         actual_image='image_in_pixeldir-pngtEXtchecksum\x00checksum_fail',
252         expected_image='image_in_pixeldir-pngtEXtchecksum\x00checksum-png')
253     tests.add('failures/unexpected/image_not_in_pixeldir.html',
254         actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fail',
255         expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-png')
256
257     # For testing that virtual test suites don't expand names containing themselves
258     # See webkit.org/b/97925 and base_unittest.PortTest.test_tests().
259     tests.add('passes/test-virtual-passes.html')
260     tests.add('passes/passes/test-virtual-passes.html')
261
262     return tests
263
264
265 # Here we use a non-standard location for the layout tests, to ensure that
266 # this works. The path contains a '.' in the name because we've seen bugs
267 # related to this before.
268
269 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests'
270 PERF_TEST_DIR = '/test.checkout/PerformanceTests'
271
272
273 # Here we synthesize an in-memory filesystem from the test list
274 # in order to fully control the test output and to demonstrate that
275 # we don't need a real filesystem to run the tests.
276 def add_unit_tests_to_mock_filesystem(filesystem):
277     # Add the test_expectations file.
278     filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/platform/test')
279     if not filesystem.exists(LAYOUT_TEST_DIR + '/platform/test/TestExpectations'):
280         filesystem.write_text_file(LAYOUT_TEST_DIR + '/platform/test/TestExpectations', """
281 Bug(test) failures/expected/crash.html [ Crash ]
282 Bug(test) failures/expected/image.html [ ImageOnlyFailure ]
283 Bug(test) failures/expected/audio.html [ Failure ]
284 Bug(test) failures/expected/image_checksum.html [ ImageOnlyFailure ]
285 Bug(test) failures/expected/mismatch.html [ ImageOnlyFailure ]
286 Bug(test) failures/expected/missing_check.html [ Missing Pass ]
287 Bug(test) failures/expected/missing_image.html [ Missing Pass ]
288 Bug(test) failures/expected/missing_audio.html [ Missing Pass ]
289 Bug(test) failures/expected/missing_text.html [ Missing Pass ]
290 Bug(test) failures/expected/newlines_leading.html [ Failure ]
291 Bug(test) failures/expected/newlines_trailing.html [ Failure ]
292 Bug(test) failures/expected/newlines_with_excess_CR.html [ Failure ]
293 Bug(test) failures/expected/reftest.html [ ImageOnlyFailure ]
294 Bug(test) failures/expected/text.html [ Failure ]
295 Bug(test) failures/expected/timeout.html [ Timeout ]
296 Bug(test) failures/expected/hang.html [ WontFix ]
297 Bug(test) failures/expected/keyboard.html [ WontFix ]
298 Bug(test) failures/expected/exception.html [ WontFix ]
299 Bug(test) failures/unexpected/pass.html [ Failure ]
300 Bug(test) passes/skipped/skip.html [ Skip ]
301 """)
302
303     filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/reftests/foo')
304     filesystem.write_text_file(LAYOUT_TEST_DIR + '/reftests/foo/reftest.list', """
305 == test.html test-ref.html
306
307 == multiple-match-success.html mismatching-ref.html
308 == multiple-match-success.html matching-ref.html
309 == multiple-match-failure.html mismatching-ref.html
310 == multiple-match-failure.html second-mismatching-ref.html
311 != multiple-mismatch-success.html mismatching-ref.html
312 != multiple-mismatch-success.html second-mismatching-ref.html
313 != multiple-mismatch-failure.html mismatching-ref.html
314 != multiple-mismatch-failure.html matching-ref.html
315 == multiple-both-success.html matching-ref.html
316 == multiple-both-success.html mismatching-ref.html
317 != multiple-both-success.html second-mismatching-ref.html
318 == multiple-both-failure.html matching-ref.html
319 != multiple-both-failure.html second-mismatching-ref.html
320 != multiple-both-failure.html matching-ref.html
321 """)
322
323     # FIXME: This test was only being ignored because of missing a leading '/'.
324     # Fixing the typo causes several tests to assert, so disabling the test entirely.
325     # Add in a file should be ignored by port.find_test_files().
326     #files[LAYOUT_TEST_DIR + '/userscripts/resources/iframe.html'] = 'iframe'
327
328     def add_file(test, suffix, contents):
329         dirname = filesystem.join(LAYOUT_TEST_DIR, test.name[0:test.name.rfind('/')])
330         base = test.base
331         filesystem.maybe_make_directory(dirname)
332         filesystem.write_binary_file(filesystem.join(dirname, base + suffix), contents)
333
334     # Add each test and the expected output, if any.
335     test_list = unit_test_list()
336     for test in test_list.tests.values():
337         add_file(test, test.name[test.name.rfind('.'):], '')
338         if test.is_reftest:
339             continue
340         if test.actual_audio:
341             add_file(test, '-expected.wav', test.expected_audio)
342             continue
343         add_file(test, '-expected.txt', test.expected_text)
344         add_file(test, '-expected.png', test.expected_image)
345
346     filesystem.write_text_file(filesystem.join(LAYOUT_TEST_DIR, 'virtual', 'passes', 'args-expected.txt'), 'args-txt --virtual-arg')
347     # Clear the list of written files so that we can watch what happens during testing.
348     filesystem.clear_written_files()
349
350
351 class TestPort(Port):
352     port_name = 'test'
353     default_port_name = 'test-mac-leopard'
354
355     """Test implementation of the Port interface."""
356     ALL_BASELINE_VARIANTS = (
357         'test-linux-x86_64',
358         'test-mac-snowleopard', 'test-mac-leopard',
359         'test-win-vista', 'test-win-win7', 'test-win-xp',
360     )
361
362     @classmethod
363     def determine_full_port_name(cls, host, options, port_name):
364         if port_name == 'test':
365             return TestPort.default_port_name
366         return port_name
367
368     def __init__(self, host, port_name=None, **kwargs):
369         Port.__init__(self, host, port_name or TestPort.default_port_name, **kwargs)
370         self._tests = unit_test_list()
371         self._flakes = set()
372         self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/TestExpectations'
373         self._results_directory = None
374
375         self._operating_system = 'mac'
376         if self._name.startswith('test-win'):
377             self._operating_system = 'win'
378         elif self._name.startswith('test-linux'):
379             self._operating_system = 'linux'
380
381         version_map = {
382             'test-win-xp': 'xp',
383             'test-win-win7': 'win7',
384             'test-win-vista': 'vista',
385             'test-mac-leopard': 'leopard',
386             'test-mac-snowleopard': 'snowleopard',
387             'test-linux-x86_64': 'lucid',
388         }
389         self._version = version_map[self._name]
390
391     def default_pixel_tests(self):
392         return True
393
394     def _path_to_driver(self):
395         # This routine shouldn't normally be called, but it is called by
396         # the mock_drt Driver. We return something, but make sure it's useless.
397         return 'MOCK _path_to_driver'
398
399     def baseline_search_path(self):
400         search_paths = {
401             'test-mac-snowleopard': ['test-mac-snowleopard'],
402             'test-mac-leopard': ['test-mac-leopard', 'test-mac-snowleopard'],
403             'test-win-win7': ['test-win-win7'],
404             'test-win-vista': ['test-win-vista', 'test-win-win7'],
405             'test-win-xp': ['test-win-xp', 'test-win-vista', 'test-win-win7'],
406             'test-linux-x86_64': ['test-linux', 'test-win-win7'],
407         }
408         return [self._webkit_baseline_path(d) for d in search_paths[self.name()]]
409
410     def default_child_processes(self):
411         return 1
412
413     def worker_startup_delay_secs(self):
414         return 0
415
416     def check_build(self, needs_http):
417         return True
418
419     def check_sys_deps(self, needs_http):
420         return True
421
422     def default_configuration(self):
423         return 'Release'
424
425     def diff_image(self, expected_contents, actual_contents, tolerance=None):
426         diffed = actual_contents != expected_contents
427         if not actual_contents and not expected_contents:
428             return (None, 0, None)
429         if not actual_contents or not expected_contents:
430             return (True, 0, None)
431         if 'ref' in expected_contents:
432             assert tolerance == 0
433         if diffed:
434             return ("< %s\n---\n> %s\n" % (expected_contents, actual_contents), 1, None)
435         return (None, 0, None)
436
437     def layout_tests_dir(self):
438         return LAYOUT_TEST_DIR
439
440     def perf_tests_dir(self):
441         return PERF_TEST_DIR
442
443     def webkit_base(self):
444         return '/test.checkout'
445
446     def _skipped_tests_for_unsupported_features(self, test_list):
447         return set(['failures/expected/skip_text.html',
448                     'failures/unexpected/skip_pass.html',
449                     'virtual/skipped'])
450
451     def name(self):
452         return self._name
453
454     def operating_system(self):
455         return self._operating_system
456
457     def default_results_directory(self):
458         return '/tmp/layout-test-results'
459
460     def setup_test_run(self):
461         pass
462
463     def _driver_class(self):
464         return TestDriver
465
466     def start_http_server(self, additional_dirs=None):
467         pass
468
469     def start_websocket_server(self):
470         pass
471
472     def stop_http_server(self):
473         pass
474
475     def stop_websocket_server(self):
476         pass
477
478     def _path_to_lighttpd(self):
479         return "/usr/sbin/lighttpd"
480
481     def _path_to_lighttpd_modules(self):
482         return "/usr/lib/lighttpd"
483
484     def _path_to_lighttpd_php(self):
485         return "/usr/bin/php-cgi"
486
487     def _path_to_apache(self):
488         return "/usr/sbin/httpd"
489
490     def _path_to_apache_config_file(self):
491         return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', 'httpd.conf')
492
493     def path_to_test_expectations_file(self):
494         return self._expectations_path
495
496     def all_test_configurations(self):
497         """Returns a sequence of the TestConfigurations the port supports."""
498         # By default, we assume we want to test every graphics type in
499         # every configuration on every system.
500         test_configurations = []
501         for version, architecture in self._all_systems():
502             for build_type in self._all_build_types():
503                 test_configurations.append(TestConfiguration(
504                     version=version,
505                     architecture=architecture,
506                     build_type=build_type))
507         return test_configurations
508
509     def _all_systems(self):
510         return (('leopard', 'x86'),
511                 ('snowleopard', 'x86'),
512                 ('xp', 'x86'),
513                 ('vista', 'x86'),
514                 ('win7', 'x86'),
515                 ('lucid', 'x86'),
516                 ('lucid', 'x86_64'))
517
518     def _all_build_types(self):
519         return ('debug', 'release')
520
521     def configuration_specifier_macros(self):
522         """To avoid surprises when introducing new macros, these are intentionally fixed in time."""
523         return {'mac': ['leopard', 'snowleopard'], 'win': ['xp', 'vista', 'win7'], 'linux': ['lucid']}
524
525     def all_baseline_variants(self):
526         return self.ALL_BASELINE_VARIANTS
527
528     def virtual_test_suites(self):
529         return [
530             VirtualTestSuite('virtual/passes', 'passes', ['--virtual-arg']),
531             VirtualTestSuite('virtual/skipped', 'failures/expected', ['--virtual-arg2']),
532         ]
533
534
535 class TestDriver(Driver):
536     """Test/Dummy implementation of the DumpRenderTree interface."""
537     next_pid = 1
538
539     def __init__(self, *args, **kwargs):
540         super(TestDriver, self).__init__(*args, **kwargs)
541         self.started = False
542         self.pid = 0
543
544     def cmd_line(self, pixel_tests, per_test_args):
545         pixel_tests_flag = '-p' if pixel_tests else ''
546         return [self._port._path_to_driver()] + [pixel_tests_flag] + self._port.get_option('additional_drt_flag', []) + per_test_args
547
548     def run_test(self, test_input, stop_when_done):
549         if not self.started:
550             self.started = True
551             self.pid = TestDriver.next_pid
552             TestDriver.next_pid += 1
553
554         start_time = time.time()
555         test_name = test_input.test_name
556         test_args = test_input.args or []
557         test = self._port._tests[test_name]
558         if test.keyboard:
559             raise KeyboardInterrupt
560         if test.exception:
561             raise ValueError('exception from ' + test_name)
562         if test.hang:
563             time.sleep((float(test_input.timeout) * 4) / 1000.0 + 1.0)  # The 1.0 comes from thread_padding_sec in layout_test_runnery.
564
565         audio = None
566         actual_text = test.actual_text
567
568         if 'flaky' in test_name and not test_name in self._port._flakes:
569             self._port._flakes.add(test_name)
570             actual_text = 'flaky text failure'
571
572         if actual_text and test_args and test_name == 'passes/args.html':
573             actual_text = actual_text + ' ' + ' '.join(test_args)
574
575         if test.actual_audio:
576             audio = base64.b64decode(test.actual_audio)
577         crashed_process_name = None
578         crashed_pid = None
579         if test.crash:
580             crashed_process_name = self._port.driver_name()
581             crashed_pid = 1
582         elif test.web_process_crash:
583             crashed_process_name = 'WebProcess'
584             crashed_pid = 2
585
586         crash_log = ''
587         if crashed_process_name:
588             crash_logs = CrashLogs(self._port.host)
589             crash_log = crash_logs.find_newest_log(crashed_process_name, None) or ''
590
591         if stop_when_done:
592             self.stop()
593
594         if test.actual_checksum == test_input.image_hash:
595             image = None
596         else:
597             image = test.actual_image
598         return DriverOutput(actual_text, image, test.actual_checksum, audio,
599             crash=test.crash or test.web_process_crash, crashed_process_name=crashed_process_name,
600             crashed_pid=crashed_pid, crash_log=crash_log,
601             test_time=time.time() - start_time, timeout=test.timeout, error=test.error, pid=self.pid)
602
603     def stop(self):
604         self.started = False