Allow a port to run tests with a custom device setup
[WebKit-https.git] / Tools / Scripts / webkitpy / port / test.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the Google name nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import sys
30 import time
31
32 from webkitpy.port import Port, Driver, DriverOutput
33 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
34 from webkitpy.common.system.filesystem_mock import MockFileSystem
35 from webkitpy.common.system.crashlogs import CrashLogs
36
37
38 # This sets basic expectations for a test. Each individual expectation
39 # can be overridden by a keyword argument in TestList.add().
40 class TestInstance(object):
41     def __init__(self, name):
42         self.name = name
43         self.base = name[(name.rfind("/") + 1):name.rfind(".")]
44         self.crash = False
45         self.web_process_crash = False
46         self.exception = False
47         self.hang = False
48         self.keyboard = False
49         self.error = ''
50         self.timeout = False
51         self.is_reftest = False
52
53         # The values of each field are treated as raw byte strings. They
54         # will be converted to unicode strings where appropriate using
55         # FileSystem.read_text_file().
56         self.actual_text = self.base + '-txt'
57         self.actual_checksum = self.base + '-checksum'
58
59         # We add the '\x8a' for the image file to prevent the value from
60         # being treated as UTF-8 (the character is invalid)
61         self.actual_image = self.base + '\x8a' + '-png' + 'tEXtchecksum\x00' + self.actual_checksum
62
63         self.expected_text = self.actual_text
64         self.expected_image = self.actual_image
65
66         self.actual_audio = None
67         self.expected_audio = None
68
69
70 # This is an in-memory list of tests, what we want them to produce, and
71 # what we want to claim are the expected results.
72 class TestList(object):
73     def __init__(self):
74         self.tests = {}
75
76     def add(self, name, **kwargs):
77         test = TestInstance(name)
78         for key, value in kwargs.items():
79             test.__dict__[key] = value
80         self.tests[name] = test
81
82     def add_reftest(self, name, reference_name, same_image):
83         self.add(name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
84         if same_image:
85             self.add(reference_name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
86         else:
87             self.add(reference_name, actual_checksum='yyy', actual_image='YYY', is_reftest=True)
88
89     def keys(self):
90         return self.tests.keys()
91
92     def __contains__(self, item):
93         return item in self.tests
94
95     def __getitem__(self, item):
96         return self.tests[item]
97
98 #
99 # These numbers may need to be updated whenever we add or delete tests.
100 #
101 TOTAL_TESTS = 72
102 TOTAL_SKIPS = 9
103 TOTAL_RETRIES = 14
104
105 UNEXPECTED_PASSES = 7
106 UNEXPECTED_FAILURES = 17
107
108
109 def unit_test_list():
110     silent_audio = "RIFF2\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x22\x56\x00\x00\x44\xAC\x00\x00\x02\x00\x10\x00data\x0E\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
111     silent_audio_with_single_bit_difference = "RIFF2\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x22\x56\x00\x00\x44\xAC\x00\x00\x02\x00\x10\x00data\x0E\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
112     audio2 = "RIFF2\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x22\x56\x00\x00\x44\xAC\x00\x00\x02\x00\x10\x00data\x0E\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
113     tests = TestList()
114     tests.add('failures/expected/crash.html', crash=True)
115     tests.add('failures/expected/exception.html', exception=True)
116     tests.add('failures/expected/timeout.html', timeout=True)
117     tests.add('failures/expected/hang.html', hang=True)
118     tests.add('failures/expected/missing_text.html', expected_text=None)
119     tests.add('failures/expected/image.html',
120               actual_image='image_fail-pngtEXtchecksum\x00checksum_fail',
121               expected_image='image-pngtEXtchecksum\x00checksum-png')
122     tests.add('failures/expected/image_checksum.html',
123               actual_checksum='image_checksum_fail-checksum',
124               actual_image='image_checksum_fail-png')
125     tests.add('failures/expected/audio.html',
126               actual_audio=silent_audio, expected_audio=audio2,
127               actual_text=None, expected_text=None,
128               actual_image=None, expected_image=None,
129               actual_checksum=None)
130     tests.add('failures/expected/keyboard.html', keyboard=True)
131     tests.add('failures/expected/missing_check.html',
132               expected_image='missing_check-png')
133     tests.add('failures/expected/missing_image.html', expected_image=None)
134     tests.add('failures/expected/missing_audio.html', expected_audio=None,
135               actual_text=None, expected_text=None,
136               actual_image=None, expected_image=None,
137               actual_checksum=None)
138     tests.add('failures/expected/missing_text.html', expected_text=None)
139     tests.add('failures/expected/newlines_leading.html',
140               expected_text="\nfoo\n", actual_text="foo\n")
141     tests.add('failures/expected/newlines_trailing.html',
142               expected_text="foo\n\n", actual_text="foo\n")
143     tests.add('failures/expected/newlines_with_excess_CR.html',
144               expected_text="foo\r\r\r\n", actual_text="foo\n")
145     tests.add('failures/expected/text.html', actual_text='text_fail-png')
146     tests.add('failures/expected/skip_text.html', actual_text='text diff')
147     tests.add('failures/flaky/text.html')
148     tests.add('failures/unexpected/missing_text.html', expected_text=None)
149     tests.add('failures/unexpected/missing_check.html', expected_image='missing-check-png')
150     tests.add('failures/unexpected/missing_image.html', expected_image=None)
151     tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="""layer at (0,0) size 800x600
152   RenderView at (0,0) size 800x600
153 layer at (0,0) size 800x34
154   RenderBlock {HTML} at (0,0) size 800x34
155     RenderBody {BODY} at (8,8) size 784x18
156       RenderText {#text} at (0,0) size 133x18
157         text run at (0,0) width 133: "This is an image test!"
158 """, expected_text=None)
159     tests.add('failures/unexpected/crash.html', crash=True)
160     tests.add('failures/unexpected/crash-with-stderr.html', crash=True,
161               error="mock-std-error-output")
162     tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_process_crash=True,
163               error="mock-std-error-output")
164     tests.add('failures/unexpected/pass.html')
165     tests.add('failures/unexpected/text-checksum.html',
166               actual_text='text-checksum_fail-txt',
167               actual_checksum='text-checksum_fail-checksum')
168     tests.add('failures/unexpected/text-image-checksum.html',
169               actual_text='text-image-checksum_fail-txt',
170               actual_image='text-image-checksum_fail-pngtEXtchecksum\x00checksum_fail',
171               actual_checksum='text-image-checksum_fail-checksum')
172     tests.add('failures/unexpected/text-image-missing.html',
173               actual_text='text-image-checksum_fail-txt',
174               expected_image=None)
175     tests.add('failures/unexpected/checksum-with-matching-image.html',
176               actual_checksum='text-image-checksum_fail-checksum')
177     tests.add('failures/unexpected/skip_pass.html')
178     tests.add('failures/unexpected/text.html', actual_text='text_fail-txt')
179     tests.add('failures/unexpected/timeout.html', timeout=True)
180     tests.add('http/tests/passes/text.html')
181     tests.add('http/tests/passes/image.html')
182     tests.add('http/tests/ssl/text.html')
183     tests.add('passes/args.html')
184     tests.add('passes/error.html', error='stuff going to stderr')
185     tests.add('passes/image.html')
186     tests.add('passes/audio.html',
187               actual_audio=silent_audio, expected_audio=silent_audio,
188               actual_text=None, expected_text=None,
189               actual_image=None, expected_image=None,
190               actual_checksum=None)
191     tests.add('passes/audio-tolerance.html',
192               actual_audio=silent_audio_with_single_bit_difference, expected_audio=silent_audio,
193               actual_text=None, expected_text=None,
194               actual_image=None, expected_image=None,
195               actual_checksum=None)
196     tests.add('passes/platform_image.html')
197     tests.add('passes/checksum_in_image.html',
198               expected_image='tEXtchecksum\x00checksum_in_image-checksum')
199     tests.add('passes/skipped/skip.html')
200
201     # Note that here the checksums don't match but the images do, so this test passes "unexpectedly".
202     # See https://bugs.webkit.org/show_bug.cgi?id=69444 .
203     tests.add('failures/unexpected/checksum.html', actual_checksum='checksum_fail-checksum')
204
205     # Text output files contain "\r\n" on Windows.  This may be
206     # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
207     tests.add('passes/text.html',
208               expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n')
209
210     # For reftests.
211     tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True)
212     tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False)
213     tests.add_reftest('passes/svgreftest.svg', 'passes/svgreftest-expected.svg', same_image=True)
214     tests.add_reftest('passes/xhtreftest.xht', 'passes/xhtreftest-expected.html', same_image=True)
215     tests.add_reftest('passes/phpreftest.php', 'passes/phpreftest-expected-mismatch.svg', same_image=False)
216     tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False)
217     tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True)
218     tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
219     tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
220     tests.add('failures/unexpected/reftest-nopixel.html', actual_checksum=None, actual_image=None, is_reftest=True)
221     tests.add('failures/unexpected/reftest-nopixel-expected.html', actual_checksum=None, actual_image=None, is_reftest=True)
222     # FIXME: Add a reftest which crashes.
223     tests.add('reftests/foo/test.html')
224     tests.add('reftests/foo/test-ref.html')
225
226     tests.add('reftests/foo/multiple-match-success.html', actual_checksum='abc', actual_image='abc')
227     tests.add('reftests/foo/multiple-match-failure.html', actual_checksum='abc', actual_image='abc')
228     tests.add('reftests/foo/multiple-mismatch-success.html', actual_checksum='abc', actual_image='abc')
229     tests.add('reftests/foo/multiple-mismatch-failure.html', actual_checksum='abc', actual_image='abc')
230     tests.add('reftests/foo/multiple-both-success.html', actual_checksum='abc', actual_image='abc')
231     tests.add('reftests/foo/multiple-both-failure.html', actual_checksum='abc', actual_image='abc')
232
233     tests.add('reftests/foo/matching-ref.html', actual_checksum='abc', actual_image='abc')
234     tests.add('reftests/foo/mismatching-ref.html', actual_checksum='def', actual_image='def')
235     tests.add('reftests/foo/second-mismatching-ref.html', actual_checksum='ghi', actual_image='ghi')
236
237     # The following files shouldn't be treated as reftests
238     tests.add_reftest('reftests/foo/unlistedtest.html', 'reftests/foo/unlistedtest-expected.html', same_image=True)
239     tests.add('reftests/foo/reference/bar/common.html')
240     tests.add('reftests/foo/reftest/bar/shared.html')
241
242     tests.add('websocket/tests/passes/text.html')
243
244     # For testing test are properly included from platform directories.
245     tests.add('platform/test-mac-leopard/http/test.html')
246     tests.add('platform/test-win-win7/http/test.html')
247
248     # For --no-http tests, test that platform specific HTTP tests are properly skipped.
249     tests.add('platform/test-snow-leopard/http/test.html')
250     tests.add('platform/test-snow-leopard/websocket/test.html')
251
252     # For testing if perf tests are running in a locked shard.
253     tests.add('perf/foo/test.html')
254     tests.add('perf/foo/test-ref.html')
255
256     # For testing --pixel-test-directories.
257     tests.add('failures/unexpected/pixeldir/image_in_pixeldir.html',
258         actual_image='image_in_pixeldir-pngtEXtchecksum\x00checksum_fail',
259         expected_image='image_in_pixeldir-pngtEXtchecksum\x00checksum-png')
260     tests.add('failures/unexpected/image_not_in_pixeldir.html',
261         actual_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum_fail',
262         expected_image='image_not_in_pixeldir-pngtEXtchecksum\x00checksum-png')
263
264     return tests
265
266
267 # Here we use a non-standard location for the layout tests, to ensure that
268 # this works. The path contains a '.' in the name because we've seen bugs
269 # related to this before.
270
271 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests'
272 PERF_TEST_DIR = '/test.checkout/PerformanceTests'
273
274
275 # Here we synthesize an in-memory filesystem from the test list
276 # in order to fully control the test output and to demonstrate that
277 # we don't need a real filesystem to run the tests.
278 def add_unit_tests_to_mock_filesystem(filesystem):
279     # Add the test_expectations file.
280     filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/platform/test')
281     if not filesystem.exists(LAYOUT_TEST_DIR + '/platform/test/TestExpectations'):
282         filesystem.write_text_file(LAYOUT_TEST_DIR + '/platform/test/TestExpectations', """
283 Bug(test) failures/expected/crash.html [ Crash ]
284 Bug(test) failures/expected/image.html [ ImageOnlyFailure ]
285 Bug(test) failures/expected/audio.html [ Failure ]
286 Bug(test) failures/expected/image_checksum.html [ ImageOnlyFailure ]
287 Bug(test) failures/expected/mismatch.html [ ImageOnlyFailure ]
288 Bug(test) failures/expected/missing_check.html [ Missing Pass ]
289 Bug(test) failures/expected/missing_image.html [ Missing Pass ]
290 Bug(test) failures/expected/missing_audio.html [ Missing Pass ]
291 Bug(test) failures/expected/missing_text.html [ Missing Pass ]
292 Bug(test) failures/expected/newlines_leading.html [ Failure ]
293 Bug(test) failures/expected/newlines_trailing.html [ Failure ]
294 Bug(test) failures/expected/newlines_with_excess_CR.html [ Failure ]
295 Bug(test) failures/expected/reftest.html [ ImageOnlyFailure ]
296 Bug(test) failures/expected/text.html [ Failure ]
297 Bug(test) failures/expected/timeout.html [ Timeout ]
298 Bug(test) failures/expected/hang.html [ WontFix ]
299 Bug(test) failures/expected/keyboard.html [ WontFix ]
300 Bug(test) failures/expected/exception.html [ WontFix ]
301 Bug(test) failures/unexpected/pass.html [ Failure ]
302 Bug(test) passes/skipped/skip.html [ Skip ]
303 """)
304
305     filesystem.maybe_make_directory(LAYOUT_TEST_DIR + '/reftests/foo')
306     filesystem.write_text_file(LAYOUT_TEST_DIR + '/reftests/foo/reftest.list', """
307 == test.html test-ref.html
308
309 == multiple-match-success.html mismatching-ref.html
310 == multiple-match-success.html matching-ref.html
311 == multiple-match-failure.html mismatching-ref.html
312 == multiple-match-failure.html second-mismatching-ref.html
313 != multiple-mismatch-success.html mismatching-ref.html
314 != multiple-mismatch-success.html second-mismatching-ref.html
315 != multiple-mismatch-failure.html mismatching-ref.html
316 != multiple-mismatch-failure.html matching-ref.html
317 == multiple-both-success.html matching-ref.html
318 == multiple-both-success.html mismatching-ref.html
319 != multiple-both-success.html second-mismatching-ref.html
320 == multiple-both-failure.html matching-ref.html
321 != multiple-both-failure.html second-mismatching-ref.html
322 != multiple-both-failure.html matching-ref.html
323 """)
324
325     # FIXME: This test was only being ignored because of missing a leading '/'.
326     # Fixing the typo causes several tests to assert, so disabling the test entirely.
327     # Add in a file should be ignored by port.find_test_files().
328     #files[LAYOUT_TEST_DIR + '/userscripts/resources/iframe.html'] = 'iframe'
329
330     def add_file(test, suffix, contents):
331         dirname = filesystem.join(LAYOUT_TEST_DIR, test.name[0:test.name.rfind('/')])
332         base = test.base
333         filesystem.maybe_make_directory(dirname)
334         filesystem.write_binary_file(filesystem.join(dirname, base + suffix), contents)
335
336     # Add each test and the expected output, if any.
337     test_list = unit_test_list()
338     for test in test_list.tests.values():
339         add_file(test, test.name[test.name.rfind('.'):], '')
340         if test.is_reftest:
341             continue
342         if test.actual_audio:
343             add_file(test, '-expected.wav', test.expected_audio)
344             continue
345         add_file(test, '-expected.txt', test.expected_text)
346         add_file(test, '-expected.png', test.expected_image)
347
348     # Clear the list of written files so that we can watch what happens during testing.
349     filesystem.clear_written_files()
350
351
352 class TestPort(Port):
353     port_name = 'test'
354     default_port_name = 'test-mac-leopard'
355
356     """Test implementation of the Port interface."""
357     ALL_BASELINE_VARIANTS = (
358         'test-linux-x86_64',
359         'test-mac-snowleopard', 'test-mac-leopard',
360         'test-win-vista', 'test-win-win7', 'test-win-xp',
361     )
362
363     @classmethod
364     def determine_full_port_name(cls, host, options, port_name):
365         if port_name == 'test':
366             return TestPort.default_port_name
367         return port_name
368
369     def __init__(self, host, port_name=None, **kwargs):
370         Port.__init__(self, host, port_name or TestPort.default_port_name, **kwargs)
371         self._tests = unit_test_list()
372         self._flakes = set()
373         self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/TestExpectations'
374         self._results_directory = None
375
376         self._operating_system = 'mac'
377         if self._name.startswith('test-win'):
378             self._operating_system = 'win'
379         elif self._name.startswith('test-linux'):
380             self._operating_system = 'linux'
381
382         version_map = {
383             'test-win-xp': 'xp',
384             'test-win-win7': 'win7',
385             'test-win-vista': 'vista',
386             'test-mac-leopard': 'leopard',
387             'test-mac-snowleopard': 'snowleopard',
388             'test-linux-x86_64': 'lucid',
389         }
390         self._version = version_map[self._name]
391
392     def default_pixel_tests(self):
393         return True
394
395     def _path_to_driver(self):
396         # This routine shouldn't normally be called, but it is called by
397         # the mock_drt Driver. We return something, but make sure it's useless.
398         return 'MOCK _path_to_driver'
399
400     def baseline_search_path(self):
401         search_paths = {
402             'test-mac-snowleopard': ['test-mac-snowleopard'],
403             'test-mac-leopard': ['test-mac-leopard', 'test-mac-snowleopard'],
404             'test-win-win7': ['test-win-win7'],
405             'test-win-vista': ['test-win-vista', 'test-win-win7'],
406             'test-win-xp': ['test-win-xp', 'test-win-vista', 'test-win-win7'],
407             'test-linux-x86_64': ['test-linux', 'test-win-win7'],
408         }
409         return [self._webkit_baseline_path(d) for d in search_paths[self.name()]]
410
411     def default_child_processes(self):
412         return 1
413
414     def worker_startup_delay_secs(self):
415         return 0
416
417     def check_build(self, needs_http):
418         return True
419
420     def check_sys_deps(self, needs_http):
421         return True
422
423     def default_configuration(self):
424         return 'Release'
425
426     def diff_image(self, expected_contents, actual_contents, tolerance=None):
427         diffed = actual_contents != expected_contents
428         if not actual_contents and not expected_contents:
429             return (None, 0, None)
430         if not actual_contents or not expected_contents:
431             return (True, 0, None)
432         if 'ref' in expected_contents:
433             assert tolerance == 0
434         if diffed:
435             return ("< %s\n---\n> %s\n" % (expected_contents, actual_contents), 1, None)
436         return (None, 0, None)
437
438     def layout_tests_dir(self):
439         return LAYOUT_TEST_DIR
440
441     def perf_tests_dir(self):
442         return PERF_TEST_DIR
443
444     def webkit_base(self):
445         return '/test.checkout'
446
447     def _skipped_tests_for_unsupported_features(self, test_list):
448         return set(['failures/expected/skip_text.html',
449                     'failures/unexpected/skip_pass.html'])
450
451     def name(self):
452         return self._name
453
454     def operating_system(self):
455         return self._operating_system
456
457     def default_results_directory(self):
458         return '/tmp/layout-test-results'
459
460     def setup_test_run(self, device_class=None):
461         pass
462
463     def _driver_class(self):
464         return TestDriver
465
466     def start_http_server(self, additional_dirs=None):
467         pass
468
469     def start_websocket_server(self):
470         pass
471
472     def stop_http_server(self):
473         pass
474
475     def stop_websocket_server(self):
476         pass
477
478     def _path_to_lighttpd(self):
479         return "/usr/sbin/lighttpd"
480
481     def _path_to_lighttpd_modules(self):
482         return "/usr/lib/lighttpd"
483
484     def _path_to_lighttpd_php(self):
485         return "/usr/bin/php-cgi"
486
487     def _path_to_apache(self):
488         return "/usr/sbin/httpd"
489
490     def _path_to_apache_config_file(self):
491         return self._filesystem.join(self.layout_tests_dir(), 'http', 'conf', 'httpd.conf')
492
493     def path_to_test_expectations_file(self):
494         return self._expectations_path
495
496     def all_test_configurations(self):
497         """Returns a sequence of the TestConfigurations the port supports."""
498         # By default, we assume we want to test every graphics type in
499         # every configuration on every system.
500         test_configurations = []
501         for version, architecture in self._all_systems():
502             for build_type in self._all_build_types():
503                 test_configurations.append(TestConfiguration(
504                     version=version,
505                     architecture=architecture,
506                     build_type=build_type))
507         return test_configurations
508
509     def _all_systems(self):
510         return (('leopard', 'x86'),
511                 ('snowleopard', 'x86'),
512                 ('xp', 'x86'),
513                 ('vista', 'x86'),
514                 ('win7', 'x86'),
515                 ('lucid', 'x86'),
516                 ('lucid', 'x86_64'))
517
518     def _all_build_types(self):
519         return ('debug', 'release')
520
521     def configuration_specifier_macros(self):
522         """To avoid surprises when introducing new macros, these are intentionally fixed in time."""
523         return {'mac': ['leopard', 'snowleopard'], 'win': ['xp', 'vista', 'win7'], 'linux': ['lucid']}
524
525     def all_baseline_variants(self):
526         return self.ALL_BASELINE_VARIANTS
527
528
529 class TestDriver(Driver):
530     """Test/Dummy implementation of the DumpRenderTree interface."""
531     next_pid = 1
532
533     def __init__(self, *args, **kwargs):
534         super(TestDriver, self).__init__(*args, **kwargs)
535         self.started = False
536         self.pid = 0
537
538     def cmd_line(self, pixel_tests, per_test_args):
539         pixel_tests_flag = '-p' if pixel_tests else ''
540         return [self._port._path_to_driver()] + [pixel_tests_flag] + self._port.get_option('additional_drt_flag', []) + per_test_args
541
542     def run_test(self, test_input, stop_when_done):
543         if not self.started:
544             self.started = True
545             self.pid = TestDriver.next_pid
546             TestDriver.next_pid += 1
547
548         start_time = time.time()
549         test_name = test_input.test_name
550         test_args = test_input.args or []
551         test = self._port._tests[test_name]
552         if test.keyboard:
553             raise KeyboardInterrupt
554         if test.exception:
555             raise ValueError('exception from ' + test_name)
556         if test.hang:
557             time.sleep((float(test_input.timeout) * 4) / 1000.0 + 1.0)  # The 1.0 comes from thread_padding_sec in layout_test_runnery.
558
559         audio = None
560         actual_text = test.actual_text
561
562         if 'flaky' in test_name and not test_name in self._port._flakes:
563             self._port._flakes.add(test_name)
564             actual_text = 'flaky text failure'
565
566         if actual_text and test_args and test_name == 'passes/args.html':
567             actual_text = actual_text + ' ' + ' '.join(test_args)
568
569         if test.actual_audio:
570             audio = test.actual_audio
571         crashed_process_name = None
572         crashed_pid = None
573         if test.crash:
574             crashed_process_name = self._port.driver_name()
575             crashed_pid = 1
576         elif test.web_process_crash:
577             crashed_process_name = 'WebProcess'
578             crashed_pid = 2
579
580         crash_log = ''
581         if crashed_process_name:
582             crash_logs = CrashLogs(self._port.host)
583             crash_log = crash_logs.find_newest_log(crashed_process_name, None) or ''
584
585         if stop_when_done:
586             self.stop()
587
588         if test.actual_checksum == test_input.image_hash:
589             image = None
590         else:
591             image = test.actual_image
592         return DriverOutput(actual_text, image, test.actual_checksum, audio,
593             crash=test.crash or test.web_process_crash, crashed_process_name=crashed_process_name,
594             crashed_pid=crashed_pid, crash_log=crash_log,
595             test_time=time.time() - start_time, timeout=test.timeout, error=test.error, pid=self.pid)
596
597     def stop(self):
598         self.started = False