4b21ba9db50924b60600d779eec8968b03a6d0f4
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / run_webkit_tests_integrationtest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
3 # Copyright (C) 2011, 2013 Apple Inc. All rights reserved.
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 import codecs
32 import json
33 import logging
34 import os
35 import platform
36 import Queue
37 import re
38 import StringIO
39 import sys
40 import thread
41 import time
42 import threading
43 import unittest
44
45 from webkitpy.common.system import outputcapture, path
46 from webkitpy.common.system.crashlogs_unittest import make_mock_crash_report_darwin
47 from webkitpy.common.system.systemhost import SystemHost
48 from webkitpy.common.host import Host
49 from webkitpy.common.host_mock import MockHost
50
51 from webkitpy import port
52 from webkitpy.layout_tests import run_webkit_tests
53 from webkitpy.layout_tests.models.test_run_results import INTERRUPTED_EXIT_STATUS
54 from webkitpy.port import Port
55 from webkitpy.port import test
56 from webkitpy.test.skip import skip_if
57
58
59 def parse_args(extra_args=None, tests_included=False, new_results=False, print_nothing=True):
60     extra_args = extra_args or []
61     args = []
62     if not '--platform' in extra_args:
63         args.extend(['--platform', 'test'])
64     if not new_results:
65         args.append('--no-new-test-results')
66
67     if not '--child-processes' in extra_args:
68         args.extend(['--child-processes', 1])
69
70     if not '--world-leaks' in extra_args:
71         args.append('--world-leaks')
72
73     args.extend(extra_args)
74     if not tests_included:
75         # We use the glob to test that globbing works.
76         args.extend(['passes',
77                      'http/tests',
78                      'websocket/tests',
79                      'failures/expected/*'])
80     return run_webkit_tests.parse_args(args)
81
82
83 def passing_run(extra_args=None, port_obj=None, tests_included=False, host=None, shared_port=True):
84     options, parsed_args = parse_args(extra_args, tests_included)
85     if not port_obj:
86         host = host or MockHost()
87         port_obj = host.port_factory.get(port_name=options.platform, options=options)
88
89     if shared_port:
90         port_obj.host.port_factory.get = lambda *args, **kwargs: port_obj
91
92     logging_stream = StringIO.StringIO()
93     run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream)
94     return run_details.exit_code == 0
95
96
97 def logging_run(extra_args=None, port_obj=None, tests_included=False, host=None, new_results=False, shared_port=True):
98     options, parsed_args = parse_args(extra_args=extra_args,
99                                       tests_included=tests_included,
100                                       print_nothing=False, new_results=new_results)
101     host = host or MockHost()
102     if not port_obj:
103         port_obj = host.port_factory.get(port_name=options.platform, options=options)
104
105     run_details, output = run_and_capture(port_obj, options, parsed_args, shared_port)
106     return (run_details, output, host.user)
107
108
109 def run_and_capture(port_obj, options, parsed_args, shared_port=True):
110     if shared_port:
111         port_obj.host.port_factory.get = lambda *args, **kwargs: port_obj
112     oc = outputcapture.OutputCapture()
113     try:
114         oc.capture_output()
115         logging_stream = StringIO.StringIO()
116         run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream)
117     finally:
118         oc.restore_output()
119     return (run_details, logging_stream)
120
121
122 def get_tests_run(args, host=None):
123     results = get_test_results(args, host)
124     return [result.test_name for result in results]
125
126
127 def get_test_batches(args, host=None):
128     results = get_test_results(args, host)
129     batches = []
130     batch = []
131     current_pid = None
132     for result in results:
133         if batch and result.pid != current_pid:
134             batches.append(batch)
135             batch = []
136         batch.append(result.test_name)
137     if batch:
138         batches.append(batch)
139     return batches
140
141
142 def get_test_results(args, host=None):
143     options, parsed_args = parse_args(args, tests_included=True)
144
145     host = host or MockHost()
146     port_obj = host.port_factory.get(port_name=options.platform, options=options)
147
148     oc = outputcapture.OutputCapture()
149     oc.capture_output()
150     logging_stream = StringIO.StringIO()
151     try:
152         run_details = run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream)
153     finally:
154         oc.restore_output()
155
156     all_results = []
157     if run_details.initial_results:
158         all_results.extend(run_details.initial_results.all_results)
159
160     if run_details.retry_results:
161         all_results.extend(run_details.retry_results.all_results)
162     return all_results
163
164
165 def parse_full_results(full_results_text):
166     json_to_eval = full_results_text.replace("ADD_RESULTS(", "").replace(");", "")
167     compressed_results = json.loads(json_to_eval)
168     return compressed_results
169
170
171 class StreamTestingMixin(object):
172     def assertContains(self, stream, string):
173         self.assertTrue(string in stream.getvalue())
174
175     def assertEmpty(self, stream):
176         self.assertFalse(stream.getvalue())
177
178     def assertNotEmpty(self, stream):
179         self.assertTrue(stream.getvalue())
180
181
182 class RunTest(unittest.TestCase, StreamTestingMixin):
183     def setUp(self):
184         # A real PlatformInfo object is used here instead of a
185         # MockPlatformInfo because we need to actually check for
186         # Windows and Mac to skip some tests.
187         self._platform = SystemHost().platform
188
189         # FIXME: Remove this when we fix test-webkitpy to work
190         # properly on cygwin (bug 63846).
191         self.should_test_processes = not self._platform.is_win()
192
193     def test_basic(self):
194         options, args = parse_args(tests_included=True)
195         logging_stream = StringIO.StringIO()
196         host = MockHost()
197         port_obj = host.port_factory.get(options.platform, options)
198         details = run_webkit_tests.run(port_obj, options, args, logging_stream)
199
200         # These numbers will need to be updated whenever we add new tests.
201         self.assertEqual(details.initial_results.total, test.TOTAL_TESTS)
202         self.assertEqual(details.initial_results.expected_skips, test.TOTAL_SKIPS)
203         self.assertEqual(len(details.initial_results.unexpected_results_by_name), test.UNEXPECTED_PASSES + test.UNEXPECTED_FAILURES)
204         self.assertEqual(details.exit_code, test.UNEXPECTED_FAILURES)
205         self.assertEqual(details.retry_results.total, test.TOTAL_RETRIES)
206
207         one_line_summary = "%d tests ran as expected, %d didn't:\n" % (
208             details.initial_results.total - details.initial_results.expected_skips - len(details.initial_results.unexpected_results_by_name),
209             len(details.initial_results.unexpected_results_by_name))
210         self.assertTrue(one_line_summary in logging_stream.buflist)
211
212         # Ensure the results were summarized properly.
213         self.assertEqual(details.summarized_results['num_regressions'], details.exit_code)
214
215         # Ensure the image diff percentage is in the results.
216         self.assertEqual(details.summarized_results['tests']['failures']['expected']['image.html']['image_diff_percent'], 1)
217
218         # Ensure the results were written out and displayed.
219         full_results_text = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
220         json_to_eval = full_results_text.replace("ADD_RESULTS(", "").replace(");", "")
221         self.assertEqual(json.loads(json_to_eval), details.summarized_results)
222
223         self.assertEqual(host.user.opened_urls, [path.abspath_to_uri(MockHost().platform, '/tmp/layout-test-results/results.html')])
224
225     def test_batch_size(self):
226         batch_tests_run = get_test_batches(['--batch-size', '2'])
227         for batch in batch_tests_run:
228             self.assertTrue(len(batch) <= 2, '%s had too many tests' % ', '.join(batch))
229
230     def test_child_processes_2(self):
231         if self.should_test_processes:
232             _, regular_output, _ = logging_run(
233                 ['--debug-rwt-logging', '--child-processes', '2'], shared_port=False)
234             self.assertTrue(any(['Running 2 ' in line for line in regular_output.buflist]))
235
236     def test_child_processes_min(self):
237         if self.should_test_processes:
238             _, regular_output, _ = logging_run(
239                 ['--debug-rwt-logging', '--child-processes', '2', '-i', 'passes/passes', 'passes'],
240                 tests_included=True, shared_port=False)
241             self.assertTrue(any(['Running 1 ' in line for line in regular_output.buflist]))
242
243     def test_dryrun(self):
244         tests_run = get_tests_run(['--dry-run'])
245         self.assertEqual(tests_run, [])
246
247         tests_run = get_tests_run(['-n'])
248         self.assertEqual(tests_run, [])
249
250     def test_exception_raised(self):
251         # Exceptions raised by a worker are treated differently depending on
252         # whether they are in-process or out. inline exceptions work as normal,
253         # which allows us to get the full stack trace and traceback from the
254         # worker. The downside to this is that it could be any error, but this
255         # is actually useful in testing.
256         #
257         # Exceptions raised in a separate process are re-packaged into
258         # WorkerExceptions (a subclass of BaseException), which have a string capture of the stack which can
259         # be printed, but don't display properly in the unit test exception handlers.
260         self.assertRaises(BaseException, logging_run,
261             ['failures/expected/exception.html', '--child-processes', '1'], tests_included=True)
262
263         if self.should_test_processes:
264             self.assertRaises(BaseException, logging_run,
265                 ['--child-processes', '2', '--force', 'failures/expected/exception.html', 'passes/text.html'], tests_included=True, shared_port=False)
266
267     def test_full_results_html(self):
268         # FIXME: verify html?
269         details, _, _ = logging_run(['--full-results-html'])
270         self.assertEqual(details.exit_code, 0)
271
272     def test_hung_thread(self):
273         details, err, _ = logging_run(['--run-singly', '--time-out-ms=50', 'failures/expected/hang.html'], tests_included=True)
274         # Note that hang.html is marked as WontFix and all WontFix tests are
275         # expected to Pass, so that actually running them generates an "unexpected" error.
276         self.assertEqual(details.exit_code, 1)
277         self.assertNotEmpty(err)
278
279     def test_keyboard_interrupt(self):
280         # Note that this also tests running a test marked as SKIP if
281         # you specify it explicitly.
282         details, _, _ = logging_run(['failures/expected/keyboard.html', '--child-processes', '1'], tests_included=True)
283         self.assertEqual(details.exit_code, INTERRUPTED_EXIT_STATUS)
284
285         if self.should_test_processes:
286             _, regular_output, _ = logging_run(['failures/expected/keyboard.html', 'passes/text.html', '--child-processes', '2', '--force'], tests_included=True, shared_port=False)
287             self.assertTrue(any(['Interrupted, exiting' in line for line in regular_output.buflist]))
288
289     def test_no_tests_found(self):
290         details, err, _ = logging_run(['resources'], tests_included=True)
291         self.assertEqual(details.exit_code, -1)
292         self.assertContains(err, 'No tests to run.\n')
293
294     def test_no_tests_found_2(self):
295         details, err, _ = logging_run(['foo'], tests_included=True)
296         self.assertEqual(details.exit_code, -1)
297         self.assertContains(err, 'No tests to run.\n')
298
299     def test_natural_order(self):
300         tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html']
301         tests_run = get_tests_run(['--order=natural'] + tests_to_run)
302         self.assertEqual(['failures/expected/missing_text.html', 'failures/expected/text.html', 'passes/args.html', 'passes/audio.html'], tests_run)
303
304     def test_natural_order_test_specified_multiple_times(self):
305         tests_to_run = ['passes/args.html', 'passes/audio.html', 'passes/audio.html', 'passes/args.html']
306         tests_run = get_tests_run(['--order=natural'] + tests_to_run)
307         self.assertEqual(['passes/args.html', 'passes/args.html', 'passes/audio.html', 'passes/audio.html'], tests_run)
308
309     def test_random_order(self):
310         tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html']
311         tests_run = get_tests_run(['--order=random'] + tests_to_run)
312         self.assertEqual(sorted(tests_to_run), sorted(tests_run))
313
314     def test_random_order_test_specified_multiple_times(self):
315         tests_to_run = ['passes/args.html', 'passes/audio.html', 'passes/audio.html', 'passes/args.html']
316         tests_run = get_tests_run(['--order=random'] + tests_to_run)
317         self.assertEqual(tests_run.count('passes/audio.html'), 2)
318         self.assertEqual(tests_run.count('passes/args.html'), 2)
319
320     def test_no_order(self):
321         tests_to_run = ['passes/audio.html', 'failures/expected/text.html', 'failures/expected/missing_text.html', 'passes/args.html']
322         tests_run = get_tests_run(['--order=none'] + tests_to_run)
323         self.assertEqual(tests_to_run, tests_run)
324
325     def test_no_order_test_specified_multiple_times(self):
326         tests_to_run = ['passes/args.html', 'passes/audio.html', 'passes/audio.html', 'passes/args.html']
327         tests_run = get_tests_run(['--order=none'] + tests_to_run)
328         self.assertEqual(tests_to_run, tests_run)
329
330     def test_no_order_with_directory_entries_in_natural_order(self):
331         tests_to_run = ['http/tests/ssl', 'http/tests/passes']
332         tests_run = get_tests_run(['--order=none'] + tests_to_run)
333         self.assertEqual(tests_run, ['http/tests/ssl/text.html', 'http/tests/passes/image.html', 'http/tests/passes/text.html'])
334
335     def test_gc_between_tests(self):
336         self.assertTrue(passing_run(['--gc-between-tests']))
337
338     def test_check_for_world_leaks(self):
339         self.assertTrue(passing_run(['--world-leaks']))
340
341     def test_complex_text(self):
342         self.assertTrue(passing_run(['--complex-text']))
343
344     def test_threaded(self):
345         self.assertTrue(passing_run(['--threaded']))
346
347     def test_repeat_each(self):
348         tests_to_run = ['passes/image.html', 'passes/text.html']
349         tests_run = get_tests_run(['--repeat-each', '2'] + tests_to_run)
350         self.assertEqual(tests_run, ['passes/image.html', 'passes/image.html', 'passes/text.html', 'passes/text.html'])
351
352     def test_ignore_flag(self):
353         # Note that passes/image.html is expected to be run since we specified it directly.
354         tests_run = get_tests_run(['-i', 'passes', 'passes/image.html'])
355         self.assertFalse('passes/text.html' in tests_run)
356         self.assertTrue('passes/image.html' in tests_run)
357
358     def test_skipped_flag(self):
359         tests_run = get_tests_run(['passes'])
360         self.assertFalse('passes/skipped/skip.html' in tests_run)
361         num_tests_run_by_default = len(tests_run)
362
363         # Check that nothing changes when we specify skipped=default.
364         self.assertEqual(len(get_tests_run(['--skipped=default', 'passes'])),
365                           num_tests_run_by_default)
366
367         # Now check that we run one more test (the skipped one).
368         tests_run = get_tests_run(['--skipped=ignore', 'passes'])
369         self.assertTrue('passes/skipped/skip.html' in tests_run)
370         self.assertEqual(len(tests_run), num_tests_run_by_default + 1)
371
372         # Now check that we only run the skipped test.
373         self.assertEqual(get_tests_run(['--skipped=only', 'passes']), ['passes/skipped/skip.html'])
374
375         # Now check that we don't run anything.
376         self.assertEqual(get_tests_run(['--skipped=always', 'passes/skipped/skip.html']), [])
377
378     def test_iterations(self):
379         tests_to_run = ['passes/image.html', 'passes/text.html']
380         tests_run = get_tests_run(['--iterations', '2'] + tests_to_run)
381         self.assertEqual(tests_run, ['passes/image.html', 'passes/text.html', 'passes/image.html', 'passes/text.html'])
382
383     def test_repeat_each_iterations_num_tests(self):
384         # The total number of tests should be: number_of_tests *
385         # repeat_each * iterations
386         host = MockHost()
387         _, err, _ = logging_run(
388             ['--iterations', '2', '--repeat-each', '4', '--debug-rwt-logging', 'passes/text.html', 'failures/expected/text.html'],
389             tests_included=True, host=host)
390         self.assertContains(err, "All 16 tests ran as expected.\n")
391
392     def test_run_chunk(self):
393         # Test that we actually select the right chunk
394         all_tests_run = get_tests_run(['passes', 'failures'])
395         chunk_tests_run = get_tests_run(['--run-chunk', '1:4', 'passes', 'failures'])
396         self.assertEqual(all_tests_run[4:8], chunk_tests_run)
397
398         # Test that we wrap around if the number of tests is not evenly divisible by the chunk size
399         tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
400         chunk_tests_run = get_tests_run(['--run-chunk', '1:3'] + tests_to_run)
401         self.assertEqual(['passes/text.html', 'passes/error.html', 'passes/image.html'], chunk_tests_run)
402
403     def test_run_force(self):
404         # This raises an exception because we run
405         # failures/expected/exception.html, which is normally SKIPped.
406
407         self.assertRaises(ValueError, logging_run, ['--force'])
408
409     def test_run_part(self):
410         # Test that we actually select the right part
411         tests_to_run = ['passes/error.html', 'passes/image.html', 'passes/platform_image.html', 'passes/text.html']
412         tests_run = get_tests_run(['--run-part', '1:2'] + tests_to_run)
413         self.assertEqual(['passes/error.html', 'passes/image.html'], tests_run)
414
415         # Test that we wrap around if the number of tests is not evenly divisible by the chunk size
416         # (here we end up with 3 parts, each with 2 tests, and we only have 4 tests total, so the
417         # last part repeats the first two tests).
418         chunk_tests_run = get_tests_run(['--run-part', '3:3'] + tests_to_run)
419         self.assertEqual(['passes/error.html', 'passes/image.html'], chunk_tests_run)
420
421     def test_run_singly(self):
422         batch_tests_run = get_test_batches(['--run-singly'])
423         for batch in batch_tests_run:
424             self.assertEqual(len(batch), 1, '%s had too many tests' % ', '.join(batch))
425
426     def test_skip_failing_tests(self):
427         # This tests that we skip both known failing and known flaky tests. Because there are
428         # no known flaky tests in the default test_expectations, we add additional expectations.
429         host = MockHost()
430         host.filesystem.write_text_file('/tmp/overrides.txt', 'Bug(x) passes/image.html [ ImageOnlyFailure Pass ]\n')
431
432         batches = get_test_batches(['--skip-failing-tests', '--additional-expectations', '/tmp/overrides.txt'], host=host)
433         has_passes_text = False
434         for batch in batches:
435             self.assertFalse('failures/expected/text.html' in batch)
436             self.assertFalse('passes/image.html' in batch)
437             has_passes_text = has_passes_text or ('passes/text.html' in batch)
438         self.assertTrue(has_passes_text)
439
440     def test_run_singly_actually_runs_tests(self):
441         details, _, _ = logging_run(['--run-singly'], tests_included=True)
442         self.assertEqual(details.exit_code, test.UNEXPECTED_FAILURES - 1)  # failures/expected/hang.html actually passes w/ --run-singly.
443
444     def test_single_file(self):
445         tests_run = get_tests_run(['passes/text.html'])
446         self.assertEqual(tests_run, ['passes/text.html'])
447
448     def test_single_file_with_prefix(self):
449         tests_run = get_tests_run(['LayoutTests/passes/text.html'])
450         self.assertEqual(['passes/text.html'], tests_run)
451
452     def test_single_skipped_file(self):
453         tests_run = get_tests_run(['failures/expected/keybaord.html'])
454         self.assertEqual([], tests_run)
455
456     def test_stderr_is_saved(self):
457         host = MockHost()
458         self.assertTrue(passing_run(host=host))
459         self.assertEqual(host.filesystem.read_text_file('/tmp/layout-test-results/passes/error-stderr.txt'),
460                           'stuff going to stderr')
461
462     def test_test_list(self):
463         host = MockHost()
464         filename = '/tmp/foo.txt'
465         host.filesystem.write_text_file(filename, 'passes/text.html')
466         tests_run = get_tests_run(['--test-list=%s' % filename], host=host)
467         self.assertEqual(['passes/text.html'], tests_run)
468         host.filesystem.remove(filename)
469         details, err, user = logging_run(['--test-list=%s' % filename], tests_included=True, host=host)
470         self.assertEqual(details.exit_code, -1)
471         self.assertNotEmpty(err)
472
473     def test_test_list_with_prefix(self):
474         host = MockHost()
475         filename = '/tmp/foo.txt'
476         host.filesystem.write_text_file(filename, 'LayoutTests/passes/text.html')
477         tests_run = get_tests_run(['--test-list=%s' % filename], host=host)
478         self.assertEqual(['passes/text.html'], tests_run)
479
480     def test_missing_and_unexpected_results(self):
481         # Test that we update expectations in place. If the expectation
482         # is missing, update the expected generic location.
483         host = MockHost()
484         details, err, _ = logging_run(['--no-show-results',
485             'failures/expected/missing_image.html',
486             'failures/unexpected/missing_text.html',
487             'failures/unexpected/text-image-checksum.html'],
488             tests_included=True, host=host)
489         file_list = host.filesystem.written_files.keys()
490         self.assertEqual(details.exit_code, 1)
491         expected_token = '"unexpected":{"text-image-checksum.html":{"report":"REGRESSION","expected":"PASS","actual":"IMAGE+TEXT","image_diff_percent":1},"missing_text.html":{"report":"MISSING","expected":"PASS","is_missing_text":true,"actual":"MISSING"}'
492         json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
493         self.assertTrue(json_string.find(expected_token) != -1)
494         self.assertTrue(json_string.find('"num_regressions":1') != -1)
495         self.assertTrue(json_string.find('"num_flaky":0') != -1)
496         self.assertTrue(json_string.find('"num_missing":1') != -1)
497
498     def test_pixel_test_directories(self):
499         host = MockHost()
500
501         """Both tests have faling checksum. We include only the first in pixel tests so only that should fail."""
502         args = ['--pixel-tests', '--pixel-test-directory', 'failures/unexpected/pixeldir',
503                 'failures/unexpected/pixeldir/image_in_pixeldir.html',
504                 'failures/unexpected/image_not_in_pixeldir.html']
505         details, err, _ = logging_run(extra_args=args, host=host, tests_included=True)
506
507         self.assertEqual(details.exit_code, 1)
508         expected_token = '"unexpected":{"pixeldir":{"image_in_pixeldir.html":{"report":"REGRESSION","expected":"PASS","actual":"IMAGE"'
509         json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
510         self.assertTrue(json_string.find(expected_token) != -1)
511
512     def test_missing_and_unexpected_results_with_custom_exit_code(self):
513         # Test that we update expectations in place. If the expectation
514         # is missing, update the expected generic location.
515         class CustomExitCodePort(test.TestPort):
516             def exit_code_from_summarized_results(self, unexpected_results):
517                 return unexpected_results['num_regressions'] + unexpected_results['num_missing']
518
519         host = MockHost()
520         options, parsed_args = run_webkit_tests.parse_args(['--pixel-tests', '--no-new-test-results'])
521         test_port = CustomExitCodePort(host, options=options)
522         details, err, _ = logging_run(['--no-show-results',
523             'failures/expected/missing_image.html',
524             'failures/unexpected/missing_text.html',
525             'failures/unexpected/text-image-checksum.html'],
526             tests_included=True, host=host, port_obj=test_port)
527         self.assertEqual(details.exit_code, 2)
528
529     def test_crash_with_stderr(self):
530         host = MockHost()
531         _, regular_output, _ = logging_run(['failures/unexpected/crash-with-stderr.html'], tests_included=True, host=host)
532         self.assertTrue(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json').find('{"crash-with-stderr.html":{"report":"REGRESSION","expected":"PASS","actual":"CRASH","has_stderr":true}}') != -1)
533
534     def test_no_image_failure_with_image_diff(self):
535         host = MockHost()
536         _, regular_output, _ = logging_run(['failures/unexpected/checksum-with-matching-image.html'], tests_included=True, host=host)
537         self.assertTrue(host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json').find('"num_regressions":0') != -1)
538
539     def test_crash_log(self):
540         # FIXME: Need to rewrite these tests to not be mac-specific, or move them elsewhere.
541         # Currently CrashLog uploading only works on Darwin and Windows.
542         if not self._platform.is_mac() or self._platform.is_win():
543             return
544         mock_crash_report = make_mock_crash_report_darwin('DumpRenderTree', 12345)
545         host = MockHost()
546         host.filesystem.write_text_file('/tmp/layout-test-results/DumpRenderTree_2011-06-13-150719_quadzen.crash', mock_crash_report)
547         _, regular_output, _ = logging_run(['failures/unexpected/crash-with-stderr.html', '--dump-render-tree'], tests_included=True, host=host)
548         expected_crash_log = mock_crash_report
549         self.assertEqual(host.filesystem.read_text_file('/tmp/layout-test-results/failures/unexpected/crash-with-stderr-crash-log.txt'), expected_crash_log)
550
551     def test_web_process_crash_log(self):
552         # FIXME: Need to rewrite these tests to not be mac-specific, or move them elsewhere.
553         # Currently CrashLog uploading only works on Darwin and Windows.
554         if not self._platform.is_mac() or self._platform.is_win():
555             return
556         mock_crash_report = make_mock_crash_report_darwin('WebProcess', 12345)
557         host = MockHost()
558         host.filesystem.write_text_file('/tmp/layout-test-results/WebProcess_2011-06-13-150719_quadzen.crash', mock_crash_report)
559         logging_run(['failures/unexpected/web-process-crash-with-stderr.html'], tests_included=True, host=host)
560         self.assertEqual(host.filesystem.read_text_file('/tmp/layout-test-results/failures/unexpected/web-process-crash-with-stderr-crash-log.txt'), mock_crash_report)
561
562     def test_exit_after_n_failures_upload(self):
563         host = MockHost()
564         details, regular_output, user = logging_run(
565            ['failures/unexpected/text-image-checksum.html', 'passes/text.html', '--exit-after-n-failures', '1'],
566            tests_included=True, host=host)
567
568         # By returning False, we know that the incremental results were generated and then deleted.
569         self.assertFalse(host.filesystem.exists('/tmp/layout-test-results/incremental_results.json'))
570
571         # This checks that we report only the number of tests that actually failed.
572         self.assertEqual(details.exit_code, 1)
573
574         # This checks that passes/text.html is considered SKIPped.
575         self.assertTrue('"skipped":1' in host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json'))
576
577         # This checks that we told the user we bailed out.
578         self.assertTrue('Exiting early after 1 failures. 1 tests run.\n' in regular_output.getvalue())
579
580         # This checks that neither test ran as expected.
581         # FIXME: This log message is confusing; tests that were skipped should be called out separately.
582         self.assertTrue('0 tests ran as expected, 2 didn\'t:\n' in regular_output.getvalue())
583
584     def test_exit_after_n_failures(self):
585         # Unexpected failures should result in tests stopping.
586         tests_run = get_tests_run(['failures/unexpected/text-image-checksum.html', 'passes/text.html', '--exit-after-n-failures', '1'])
587         self.assertEqual(['failures/unexpected/text-image-checksum.html'], tests_run)
588
589         # But we'll keep going for expected ones.
590         tests_run = get_tests_run(['failures/expected/text.html', 'passes/text.html', '--exit-after-n-failures', '1'])
591         self.assertEqual(['failures/expected/text.html', 'passes/text.html'], tests_run)
592
593     def test_exit_after_n_crashes(self):
594         # Unexpected crashes should result in tests stopping.
595         tests_run = get_tests_run(['failures/unexpected/crash.html', 'passes/text.html', '--exit-after-n-crashes-or-timeouts', '1'])
596         self.assertEqual(['failures/unexpected/crash.html'], tests_run)
597
598         # Same with timeouts.
599         tests_run = get_tests_run(['failures/unexpected/timeout.html', 'passes/text.html', '--exit-after-n-crashes-or-timeouts', '1'])
600         self.assertEqual(['failures/unexpected/timeout.html'], tests_run)
601
602         # But we'll keep going for expected ones.
603         tests_run = get_tests_run(['failures/expected/crash.html', 'passes/text.html', '--exit-after-n-crashes-or-timeouts', '1'])
604         self.assertEqual(['failures/expected/crash.html', 'passes/text.html'], tests_run)
605
606     def test_results_directory_absolute(self):
607         # We run a configuration that should fail, to generate output, then
608         # look for what the output results url was.
609
610         host = MockHost()
611         with host.filesystem.mkdtemp() as tmpdir:
612             _, _, user = logging_run(['--results-directory=' + str(tmpdir)], tests_included=True, host=host)
613             self.assertEqual(user.opened_urls, [path.abspath_to_uri(host.platform, host.filesystem.join(tmpdir, 'results.html'))])
614
615     def test_results_directory_default(self):
616         # We run a configuration that should fail, to generate output, then
617         # look for what the output results url was.
618
619         # This is the default location.
620         _, _, user = logging_run(tests_included=True)
621         self.assertEqual(user.opened_urls, [path.abspath_to_uri(MockHost().platform, '/tmp/layout-test-results/results.html')])
622
623     def test_results_directory_relative(self):
624         # We run a configuration that should fail, to generate output, then
625         # look for what the output results url was.
626         host = MockHost()
627         host.filesystem.maybe_make_directory('/tmp/cwd')
628         host.filesystem.chdir('/tmp/cwd')
629         _, _, user = logging_run(['--results-directory=foo'], tests_included=True, host=host)
630         self.assertEqual(user.opened_urls, [path.abspath_to_uri(host.platform, '/tmp/cwd/foo/results.html')])
631
632     def test_retrying_and_flaky_tests(self):
633         host = MockHost()
634         details, err, _ = logging_run(['--debug-rwt-logging', 'failures/flaky'], tests_included=True, host=host)
635         self.assertEqual(details.exit_code, 0)
636         self.assertTrue('Retrying' in err.getvalue())
637         self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt'))
638         self.assertFalse(host.filesystem.exists('/tmp/layout-test-results/retries/failures/flaky/text-actual.txt'))
639
640         # Now we test that --clobber-old-results does remove the old entries and the old retries,
641         # and that we don't retry again.
642         host = MockHost()
643         details, err, _ = logging_run(['--no-retry-failures', '--clobber-old-results', 'failures/flaky'], tests_included=True, host=host)
644         self.assertEqual(details.exit_code, 1)
645         self.assertTrue('Clobbering old results' in err.getvalue())
646         self.assertTrue('flaky/text.html' in err.getvalue())
647         self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/flaky/text-actual.txt'))
648         self.assertFalse(host.filesystem.exists('retries'))
649
650     def test_retrying_force_pixel_tests(self):
651         host = MockHost()
652         details, err, _ = logging_run(['--no-pixel-tests', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host)
653         self.assertEqual(details.exit_code, 1)
654         self.assertTrue('Retrying' in err.getvalue())
655         self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/unexpected/text-image-checksum-actual.txt'))
656         self.assertFalse(host.filesystem.exists('/tmp/layout-test-results/failures/unexpected/text-image-checksum-actual.png'))
657         self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/retries/failures/unexpected/text-image-checksum-actual.txt'))
658         self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/retries/failures/unexpected/text-image-checksum-actual.png'))
659         json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
660         json = parse_full_results(json_string)
661         self.assertEqual(json["tests"]["failures"]["unexpected"]["text-image-checksum.html"],
662             {"expected": "PASS", "actual": "TEXT IMAGE+TEXT", "image_diff_percent": 1, "report": "REGRESSION"})
663         self.assertFalse(json["pixel_tests_enabled"])
664         self.assertEqual(details.enabled_pixel_tests_in_retry, True)
665
666     def test_failed_text_with_missing_pixel_results_on_retry(self):
667         # Test what happens when pixel results are missing on retry.
668         host = MockHost()
669         details, err, _ = logging_run(['--no-show-results',
670             '--no-new-test-results', '--no-pixel-tests',
671             'failures/unexpected/text-image-missing.html'],
672             tests_included=True, host=host)
673         file_list = host.filesystem.written_files.keys()
674         self.assertEqual(details.exit_code, 1)
675         expected_token = '"unexpected":{"text-image-missing.html":{"report":"REGRESSION","expected":"PASS","actual":"TEXT MISSING","is_missing_image":true}}'
676         json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
677         self.assertTrue(json_string.find(expected_token) != -1)
678         self.assertTrue(json_string.find('"num_regressions":1') != -1)
679         self.assertTrue(json_string.find('"num_flaky":0') != -1)
680         self.assertTrue(json_string.find('"num_missing":1') != -1)
681
682     def test_retrying_uses_retries_directory(self):
683         host = MockHost()
684         details, err, _ = logging_run(['--debug-rwt-logging', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host)
685         self.assertEqual(details.exit_code, 1)
686         self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/failures/unexpected/text-image-checksum-actual.txt'))
687         self.assertTrue(host.filesystem.exists('/tmp/layout-test-results/retries/failures/unexpected/text-image-checksum-actual.txt'))
688
689     def test_run_order__inline(self):
690         # These next tests test that we run the tests in ascending alphabetical
691         # order per directory. HTTP tests are sharded separately from other tests,
692         # so we have to test both.
693         tests_run = get_tests_run(['-i', 'passes/passes', 'passes'])
694         self.assertEqual(tests_run, sorted(tests_run))
695
696         tests_run = get_tests_run(['http/tests/passes'])
697         self.assertEqual(tests_run, sorted(tests_run))
698
699     def test_tolerance(self):
700         class ImageDiffTestPort(test.TestPort):
701             def diff_image(self, expected_contents, actual_contents, tolerance=None):
702                 self.tolerance_used_for_diff_image = self._options.tolerance
703                 return (True, 1, None)
704
705         def get_port_for_run(args):
706             options, parsed_args = run_webkit_tests.parse_args(args)
707             host = MockHost()
708             test_port = ImageDiffTestPort(host, options=options)
709             res = passing_run(args, port_obj=test_port, tests_included=True)
710             self.assertTrue(res)
711             return test_port
712
713         base_args = ['--pixel-tests', '--no-new-test-results', 'failures/expected/*']
714
715         # If we pass in an explicit tolerance argument, then that will be used.
716         test_port = get_port_for_run(base_args + ['--tolerance', '.1'])
717         self.assertEqual(0.1, test_port.tolerance_used_for_diff_image)
718         test_port = get_port_for_run(base_args + ['--tolerance', '0'])
719         self.assertEqual(0, test_port.tolerance_used_for_diff_image)
720
721         # Otherwise the port's default tolerance behavior (including ignoring it)
722         # should be used.
723         test_port = get_port_for_run(base_args)
724         self.assertEqual(None, test_port.tolerance_used_for_diff_image)
725
726     def test_reftest_run(self):
727         tests_run = get_tests_run(['passes/reftest.html'])
728         self.assertEqual(['passes/reftest.html'], tests_run)
729
730     def test_reftest_run_reftests_if_pixel_tests_are_disabled(self):
731         tests_run = get_tests_run(['--no-pixel-tests', 'passes/reftest.html'])
732         self.assertEqual(['passes/reftest.html'], tests_run)
733
734     def test_reftest_skip_reftests_if_no_ref_tests(self):
735         tests_run = get_tests_run(['--no-ref-tests', 'passes/reftest.html'])
736         self.assertEqual([], tests_run)
737         tests_run = get_tests_run(['--no-ref-tests', '--no-pixel-tests', 'passes/reftest.html'])
738         self.assertEqual([], tests_run)
739
740     def test_reftest_expected_html_should_be_ignored(self):
741         tests_run = get_tests_run(['passes/reftest-expected.html'])
742         self.assertEqual([], tests_run)
743
744     def test_reftest_driver_should_run_expected_html(self):
745         tests_run = get_test_results(['passes/reftest.html'])
746         self.assertEqual(tests_run[0].references, ['passes/reftest-expected.html'])
747
748     def test_reftest_driver_should_run_expected_mismatch_html(self):
749         tests_run = get_test_results(['passes/mismatch.html'])
750         self.assertEqual(tests_run[0].references, ['passes/mismatch-expected-mismatch.html'])
751
752     def test_reftest_should_not_use_naming_convention_if_not_listed_in_reftestlist(self):
753         host = MockHost()
754         _, err, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, host=host)
755         json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
756         self.assertTrue(json_string.find('"unlistedtest.html":{"report":"MISSING","expected":"PASS","is_missing_text":true,"actual":"MISSING","is_missing_image":true}') != -1)
757         self.assertTrue(json_string.find('"num_regressions":4') != -1)
758         self.assertTrue(json_string.find('"num_flaky":0') != -1)
759         self.assertTrue(json_string.find('"num_missing":1') != -1)
760
761     def test_additional_platform_directory(self):
762         self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo']))
763         self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/../foo']))
764         self.assertTrue(passing_run(['--additional-platform-directory', '/tmp/foo', '--additional-platform-directory', '/tmp/bar']))
765         self.assertTrue(passing_run(['--additional-platform-directory', 'foo']))
766
767     def test_additional_expectations(self):
768         host = MockHost()
769         host.filesystem.write_text_file('/tmp/overrides.txt', 'Bug(x) failures/unexpected/mismatch.html [ ImageOnlyFailure ]\n')
770         self.assertTrue(passing_run(['--additional-expectations', '/tmp/overrides.txt', 'failures/unexpected/mismatch.html'],
771                                     tests_included=True, host=host))
772
773     def test_no_http_and_force(self):
774         # See test_run_force, using --force raises an exception.
775         # FIXME: We would like to check the warnings generated.
776         self.assertRaises(ValueError, logging_run, ['--force', '--no-http'])
777
778     @staticmethod
779     def has_test_of_type(tests, type):
780         return [test for test in tests if type in test]
781
782     def test_no_http_tests(self):
783         batch_tests_dryrun = get_tests_run(['LayoutTests/http', 'websocket/'])
784         self.assertTrue(RunTest.has_test_of_type(batch_tests_dryrun, 'http'))
785         self.assertTrue(RunTest.has_test_of_type(batch_tests_dryrun, 'websocket'))
786
787         batch_tests_run_no_http = get_tests_run(['--no-http', 'LayoutTests/http', 'websocket/'])
788         self.assertFalse(RunTest.has_test_of_type(batch_tests_run_no_http, 'http'))
789         self.assertFalse(RunTest.has_test_of_type(batch_tests_run_no_http, 'websocket'))
790
791         batch_tests_run_http = get_tests_run(['--http', 'LayoutTests/http', 'websocket/'])
792         self.assertTrue(RunTest.has_test_of_type(batch_tests_run_http, 'http'))
793         self.assertTrue(RunTest.has_test_of_type(batch_tests_run_http, 'websocket'))
794
795     def test_platform_tests_are_found(self):
796         tests_run = get_tests_run(['--platform', 'test-mac-leopard', 'http'])
797         self.assertTrue('platform/test-mac-leopard/http/test.html' in tests_run)
798         self.assertFalse('platform/test-win-win7/http/test.html' in tests_run)
799
800     def test_output_diffs(self):
801         # Test to ensure that we don't generate -pretty.html if PrettyPatch isn't available.
802         host = MockHost()
803         _, err, _ = logging_run(['--pixel-tests', 'failures/unexpected/text-image-checksum.html'], tests_included=True, host=host)
804         written_files = host.filesystem.written_files
805         self.assertTrue(any(path.endswith('-diff.txt') for path in written_files.keys()))
806         self.assertFalse(any(path.endswith('-pretty-diff.html') for path in written_files.keys()))
807
808         full_results_text = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
809         full_results = json.loads(full_results_text.replace("ADD_RESULTS(", "").replace(");", ""))
810         self.assertEqual(full_results['has_pretty_patch'], False)
811
812     def test_unsupported_platform(self):
813         stdout = StringIO.StringIO()
814         stderr = StringIO.StringIO()
815         res = run_webkit_tests.main(['--platform', 'foo'], stdout, stderr)
816
817         self.assertEqual(res, run_webkit_tests.EXCEPTIONAL_EXIT_STATUS)
818         self.assertEqual(stdout.getvalue(), '')
819         self.assertTrue('unsupported platform' in stderr.getvalue())
820
821     def test_verbose_in_child_processes(self):
822         # When we actually run multiple processes, we may have to reconfigure logging in the
823         # child process (e.g., on win32) and we need to make sure that works and we still
824         # see the verbose log output. However, we can't use logging_run() because using
825         # outputcapture to capture stdout and stderr latter results in a nonpicklable host.
826
827         # Test is flaky on Windows: https://bugs.webkit.org/show_bug.cgi?id=98559
828         if not self.should_test_processes:
829             return
830
831         options, parsed_args = parse_args(['--verbose', '--fully-parallel', '--child-processes', '2', 'passes/text.html', 'passes/image.html'], tests_included=True, print_nothing=False)
832         host = MockHost()
833         port_obj = host.port_factory.get(port_name=options.platform, options=options)
834         logging_stream = StringIO.StringIO()
835         run_webkit_tests.run(port_obj, options, parsed_args, logging_stream=logging_stream)
836         self.assertTrue('text.html passed' in logging_stream.getvalue())
837         self.assertTrue('image.html passed' in logging_stream.getvalue())
838
839
840 class EndToEndTest(unittest.TestCase):
841     def test_reftest_with_two_notrefs(self):
842         # Test that we update expectations in place. If the expectation
843         # is missing, update the expected generic location.
844         host = MockHost()
845         _, _, _ = logging_run(['--no-show-results', 'reftests/foo/'], tests_included=True, host=host)
846         file_list = host.filesystem.written_files.keys()
847
848         json_string = host.filesystem.read_text_file('/tmp/layout-test-results/full_results.json')
849         json = parse_full_results(json_string)
850         self.assertTrue("multiple-match-success.html" not in json["tests"]["reftests"]["foo"])
851         self.assertTrue("multiple-mismatch-success.html" not in json["tests"]["reftests"]["foo"])
852         self.assertTrue("multiple-both-success.html" not in json["tests"]["reftests"]["foo"])
853         self.assertEqual(json["tests"]["reftests"]["foo"]["multiple-match-failure.html"],
854             {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["=="], "image_diff_percent": 1, "report": "REGRESSION"})
855         self.assertEqual(json["tests"]["reftests"]["foo"]["multiple-mismatch-failure.html"],
856             {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["!="], "report": "REGRESSION"})
857         self.assertEqual(json["tests"]["reftests"]["foo"]["multiple-both-failure.html"],
858             {"expected": "PASS", "actual": "IMAGE", "reftest_type": ["==", "!="], "report": "REGRESSION"})
859
860
861 class RebaselineTest(unittest.TestCase, StreamTestingMixin):
862     def assertBaselines(self, file_list, file, extensions, err):
863         "assert that the file_list contains the baselines."""
864         for ext in extensions:
865             baseline = file + "-expected" + ext
866             baseline_msg = 'Writing new expected result "%s"\n' % baseline
867             self.assertTrue(any(f.find(baseline) != -1 for f in file_list))
868             self.assertContains(err, baseline_msg)
869
870     # FIXME: Add tests to ensure that we're *not* writing baselines when we're not
871     # supposed to be.
872
873     def test_reset_results(self):
874         # Test that we update expectations in place. If the expectation
875         # is missing, update the expected generic location.
876         host = MockHost()
877         details, err, _ = logging_run(
878             ['--pixel-tests', '--reset-results', 'passes/image.html', 'failures/expected/missing_image.html'],
879             tests_included=True, host=host, new_results=True)
880         file_list = host.filesystem.written_files.keys()
881         self.assertEqual(details.exit_code, 0)
882         self.assertEqual(len(file_list), 9)
883         self.assertBaselines(file_list, "passes/image", [".txt", ".png"], err)
884         self.assertBaselines(file_list, "failures/expected/missing_image", [".txt", ".png"], err)
885
886     def test_missing_results(self):
887         # Test that we update expectations in place. If the expectation
888         # is missing, update the expected generic location.
889         host = MockHost()
890         details, err, _ = logging_run(['--no-show-results',
891             'failures/unexpected/missing_text.html',
892             'failures/unexpected/missing_image.html',
893             'failures/unexpected/missing_audio.html',
894             'failures/unexpected/missing_render_tree_dump.html'],
895             tests_included=True, host=host, new_results=True)
896         file_list = host.filesystem.written_files.keys()
897         self.assertEqual(details.exit_code, 0)
898         self.assertEqual(len(file_list), 11)
899         self.assertBaselines(file_list, "failures/unexpected/missing_text", [".txt"], err)
900         self.assertBaselines(file_list, "platform/test/failures/unexpected/missing_image", [".png"], err)
901         self.assertBaselines(file_list, "platform/test/failures/unexpected/missing_render_tree_dump", [".txt"], err)
902
903     def test_new_baseline(self):
904         # Test that we update the platform expectations in the version-specific directories
905         # for both existing and new baselines.
906         host = MockHost()
907         details, err, _ = logging_run(
908             ['--pixel-tests', '--new-baseline', 'passes/image.html', 'failures/expected/missing_image.html'],
909             tests_included=True, host=host, new_results=True)
910         file_list = host.filesystem.written_files.keys()
911         self.assertEqual(details.exit_code, 0)
912         self.assertEqual(len(file_list), 9)
913         self.assertBaselines(file_list,
914             "platform/test-mac-leopard/passes/image", [".txt", ".png"], err)
915         self.assertBaselines(file_list,
916             "platform/test-mac-leopard/failures/expected/missing_image", [".txt", ".png"], err)
917
918
919 class PortTest(unittest.TestCase):
920     def assert_mock_port_works(self, port_name, args=[]):
921         self.assertTrue(passing_run(args + ['--platform', 'mock-' + port_name, 'fast/harness/results.html'], tests_included=True, host=Host()))
922
923     def disabled_test_mac_lion(self):
924         self.assert_mock_port_works('mac-lion')
925
926
927 class MainTest(unittest.TestCase):
928     def test_exception_handling(self):
929         orig_run_fn = run_webkit_tests.run
930
931         # unused args pylint: disable=W0613
932         def interrupting_run(port, options, args, stderr):
933             raise KeyboardInterrupt
934
935         def successful_run(port, options, args, stderr):
936
937             class FakeRunDetails(object):
938                 exit_code = -1
939
940             return FakeRunDetails()
941
942         def exception_raising_run(port, options, args, stderr):
943             assert False
944
945         stdout = StringIO.StringIO()
946         stderr = StringIO.StringIO()
947         try:
948             run_webkit_tests.run = interrupting_run
949             res = run_webkit_tests.main([], stdout, stderr)
950             self.assertEqual(res, INTERRUPTED_EXIT_STATUS)
951
952             run_webkit_tests.run = successful_run
953             res = run_webkit_tests.main(['--platform', 'test'], stdout, stderr)
954             self.assertEqual(res, -1)
955
956             run_webkit_tests.run = exception_raising_run
957             res = run_webkit_tests.main([], stdout, stderr)
958             self.assertEqual(res, run_webkit_tests.EXCEPTIONAL_EXIT_STATUS)
959         finally:
960             run_webkit_tests.run = orig_run_fn