webkitpy: Implement device type specific expected results (Part 2)
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_run_results_unittest.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import unittest
30
31 from webkitpy.common.host_mock import MockHost
32 from webkitpy.layout_tests.models import test_expectations
33 from webkitpy.layout_tests.models import test_failures
34 from webkitpy.layout_tests.models import test_results
35 from webkitpy.layout_tests.models import test_run_results
36 from webkitpy.tool.mocktool import MockOptions
37
38
39 def get_result(test_name, result_type=test_expectations.PASS, run_time=0):
40     failures = []
41     if result_type == test_expectations.TIMEOUT:
42         failures = [test_failures.FailureTimeout()]
43     elif result_type == test_expectations.AUDIO:
44         failures = [test_failures.FailureAudioMismatch()]
45     elif result_type == test_expectations.CRASH:
46         failures = [test_failures.FailureCrash()]
47     elif result_type == test_expectations.LEAK:
48         failures = [test_failures.FailureDocumentLeak(['http://localhost:8000/failures/expected/leak.html'])]
49     return test_results.TestResult(test_name, failures=failures, test_run_time=run_time)
50
51
52 def run_results(port):
53     tests = ['passes/text.html', 'failures/expected/timeout.html', 'failures/expected/crash.html', 'failures/expected/hang.html',
54              'failures/expected/audio.html', 'failures/expected/leak.html']
55     expectations = test_expectations.TestExpectations(port, tests)
56     expectations.parse_all_expectations()
57     return test_run_results.TestRunResults(expectations, len(tests))
58
59
60 def summarized_results(port, expected, passing, flaky, include_passes=False):
61     test_is_slow = False
62
63     initial_results = run_results(port)
64     if expected:
65         initial_results.add(get_result('passes/text.html', test_expectations.PASS), expected, test_is_slow)
66         initial_results.add(get_result('failures/expected/audio.html', test_expectations.AUDIO), expected, test_is_slow)
67         initial_results.add(get_result('failures/expected/timeout.html', test_expectations.TIMEOUT), expected, test_is_slow)
68         initial_results.add(get_result('failures/expected/crash.html', test_expectations.CRASH), expected, test_is_slow)
69
70         if port._options.pixel_tests:
71             initial_results.add(get_result('failures/expected/pixel-fail.html', test_expectations.IMAGE), expected, test_is_slow)
72         else:
73             initial_results.add(get_result('failures/expected/pixel-fail.html', test_expectations.PASS), expected, test_is_slow)
74
75         if port._options.world_leaks:
76             initial_results.add(get_result('failures/expected/leak.html', test_expectations.LEAK), expected, test_is_slow)
77         else:
78             initial_results.add(get_result('failures/expected/leak.html', test_expectations.PASS), expected, test_is_slow)
79
80     elif passing:
81         initial_results.add(get_result('passes/text.html'), expected, test_is_slow)
82         initial_results.add(get_result('failures/expected/audio.html'), expected, test_is_slow)
83         initial_results.add(get_result('failures/expected/timeout.html'), expected, test_is_slow)
84         initial_results.add(get_result('failures/expected/crash.html'), expected, test_is_slow)
85
86         if port._options.pixel_tests:
87             initial_results.add(get_result('failures/expected/pixel-fail.html'), expected, test_is_slow)
88         else:
89             initial_results.add(get_result('failures/expected/pixel-fail.html', test_expectations.IMAGE), expected, test_is_slow)
90
91         if port._options.world_leaks:
92             initial_results.add(get_result('failures/expected/leak.html'), expected, test_is_slow)
93         else:
94             initial_results.add(get_result('failures/expected/leak.html', test_expectations.PASS), expected, test_is_slow)
95     else:
96         initial_results.add(get_result('passes/text.html', test_expectations.TIMEOUT), expected, test_is_slow)
97         initial_results.add(get_result('failures/expected/audio.html', test_expectations.AUDIO), expected, test_is_slow)
98         initial_results.add(get_result('failures/expected/timeout.html', test_expectations.CRASH), expected, test_is_slow)
99         initial_results.add(get_result('failures/expected/crash.html', test_expectations.TIMEOUT), expected, test_is_slow)
100         initial_results.add(get_result('failures/expected/pixel-fail.html', test_expectations.TIMEOUT), expected, test_is_slow)
101         initial_results.add(get_result('failures/expected/leak.html', test_expectations.CRASH), expected, test_is_slow)
102
103         # we only list hang.html here, since normally this is WontFix
104         initial_results.add(get_result('failures/expected/hang.html', test_expectations.TIMEOUT), expected, test_is_slow)
105
106     if flaky:
107         retry_results = run_results(port)
108         retry_results.add(get_result('passes/text.html'), True, test_is_slow)
109         retry_results.add(get_result('failures/expected/timeout.html'), True, test_is_slow)
110         retry_results.add(get_result('failures/expected/crash.html'), True, test_is_slow)
111         retry_results.add(get_result('failures/expected/pixel-fail.html'), True, test_is_slow)
112         retry_results.add(get_result('failures/expected/leak.html'), True, test_is_slow)
113     else:
114         retry_results = None
115
116     return test_run_results.summarize_results(port, {None: initial_results.expectations}, initial_results, retry_results,
117         enabled_pixel_tests_in_retry=False, include_passes=include_passes)
118
119
120 class InterpretTestFailuresTest(unittest.TestCase):
121     def setUp(self):
122         host = MockHost()
123         self.port = host.port_factory.get(port_name='test')
124
125     def test_interpret_test_failures(self):
126         test_dict = test_run_results._interpret_test_failures([test_failures.FailureImageHashMismatch(diff_percent=0.42)])
127         self.assertEqual(test_dict['image_diff_percent'], 0.42)
128
129         test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatch(self.port.abspath_for_test('foo/reftest-expected.html'))])
130         self.assertIn('image_diff_percent', test_dict)
131
132         test_dict = test_run_results._interpret_test_failures([test_failures.FailureReftestMismatchDidNotOccur(self.port.abspath_for_test('foo/reftest-expected-mismatch.html'))])
133         self.assertEqual(len(test_dict), 0)
134
135         test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingAudio()])
136         self.assertIn('is_missing_audio', test_dict)
137
138         test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingResult()])
139         self.assertIn('is_missing_text', test_dict)
140
141         test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingImage()])
142         self.assertIn('is_missing_image', test_dict)
143
144         test_dict = test_run_results._interpret_test_failures([test_failures.FailureMissingImageHash()])
145         self.assertIn('is_missing_image', test_dict)
146
147
148 class SummarizedResultsTest(unittest.TestCase):
149     def setUp(self):
150         host = MockHost(initialize_scm_by_default=False)
151         self.port = host.port_factory.get(port_name='test', options=MockOptions(http=True, pixel_tests=False, world_leaks=False))
152
153     def test_no_svn_revision(self):
154         summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
155         self.assertNotIn('revision', summary)
156
157     def test_svn_revision(self):
158         self.port._options.builder_name = 'dummy builder'
159         summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
160         self.assertNotEquals(summary['revision'], '')
161
162     def test_summarized_results_wontfix(self):
163         self.port._options.builder_name = 'dummy builder'
164         summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
165         self.assertTrue(summary['tests']['failures']['expected']['hang.html']['wontfix'])
166
167     def test_summarized_results_include_passes(self):
168         self.port._options.builder_name = 'dummy builder'
169         summary = summarized_results(self.port, expected=False, passing=True, flaky=False, include_passes=True)
170         self.assertEqual(summary['tests']['passes']['text.html']['expected'], 'PASS')
171
172     def test_summarized_results_world_leaks_disabled(self):
173         self.port._options.builder_name = 'dummy builder'
174         summary = summarized_results(self.port, expected=False, passing=True, flaky=False, include_passes=True)
175         self.assertEqual(summary['tests']['failures']['expected']['leak.html']['expected'], 'PASS')