bd811db0edb7fb3ddda0514009d049c665dafcbd
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_failures.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import cPickle
30 import logging
31
32 from webkitpy.layout_tests.models import test_expectations
33
34 _log = logging.getLogger(__name__)
35
36
37 def is_reftest_failure(failure_list):
38     failure_types = [type(f) for f in failure_list]
39     return set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated)).intersection(failure_types)
40
41
42 # FIXME: This is backwards.  Each TestFailure subclass should know what
43 # test_expectation type it corresponds too.  Then this method just
44 # collects them all from the failure list and returns the worst one.
45 def determine_result_type(failure_list):
46     """Takes a set of test_failures and returns which result type best fits
47     the list of failures. "Best fits" means we use the worst type of failure.
48
49     Returns:
50       one of the test_expectations result types - PASS, FAIL, CRASH, etc."""
51
52     if not failure_list or len(failure_list) == 0:
53         return test_expectations.PASS
54
55     failure_types = [type(f) for f in failure_list]
56     if FailureCrash in failure_types:
57         return test_expectations.CRASH
58     elif FailureTimeout in failure_types:
59         return test_expectations.TIMEOUT
60     elif FailureEarlyExit in failure_types:
61         return test_expectations.SKIP
62     elif (FailureMissingResult in failure_types or
63           FailureMissingImage in failure_types or
64           FailureMissingImageHash in failure_types or
65           FailureMissingAudio in failure_types or
66           FailureNotTested in failure_types):
67         return test_expectations.MISSING
68     else:
69         is_text_failure = FailureTextMismatch in failure_types
70         is_image_failure = (FailureImageHashIncorrect in failure_types or
71                             FailureImageHashMismatch in failure_types)
72         is_audio_failure = (FailureAudioMismatch in failure_types)
73         if is_text_failure and is_image_failure:
74             return test_expectations.IMAGE_PLUS_TEXT
75         elif is_text_failure:
76             return test_expectations.TEXT
77         elif is_image_failure or is_reftest_failure(failure_list):
78             return test_expectations.IMAGE
79         elif is_audio_failure:
80             return test_expectations.AUDIO
81         else:
82             raise ValueError("unclassifiable set of failures: "
83                              + str(failure_types))
84
85
86 class TestFailure(object):
87     """Abstract base class that defines the failure interface."""
88
89     @staticmethod
90     def loads(s):
91         """Creates a TestFailure object from the specified string."""
92         return cPickle.loads(s)
93
94     def message(self):
95         """Returns a string describing the failure in more detail."""
96         raise NotImplementedError
97
98     def __eq__(self, other):
99         return self.__class__.__name__ == other.__class__.__name__
100
101     def __ne__(self, other):
102         return self.__class__.__name__ != other.__class__.__name__
103
104     def __hash__(self):
105         return hash(self.__class__.__name__)
106
107     def dumps(self):
108         """Returns the string/JSON representation of a TestFailure."""
109         return cPickle.dumps(self)
110
111     def driver_needs_restart(self):
112         """Returns True if we should kill DumpRenderTree/WebKitTestRunner before the next test."""
113         return False
114
115     def write_failure(self, writer, driver_output, expected_driver_output, port):
116         assert isinstance(self, (FailureTimeout, FailureReftestNoImagesGenerated))
117
118
119 class FailureText(TestFailure):
120     def write_failure(self, writer, driver_output, expected_driver_output, port):
121         writer.write_text_files(driver_output.text, expected_driver_output.text)
122         writer.create_text_diff_and_write_result(driver_output.text, expected_driver_output.text)
123
124
125 class FailureAudio(TestFailure):
126     def write_failure(self, writer, driver_output, expected_driver_output, port):
127         writer.write_audio_files(driver_output.audio, expected_driver_output.audio)
128         writer.create_audio_diff_and_write_result(driver_output.audio, expected_driver_output.audio)
129
130
131 class FailureTimeout(TestFailure):
132     def __init__(self, is_reftest=False):
133         super(FailureTimeout, self).__init__()
134         self.is_reftest = is_reftest
135
136     def message(self):
137         return "test timed out"
138
139     def driver_needs_restart(self):
140         return True
141
142
143 class FailureCrash(TestFailure):
144     def __init__(self, is_reftest=False, process_name='DumpRenderTree', pid=None):
145         super(FailureCrash, self).__init__()
146         self.process_name = process_name
147         self.pid = pid
148         self.is_reftest = is_reftest
149
150     def message(self):
151         if self.pid:
152             return "%s crashed [pid=%d]" % (self.process_name, self.pid)
153         return self.process_name + " crashed"
154
155     def driver_needs_restart(self):
156         return True
157
158     def write_failure(self, writer, driver_output, expected_driver_output, port):
159         crashed_driver_output = expected_driver_output if self.is_reftest else driver_output
160         writer.write_crash_log(crashed_driver_output.crash_log)
161
162
163 class FailureMissingResult(FailureText):
164     def message(self):
165         return "-expected.txt was missing"
166
167
168 class FailureNotTested(FailureText):
169     def message(self):
170         return 'test was not run'
171
172 class FailureTextMismatch(FailureText):
173     def message(self):
174         return "text diff"
175
176
177 class FailureMissingImageHash(TestFailure):
178     def message(self):
179         return "-expected.png was missing an embedded checksum"
180
181     def write_failure(self, writer, driver_output, expected_driver_output, port):
182         writer.write_image_files(driver_output.image, expected_driver_output.image)
183
184
185 class FailureMissingImage(TestFailure):
186     def message(self):
187         return "-expected.png was missing"
188
189     def write_failure(self, writer, driver_output, expected_driver_output, port):
190         writer.write_image_files(driver_output.image, expected_image=None)
191
192
193 class FailureImageHashMismatch(TestFailure):
194     def __init__(self, diff_percent=0):
195         super(FailureImageHashMismatch, self).__init__()
196         self.diff_percent = diff_percent
197
198     def message(self):
199         return "image diff"
200
201     def write_failure(self, writer, driver_output, expected_driver_output, port):
202         writer.write_image_files(driver_output.image, expected_driver_output.image)
203         writer.write_image_diff_files(driver_output.image_diff)
204
205
206 class FailureImageHashIncorrect(TestFailure):
207     def message(self):
208         return "-expected.png embedded checksum is incorrect"
209
210
211 class FailureReftestMismatch(TestFailure):
212     def __init__(self, reference_filename=None):
213         super(FailureReftestMismatch, self).__init__()
214         self.reference_filename = reference_filename
215         self.diff_percent = None
216
217     def message(self):
218         return "reference mismatch"
219
220     def write_failure(self, writer, driver_output, expected_driver_output, port):
221         writer.write_image_files(driver_output.image, expected_driver_output.image)
222         # FIXME: This work should be done earlier in the pipeline (e.g., when we compare images for non-ref tests).
223         # FIXME: We should always have 2 images here.
224         if driver_output.image and expected_driver_output.image:
225             diff_image, diff_percent, err_str = port.diff_image(expected_driver_output.image, driver_output.image, tolerance=0)
226             if diff_image:
227                 writer.write_image_diff_files(diff_image)
228                 self.diff_percent = diff_percent
229             else:
230                 _log.warn('ref test mismatch did not produce an image diff.')
231         writer.write_reftest(self.reference_filename)
232
233
234 class FailureReftestMismatchDidNotOccur(TestFailure):
235     def __init__(self, reference_filename=None):
236         super(FailureReftestMismatchDidNotOccur, self).__init__()
237         self.reference_filename = reference_filename
238
239     def message(self):
240         return "reference mismatch didn't happen"
241
242     def write_failure(self, writer, driver_output, expected_driver_output, port):
243         writer.write_image_files(driver_output.image, expected_image=None)
244         writer.write_reftest(self.reference_filename)
245
246
247 class FailureReftestNoImagesGenerated(TestFailure):
248     def __init__(self, reference_filename=None):
249         super(FailureReftestNoImagesGenerated, self).__init__()
250         self.reference_filename = reference_filename
251
252     def message(self):
253         return "reference didn't generate pixel results."
254
255
256 class FailureMissingAudio(FailureAudio):
257     def message(self):
258         return "expected audio result was missing"
259
260
261 class FailureAudioMismatch(FailureAudio):
262     def message(self):
263         return "audio mismatch"
264
265
266 class FailureEarlyExit(TestFailure):
267     def message(self):
268         return "skipped due to early exit"
269
270
271 # Convenient collection of all failure classes for anything that might
272 # need to enumerate over them all.
273 ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult, FailureNotTested,
274                        FailureTextMismatch, FailureMissingImageHash,
275                        FailureMissingImage, FailureImageHashMismatch,
276                        FailureImageHashIncorrect, FailureReftestMismatch,
277                        FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated,
278                        FailureMissingAudio, FailureAudioMismatch,
279                        FailureEarlyExit)