fix style failures in webkitpy: expected 2 blank lines found 1
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / models / test_failures.py
1 # Copyright (C) 2010 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29 import cPickle
30 import logging
31
32 from webkitpy.layout_tests.models import test_expectations
33
34 _log = logging.getLogger(__name__)
35
36
37 def is_reftest_failure(failure_list):
38     failure_types = [type(f) for f in failure_list]
39     return set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated)).intersection(failure_types)
40
41
42 # FIXME: This is backwards.  Each TestFailure subclass should know what
43 # test_expectation type it corresponds too.  Then this method just
44 # collects them all from the failure list and returns the worst one.
45 def determine_result_type(failure_list):
46     """Takes a set of test_failures and returns which result type best fits
47     the list of failures. "Best fits" means we use the worst type of failure.
48
49     Returns:
50       one of the test_expectations result types - PASS, FAIL, CRASH, etc."""
51
52     if not failure_list or len(failure_list) == 0:
53         return test_expectations.PASS
54
55     failure_types = [type(f) for f in failure_list]
56     if FailureCrash in failure_types:
57         return test_expectations.CRASH
58     elif FailureTimeout in failure_types:
59         return test_expectations.TIMEOUT
60     elif FailureEarlyExit in failure_types:
61         return test_expectations.SKIP
62     elif (FailureMissingResult in failure_types or
63           FailureMissingImage in failure_types or
64           FailureMissingImageHash in failure_types or
65           FailureMissingAudio in failure_types or
66           FailureNotTested in failure_types):
67         return test_expectations.MISSING
68     else:
69         is_text_failure = FailureTextMismatch in failure_types
70         is_image_failure = (FailureImageHashIncorrect in failure_types or
71                             FailureImageHashMismatch in failure_types)
72         is_audio_failure = (FailureAudioMismatch in failure_types)
73         if is_text_failure and is_image_failure:
74             return test_expectations.IMAGE_PLUS_TEXT
75         elif is_text_failure:
76             return test_expectations.TEXT
77         elif is_image_failure or is_reftest_failure(failure_list):
78             return test_expectations.IMAGE
79         elif is_audio_failure:
80             return test_expectations.AUDIO
81         else:
82             raise ValueError("unclassifiable set of failures: "
83                              + str(failure_types))
84
85
86 class TestFailure(object):
87     """Abstract base class that defines the failure interface."""
88
89     @staticmethod
90     def loads(s):
91         """Creates a TestFailure object from the specified string."""
92         return cPickle.loads(s)
93
94     def message(self):
95         """Returns a string describing the failure in more detail."""
96         raise NotImplementedError
97
98     def __eq__(self, other):
99         return self.__class__.__name__ == other.__class__.__name__
100
101     def __ne__(self, other):
102         return self.__class__.__name__ != other.__class__.__name__
103
104     def __hash__(self):
105         return hash(self.__class__.__name__)
106
107     def dumps(self):
108         """Returns the string/JSON representation of a TestFailure."""
109         return cPickle.dumps(self)
110
111     def driver_needs_restart(self):
112         """Returns True if we should kill DumpRenderTree/WebKitTestRunner before the next test."""
113         return False
114
115     def write_failure(self, writer, driver_output, expected_driver_output, port):
116         assert isinstance(self, (FailureTimeout, FailureReftestNoImagesGenerated))
117
118
119 class FailureText(TestFailure):
120     def write_failure(self, writer, driver_output, expected_driver_output, port):
121         writer.write_text_files(driver_output.text, expected_driver_output.text)
122         writer.create_text_diff_and_write_result(driver_output.text, expected_driver_output.text)
123
124
125 class FailureAudio(TestFailure):
126     def write_failure(self, writer, driver_output, expected_driver_output, port):
127         writer.write_audio_files(driver_output.audio, expected_driver_output.audio)
128         writer.create_audio_diff_and_write_result(driver_output.audio, expected_driver_output.audio)
129
130
131 class FailureTimeout(TestFailure):
132     def __init__(self, is_reftest=False):
133         super(FailureTimeout, self).__init__()
134         self.is_reftest = is_reftest
135
136     def message(self):
137         return "test timed out"
138
139     def driver_needs_restart(self):
140         return True
141
142
143 class FailureCrash(TestFailure):
144     def __init__(self, is_reftest=False, process_name='DumpRenderTree', pid=None):
145         super(FailureCrash, self).__init__()
146         self.process_name = process_name
147         self.pid = pid
148         self.is_reftest = is_reftest
149
150     def message(self):
151         if self.pid:
152             return "%s crashed [pid=%d]" % (self.process_name, self.pid)
153         return self.process_name + " crashed"
154
155     def driver_needs_restart(self):
156         return True
157
158     def write_failure(self, writer, driver_output, expected_driver_output, port):
159         crashed_driver_output = expected_driver_output if self.is_reftest else driver_output
160         writer.write_crash_log(crashed_driver_output.crash_log)
161
162
163 class FailureMissingResult(FailureText):
164     def message(self):
165         return "-expected.txt was missing"
166
167
168 class FailureNotTested(FailureText):
169     def message(self):
170         return 'test was not run'
171
172
173 class FailureTextMismatch(FailureText):
174     def message(self):
175         return "text diff"
176
177
178 class FailureMissingImageHash(TestFailure):
179     def message(self):
180         return "-expected.png was missing an embedded checksum"
181
182     def write_failure(self, writer, driver_output, expected_driver_output, port):
183         writer.write_image_files(driver_output.image, expected_driver_output.image)
184
185
186 class FailureMissingImage(TestFailure):
187     def message(self):
188         return "-expected.png was missing"
189
190     def write_failure(self, writer, driver_output, expected_driver_output, port):
191         writer.write_image_files(driver_output.image, expected_image=None)
192
193
194 class FailureImageHashMismatch(TestFailure):
195     def __init__(self, diff_percent=0):
196         super(FailureImageHashMismatch, self).__init__()
197         self.diff_percent = diff_percent
198
199     def message(self):
200         return "image diff"
201
202     def write_failure(self, writer, driver_output, expected_driver_output, port):
203         writer.write_image_files(driver_output.image, expected_driver_output.image)
204         writer.write_image_diff_files(driver_output.image_diff)
205
206
207 class FailureImageHashIncorrect(TestFailure):
208     def message(self):
209         return "-expected.png embedded checksum is incorrect"
210
211
212 class FailureReftestMismatch(TestFailure):
213     def __init__(self, reference_filename=None):
214         super(FailureReftestMismatch, self).__init__()
215         self.reference_filename = reference_filename
216         self.diff_percent = None
217
218     def message(self):
219         return "reference mismatch"
220
221     def write_failure(self, writer, driver_output, expected_driver_output, port):
222         writer.write_image_files(driver_output.image, expected_driver_output.image)
223         # FIXME: This work should be done earlier in the pipeline (e.g., when we compare images for non-ref tests).
224         # FIXME: We should always have 2 images here.
225         if driver_output.image and expected_driver_output.image:
226             diff_image, diff_percent, err_str = port.diff_image(expected_driver_output.image, driver_output.image, tolerance=0)
227             if diff_image:
228                 writer.write_image_diff_files(diff_image)
229                 self.diff_percent = diff_percent
230             else:
231                 _log.warn('ref test mismatch did not produce an image diff.')
232         writer.write_reftest(self.reference_filename)
233
234
235 class FailureReftestMismatchDidNotOccur(TestFailure):
236     def __init__(self, reference_filename=None):
237         super(FailureReftestMismatchDidNotOccur, self).__init__()
238         self.reference_filename = reference_filename
239
240     def message(self):
241         return "reference mismatch didn't happen"
242
243     def write_failure(self, writer, driver_output, expected_driver_output, port):
244         writer.write_image_files(driver_output.image, expected_image=None)
245         writer.write_reftest(self.reference_filename)
246
247
248 class FailureReftestNoImagesGenerated(TestFailure):
249     def __init__(self, reference_filename=None):
250         super(FailureReftestNoImagesGenerated, self).__init__()
251         self.reference_filename = reference_filename
252
253     def message(self):
254         return "reference didn't generate pixel results."
255
256
257 class FailureMissingAudio(FailureAudio):
258     def message(self):
259         return "expected audio result was missing"
260
261
262 class FailureAudioMismatch(FailureAudio):
263     def message(self):
264         return "audio mismatch"
265
266
267 class FailureEarlyExit(TestFailure):
268     def message(self):
269         return "skipped due to early exit"
270
271
272 # Convenient collection of all failure classes for anything that might
273 # need to enumerate over them all.
274 ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult, FailureNotTested,
275                        FailureTextMismatch, FailureMissingImageHash,
276                        FailureMissingImage, FailureImageHashMismatch,
277                        FailureImageHashIncorrect, FailureReftestMismatch,
278                        FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated,
279                        FailureMissingAudio, FailureAudioMismatch,
280                        FailureEarlyExit)