Tools/Scripts/webkitpy/layout_tests/models/test_failures.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 #
   3 # Redistribution and use in source and binary forms, with or without
   4 # modification, are permitted provided that the following conditions are
   5 # met:
   6 #
   7 #     * Redistributions of source code must retain the above copyright
   8 # notice, this list of conditions and the following disclaimer.
   9 #     * Redistributions in binary form must reproduce the above
  10 # copyright notice, this list of conditions and the following disclaimer
  11 # in the documentation and/or other materials provided with the
  12 # distribution.
  13 #     * Neither the name of Google Inc. nor the names of its
  14 # contributors may be used to endorse or promote products derived from
  15 # this software without specific prior written permission.
  16 #
  17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28
  29 import cPickle
  30 import logging
  31
  32 from webkitpy.layout_tests.models import test_expectations
  33
  34 _log = logging.getLogger(__name__)
  35
  36
  37 def is_reftest_failure(failure_list):
  38     failure_types = [type(f) for f in failure_list]
  39     return set((FailureReftestMismatch, FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated)).intersection(failure_types)
  40
  41
  42 # FIXME: This is backwards.  Each TestFailure subclass should know what
  43 # test_expectation type it corresponds too.  Then this method just
  44 # collects them all from the failure list and returns the worst one.
  45 def determine_result_type(failure_list):
  46     """Takes a set of test_failures and returns which result type best fits
  47     the list of failures. "Best fits" means we use the worst type of failure.
  48
  49     Returns:
  50       one of the test_expectations result types - PASS, FAIL, CRASH, etc."""
  51
  52     if not failure_list or len(failure_list) == 0:
  53         return test_expectations.PASS
  54
  55     failure_types = [type(f) for f in failure_list]
  56     if FailureCrash in failure_types:
  57         return test_expectations.CRASH
  58     elif FailureTimeout in failure_types:
  59         return test_expectations.TIMEOUT
  60     elif FailureEarlyExit in failure_types:
  61         return test_expectations.SKIP
  62     elif (FailureMissingResult in failure_types or
  63           FailureMissingImage in failure_types or
  64           FailureMissingImageHash in failure_types or
  65           FailureMissingAudio in failure_types or
  66           FailureNotTested in failure_types):
  67         return test_expectations.MISSING
  68     else:
  69         is_text_failure = FailureTextMismatch in failure_types
  70         is_image_failure = (FailureImageHashIncorrect in failure_types or
  71                             FailureImageHashMismatch in failure_types)
  72         is_audio_failure = (FailureAudioMismatch in failure_types)
  73         if is_text_failure and is_image_failure:
  74             return test_expectations.IMAGE_PLUS_TEXT
  75         elif is_text_failure:
  76             return test_expectations.TEXT
  77         elif is_image_failure or is_reftest_failure(failure_list):
  78             return test_expectations.IMAGE
  79         elif is_audio_failure:
  80             return test_expectations.AUDIO
  81         else:
  82             raise ValueError("unclassifiable set of failures: "
  83                              + str(failure_types))
  84
  85
  86 class TestFailure(object):
  87     """Abstract base class that defines the failure interface."""
  88
  89     @staticmethod
  90     def loads(s):
  91         """Creates a TestFailure object from the specified string."""
  92         return cPickle.loads(s)
  93
  94     def message(self):
  95         """Returns a string describing the failure in more detail."""
  96         raise NotImplementedError
  97
  98     def __eq__(self, other):
  99         return self.__class__.__name__ == other.__class__.__name__
 100
 101     def __ne__(self, other):
 102         return self.__class__.__name__ != other.__class__.__name__
 103
 104     def __hash__(self):
 105         return hash(self.__class__.__name__)
 106
 107     def dumps(self):
 108         """Returns the string/JSON representation of a TestFailure."""
 109         return cPickle.dumps(self)
 110
 111     def driver_needs_restart(self):
 112         """Returns True if we should kill DumpRenderTree/WebKitTestRunner before the next test."""
 113         return False
 114
 115     def write_failure(self, writer, driver_output, expected_driver_output, port):
 116         assert isinstance(self, (FailureTimeout, FailureReftestNoImagesGenerated))
 117
 118
 119 class FailureText(TestFailure):
 120     def write_failure(self, writer, driver_output, expected_driver_output, port):
 121         writer.write_text_files(driver_output.text, expected_driver_output.text)
 122         writer.create_text_diff_and_write_result(driver_output.text, expected_driver_output.text)
 123
 124
 125 class FailureAudio(TestFailure):
 126     def write_failure(self, writer, driver_output, expected_driver_output, port):
 127         writer.write_audio_files(driver_output.audio, expected_driver_output.audio)
 128         writer.create_audio_diff_and_write_result(driver_output.audio, expected_driver_output.audio)
 129
 130
 131 class FailureTimeout(TestFailure):
 132     def __init__(self, is_reftest=False):
 133         super(FailureTimeout, self).__init__()
 134         self.is_reftest = is_reftest
 135
 136     def message(self):
 137         return "test timed out"
 138
 139     def driver_needs_restart(self):
 140         return True
 141
 142
 143 class FailureCrash(TestFailure):
 144     def __init__(self, is_reftest=False, process_name='DumpRenderTree', pid=None):
 145         super(FailureCrash, self).__init__()
 146         self.process_name = process_name
 147         self.pid = pid
 148         self.is_reftest = is_reftest
 149
 150     def message(self):
 151         if self.pid:
 152             return "%s crashed [pid=%d]" % (self.process_name, self.pid)
 153         return self.process_name + " crashed"
 154
 155     def driver_needs_restart(self):
 156         return True
 157
 158     def write_failure(self, writer, driver_output, expected_driver_output, port):
 159         crashed_driver_output = expected_driver_output if self.is_reftest else driver_output
 160         writer.write_crash_log(crashed_driver_output.crash_log)
 161
 162
 163 class FailureMissingResult(FailureText):
 164     def message(self):
 165         return "-expected.txt was missing"
 166
 167
 168 class FailureNotTested(FailureText):
 169     def message(self):
 170         return 'test was not run'
 171
 172 class FailureTextMismatch(FailureText):
 173     def message(self):
 174         return "text diff"
 175
 176
 177 class FailureMissingImageHash(TestFailure):
 178     def message(self):
 179         return "-expected.png was missing an embedded checksum"
 180
 181     def write_failure(self, writer, driver_output, expected_driver_output, port):
 182         writer.write_image_files(driver_output.image, expected_driver_output.image)
 183
 184
 185 class FailureMissingImage(TestFailure):
 186     def message(self):
 187         return "-expected.png was missing"
 188
 189     def write_failure(self, writer, driver_output, expected_driver_output, port):
 190         writer.write_image_files(driver_output.image, expected_image=None)
 191
 192
 193 class FailureImageHashMismatch(TestFailure):
 194     def __init__(self, diff_percent=0):
 195         super(FailureImageHashMismatch, self).__init__()
 196         self.diff_percent = diff_percent
 197
 198     def message(self):
 199         return "image diff"
 200
 201     def write_failure(self, writer, driver_output, expected_driver_output, port):
 202         writer.write_image_files(driver_output.image, expected_driver_output.image)
 203         writer.write_image_diff_files(driver_output.image_diff)
 204
 205
 206 class FailureImageHashIncorrect(TestFailure):
 207     def message(self):
 208         return "-expected.png embedded checksum is incorrect"
 209
 210
 211 class FailureReftestMismatch(TestFailure):
 212     def __init__(self, reference_filename=None):
 213         super(FailureReftestMismatch, self).__init__()
 214         self.reference_filename = reference_filename
 215         self.diff_percent = None
 216
 217     def message(self):
 218         return "reference mismatch"
 219
 220     def write_failure(self, writer, driver_output, expected_driver_output, port):
 221         writer.write_image_files(driver_output.image, expected_driver_output.image)
 222         # FIXME: This work should be done earlier in the pipeline (e.g., when we compare images for non-ref tests).
 223         # FIXME: We should always have 2 images here.
 224         if driver_output.image and expected_driver_output.image:
 225             diff_image, diff_percent, err_str = port.diff_image(expected_driver_output.image, driver_output.image, tolerance=0)
 226             if diff_image:
 227                 writer.write_image_diff_files(diff_image)
 228                 self.diff_percent = diff_percent
 229             else:
 230                 _log.warn('ref test mismatch did not produce an image diff.')
 231         writer.write_reftest(self.reference_filename)
 232
 233
 234 class FailureReftestMismatchDidNotOccur(TestFailure):
 235     def __init__(self, reference_filename=None):
 236         super(FailureReftestMismatchDidNotOccur, self).__init__()
 237         self.reference_filename = reference_filename
 238
 239     def message(self):
 240         return "reference mismatch didn't happen"
 241
 242     def write_failure(self, writer, driver_output, expected_driver_output, port):
 243         writer.write_image_files(driver_output.image, expected_image=None)
 244         writer.write_reftest(self.reference_filename)
 245
 246
 247 class FailureReftestNoImagesGenerated(TestFailure):
 248     def __init__(self, reference_filename=None):
 249         super(FailureReftestNoImagesGenerated, self).__init__()
 250         self.reference_filename = reference_filename
 251
 252     def message(self):
 253         return "reference didn't generate pixel results."
 254
 255
 256 class FailureMissingAudio(FailureAudio):
 257     def message(self):
 258         return "expected audio result was missing"
 259
 260
 261 class FailureAudioMismatch(FailureAudio):
 262     def message(self):
 263         return "audio mismatch"
 264
 265
 266 class FailureEarlyExit(TestFailure):
 267     def message(self):
 268         return "skipped due to early exit"
 269
 270
 271 # Convenient collection of all failure classes for anything that might
 272 # need to enumerate over them all.
 273 ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult, FailureNotTested,
 274                        FailureTextMismatch, FailureMissingImageHash,
 275                        FailureMissingImage, FailureImageHashMismatch,
 276                        FailureImageHashIncorrect, FailureReftestMismatch,
 277                        FailureReftestMismatchDidNotOccur, FailureReftestNoImagesGenerated,
 278                        FailureMissingAudio, FailureAudioMismatch,
 279                        FailureEarlyExit)