Tools/Scripts/webkitpy/layout_tests/models/test_run_results.py

   1 # Copyright (C) 2010 Google Inc. All rights reserved.
   2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   3 #
   4 # Redistribution and use in source and binary forms, with or without
   5 # modification, are permitted provided that the following conditions are
   6 # met:
   7 #
   8 #     * Redistributions of source code must retain the above copyright
   9 # notice, this list of conditions and the following disclaimer.
  10 #     * Redistributions in binary form must reproduce the above
  11 # copyright notice, this list of conditions and the following disclaimer
  12 # in the documentation and/or other materials provided with the
  13 # distribution.
  14 #     * Neither the name of Google Inc. nor the names of its
  15 # contributors may be used to endorse or promote products derived from
  16 # this software without specific prior written permission.
  17 #
  18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  29
  30 import datetime
  31 import logging
  32 import signal
  33
  34 from webkitpy.layout_tests.models import test_expectations
  35 from webkitpy.layout_tests.models import test_failures
  36
  37
  38 _log = logging.getLogger(__name__)
  39
  40 INTERRUPTED_EXIT_STATUS = signal.SIGINT + 128
  41
  42
  43 class TestRunResults(object):
  44     def __init__(self, expectations, num_tests):
  45         self.total = num_tests
  46         self.remaining = self.total
  47         self.expectations = expectations
  48         self.expected = 0
  49         self.unexpected = 0
  50         self.unexpected_failures = 0
  51         self.unexpected_crashes = 0
  52         self.unexpected_timeouts = 0
  53         self.tests_by_expectation = {}
  54         self.tests_by_timeline = {}
  55         self.results_by_name = {}  # Map of test name to the last result for the test.
  56         self.all_results = []  # All results from a run, including every iteration of every test.
  57         self.unexpected_results_by_name = {}
  58         self.failures_by_name = {}
  59         self.total_failures = 0
  60         self.expected_skips = 0
  61         for expectation in test_expectations.TestExpectations.EXPECTATIONS.values():
  62             self.tests_by_expectation[expectation] = set()
  63         for timeline in test_expectations.TestExpectations.TIMELINES.values():
  64             self.tests_by_timeline[timeline] = expectations.model().get_tests_with_timeline(timeline)
  65         self.slow_tests = set()
  66         self.interrupted = False
  67         self.keyboard_interrupted = False
  68
  69     def add(self, test_result, expected, test_is_slow):
  70         self.tests_by_expectation[test_result.type].add(test_result.test_name)
  71         self.results_by_name[test_result.test_name] = test_result
  72         if test_result.is_other_crash:
  73             return
  74         if test_result.type != test_expectations.SKIP:
  75             self.all_results.append(test_result)
  76         self.remaining -= 1
  77         if len(test_result.failures):
  78             self.total_failures += 1
  79             self.failures_by_name[test_result.test_name] = test_result.failures
  80         if expected:
  81             self.expected += 1
  82             if test_result.type == test_expectations.SKIP:
  83                 self.expected_skips += 1
  84         else:
  85             self.unexpected_results_by_name[test_result.test_name] = test_result
  86             self.unexpected += 1
  87             if len(test_result.failures):
  88                 self.unexpected_failures += 1
  89             if test_result.type == test_expectations.CRASH:
  90                 self.unexpected_crashes += 1
  91             elif test_result.type == test_expectations.TIMEOUT:
  92                 self.unexpected_timeouts += 1
  93         if test_is_slow:
  94             self.slow_tests.add(test_result.test_name)
  95
  96     def change_result_to_failure(self, existing_result, new_result, existing_expected, new_expected):
  97         assert existing_result.test_name == new_result.test_name
  98         if existing_result.type is new_result.type:
  99             return
 100
 101         self.tests_by_expectation[existing_result.type].remove(existing_result.test_name)
 102         self.tests_by_expectation[new_result.type].add(new_result.test_name)
 103
 104         had_failures = len(existing_result.failures) > 0
 105
 106         existing_result.convert_to_failure(new_result)
 107
 108         if not had_failures and len(existing_result.failures):
 109             self.total_failures += 1
 110
 111         if len(existing_result.failures):
 112             self.failures_by_name[existing_result.test_name] = existing_result.failures
 113
 114         if not existing_expected and new_expected:
 115             # test changed from unexpected to expected
 116             self.expected += 1
 117             self.unexpected_results_by_name.pop(existing_result.test_name, None)
 118             self.unexpected -= 1
 119             if had_failures:
 120                 self.unexpected_failures -= 1
 121         elif existing_expected and not new_expected:
 122             # test changed from expected to unexpected
 123             self.expected -= 1
 124             self.unexpected_results_by_name[existing_result.test_name] = existing_result
 125             self.unexpected += 1
 126             if len(existing_result.failures):
 127                 self.unexpected_failures += 1
 128
 129     def merge(self, test_run_results):
 130         if not test_run_results:
 131             return self
 132         # self.expectations should be the same for both
 133         self.total += test_run_results.total
 134         self.remaining += test_run_results.remaining
 135         self.expected += test_run_results.expected
 136         self.unexpected += test_run_results.unexpected
 137         self.unexpected_failures += test_run_results.unexpected_failures
 138         self.unexpected_crashes += test_run_results.unexpected_crashes
 139         self.unexpected_timeouts += test_run_results.unexpected_timeouts
 140         self.tests_by_expectation.update(test_run_results.tests_by_expectation)
 141         self.tests_by_timeline.update(test_run_results.tests_by_timeline)
 142         self.results_by_name.update(test_run_results.results_by_name)
 143         self.all_results += test_run_results.all_results
 144         self.unexpected_results_by_name.update(test_run_results.unexpected_results_by_name)
 145         self.failures_by_name.update(test_run_results.failures_by_name)
 146         self.total_failures += test_run_results.total_failures
 147         self.expected_skips += test_run_results.expected_skips
 148         self.tests_by_expectation.update(test_run_results.tests_by_expectation)
 149         self.tests_by_timeline.update(test_run_results.tests_by_timeline)
 150         self.slow_tests.update(test_run_results.slow_tests)
 151
 152         self.interrupted |= test_run_results.interrupted
 153         self.keyboard_interrupted |= test_run_results.keyboard_interrupted
 154         return self
 155
 156
 157 class RunDetails(object):
 158     def __init__(self, exit_code, summarized_results=None, initial_results=None, retry_results=None, enabled_pixel_tests_in_retry=False):
 159         self.exit_code = exit_code
 160         self.summarized_results = summarized_results
 161         self.initial_results = initial_results
 162         self.retry_results = retry_results
 163         self.enabled_pixel_tests_in_retry = enabled_pixel_tests_in_retry
 164
 165
 166 def _interpret_test_failures(failures):
 167     test_dict = {}
 168
 169     failure_types = [type(failure) for failure in failures]
 170     # FIXME: get rid of all this is_* values once there is a 1:1 map between
 171     # TestFailure type and test_expectations.EXPECTATION.
 172     if test_failures.FailureMissingAudio in failure_types:
 173         test_dict['is_missing_audio'] = True
 174
 175     if test_failures.FailureMissingResult in failure_types:
 176         test_dict['is_missing_text'] = True
 177
 178     if test_failures.FailureMissingImage in failure_types or test_failures.FailureMissingImageHash in failure_types:
 179         test_dict['is_missing_image'] = True
 180
 181     if test_failures.FailureDocumentLeak in failure_types:
 182         leaks = []
 183         for failure in failures:
 184             if isinstance(failure, test_failures.FailureDocumentLeak):
 185                 for url in failure.leaked_document_urls:
 186                     leaks.append({"document": url})
 187         test_dict['leaks'] = leaks
 188
 189     if 'image_diff_percent' not in test_dict:
 190         for failure in failures:
 191             if isinstance(failure, test_failures.FailureImageHashMismatch) or isinstance(failure, test_failures.FailureReftestMismatch):
 192                 test_dict['image_diff_percent'] = failure.diff_percent
 193
 194     return test_dict
 195
 196
 197 # These results must match ones in print_unexpected_results() in views/buildbot_results.py.
 198 def summarize_results(port_obj, expectations, initial_results, retry_results, enabled_pixel_tests_in_retry, include_passes=False, include_time_and_modifiers=False):
 199     """Returns a dictionary containing a summary of the test runs, with the following fields:
 200         'version': a version indicator
 201         'fixable': The number of fixable tests (NOW - PASS)
 202         'skipped': The number of skipped tests (NOW & SKIPPED)
 203         'num_regressions': The number of non-flaky failures
 204         'num_flaky': The number of flaky failures
 205         'num_missing': The number of tests with missing results
 206         'num_passes': The number of unexpected passes
 207         'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
 208         'date': the current date and time
 209     """
 210     results = {}
 211     results['version'] = 4
 212
 213     tbe = initial_results.tests_by_expectation
 214     tbt = initial_results.tests_by_timeline
 215     results['fixable'] = len(tbt[test_expectations.NOW] - tbe[test_expectations.PASS])
 216     results['skipped'] = len(tbt[test_expectations.NOW] & tbe[test_expectations.SKIP])
 217
 218     num_passes = 0
 219     num_flaky = 0
 220     num_missing = 0
 221     num_regressions = 0
 222     keywords = {}
 223     for expectation_string, expectation_enum in test_expectations.TestExpectations.EXPECTATIONS.iteritems():
 224         keywords[expectation_enum] = expectation_string.upper()
 225
 226     for modifier_string, modifier_enum in test_expectations.TestExpectations.MODIFIERS.iteritems():
 227         keywords[modifier_enum] = modifier_string.upper()
 228
 229     tests = {}
 230     other_crashes_dict = {}
 231
 232     for test_name, result in initial_results.results_by_name.iteritems():
 233         # Note that if a test crashed in the original run, we ignore
 234         # whether or not it crashed when we retried it (if we retried it),
 235         # and always consider the result not flaky.
 236         expected = expectations.model().get_expectations_string(test_name)
 237         result_type = result.type
 238         actual = [keywords[result_type]]
 239
 240         if result_type == test_expectations.SKIP:
 241             continue
 242
 243         if result.is_other_crash:
 244             other_crashes_dict[test_name] = {}
 245             continue
 246
 247         test_dict = {}
 248         if result.has_stderr:
 249             test_dict['has_stderr'] = True
 250
 251         if result.reftest_type:
 252             test_dict.update(reftest_type=list(result.reftest_type))
 253
 254         if expectations.model().has_modifier(test_name, test_expectations.WONTFIX):
 255             test_dict['wontfix'] = True
 256
 257         if result_type == test_expectations.PASS:
 258             num_passes += 1
 259             # FIXME: include passing tests that have stderr output.
 260             if expected == 'PASS' and not include_passes:
 261                 continue
 262         elif result_type == test_expectations.CRASH:
 263             if test_name in initial_results.unexpected_results_by_name:
 264                 num_regressions += 1
 265                 test_dict['report'] = 'REGRESSION'
 266         elif result_type == test_expectations.MISSING:
 267             if test_name in initial_results.unexpected_results_by_name:
 268                 num_missing += 1
 269                 test_dict['report'] = 'MISSING'
 270         elif result_type == test_expectations.LEAK:
 271             if test_name in initial_results.unexpected_results_by_name:
 272                 num_regressions += 1
 273                 test_dict['report'] = 'REGRESSION'
 274         elif test_name in initial_results.unexpected_results_by_name:
 275             if retry_results and test_name not in retry_results.unexpected_results_by_name:
 276                 actual.extend(expectations.model().get_expectations_string(test_name).split(" "))
 277                 num_flaky += 1
 278                 test_dict['report'] = 'FLAKY'
 279             elif retry_results:
 280                 retry_result_type = retry_results.unexpected_results_by_name[test_name].type
 281                 if result_type != retry_result_type:
 282                     if enabled_pixel_tests_in_retry and result_type == test_expectations.TEXT and (retry_result_type == test_expectations.IMAGE_PLUS_TEXT or retry_result_type == test_expectations.MISSING):
 283                         if retry_result_type == test_expectations.MISSING:
 284                             num_missing += 1
 285                         num_regressions += 1
 286                         test_dict['report'] = 'REGRESSION'
 287                     else:
 288                         num_flaky += 1
 289                         test_dict['report'] = 'FLAKY'
 290                     actual.append(keywords[retry_result_type])
 291                 else:
 292                     num_regressions += 1
 293                     test_dict['report'] = 'REGRESSION'
 294             else:
 295                 num_regressions += 1
 296                 test_dict['report'] = 'REGRESSION'
 297
 298         test_dict['expected'] = expected
 299         test_dict['actual'] = " ".join(actual)
 300         if include_time_and_modifiers:
 301             test_dict['time'] = round(1000 * result.test_run_time)
 302             # FIXME: Fix get_modifiers to return modifiers in new format.
 303             test_dict['modifiers'] = ' '.join(expectations.model().get_modifiers(test_name)).replace('BUGWK', 'webkit.org/b/')
 304
 305         test_dict.update(_interpret_test_failures(result.failures))
 306
 307         if retry_results:
 308             retry_result = retry_results.unexpected_results_by_name.get(test_name)
 309             if retry_result:
 310                 test_dict.update(_interpret_test_failures(retry_result.failures))
 311
 312         # Store test hierarchically by directory. e.g.
 313         # foo/bar/baz.html: test_dict
 314         # foo/bar/baz1.html: test_dict
 315         #
 316         # becomes
 317         # foo: {
 318         #     bar: {
 319         #         baz.html: test_dict,
 320         #         baz1.html: test_dict
 321         #     }
 322         # }
 323         parts = test_name.split('/')
 324         current_map = tests
 325         for i, part in enumerate(parts):
 326             if i == (len(parts) - 1):
 327                 current_map[part] = test_dict
 328                 break
 329             if part not in current_map:
 330                 current_map[part] = {}
 331             current_map = current_map[part]
 332
 333     results['tests'] = tests
 334     results['num_passes'] = num_passes
 335     results['num_flaky'] = num_flaky
 336     results['num_missing'] = num_missing
 337     results['num_regressions'] = num_regressions
 338     results['uses_expectations_file'] = port_obj.uses_test_expectations_file()
 339     results['interrupted'] = initial_results.interrupted  # Does results.html have enough information to compute this itself? (by checking total number of results vs. total number of tests?)
 340     results['layout_tests_dir'] = port_obj.layout_tests_dir()
 341     results['has_pretty_patch'] = port_obj.pretty_patch.pretty_patch_available()
 342     results['pixel_tests_enabled'] = port_obj.get_option('pixel_tests')
 343     results['other_crashes'] = other_crashes_dict
 344     results['date'] = datetime.datetime.now().strftime("%I:%M%p on %B %d, %Y")
 345
 346     try:
 347         # We only use the svn revision for using trac links in the results.html file,
 348         # Don't do this by default since it takes >100ms.
 349         # FIXME: Do we really need to populate this both here and in the json_results_generator?
 350         if port_obj.get_option("builder_name"):
 351             port_obj.host.initialize_scm()
 352             results['revision'] = port_obj.host.scm().head_svn_revision()
 353     except Exception as e:
 354         _log.warn("Failed to determine svn revision for checkout (cwd: %s, webkit_base: %s), leaving 'revision' key blank in full_results.json.\n%s" % (port_obj._filesystem.getcwd(), port_obj.path_from_webkit_base(), e))
 355         # Handle cases where we're running outside of version control.
 356         import traceback
 357         _log.debug('Failed to learn head svn revision:')
 358         _log.debug(traceback.format_exc())
 359         results['revision'] = ""
 360
 361     return results