Log when leak detection changes the test result

https://bugs.webkit.org/show_bug.cgi?id=189293

Reviewed by Jon Lee.

When you have an "Leak" expectation in TestExpectations, it's confusing to see a test unexpectedly pass,
but then show up as an expected fail at the end (because leak detection happens at the end of a shard).
So log when leak detection changes a test result.

* Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py:
(LayoutTestRunner._annotate_results_with_additional_failures):
* Scripts/webkitpy/layout_tests/models/test_expectations.py:
(TestExpectations):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@235771 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/Tools/ChangeLog b/Tools/ChangeLog
index f722a48..69f9607 100644 (file)
--- a/Tools/ChangeLog
+++ b/Tools/ChangeLog
@@ -1,3 +1,19 @@
+2018-09-06  Simon Fraser  <simon.fraser@apple.com>
+
+        Log when leak detection changes the test result
+        https://bugs.webkit.org/show_bug.cgi?id=189293
+
+        Reviewed by Jon Lee.
+        
+        When you have an "Leak" expectation in TestExpectations, it's confusing to see a test unexpectedly pass,
+        but then show up as an expected fail at the end (because leak detection happens at the end of a shard).
+        So log when leak detection changes a test result.
+
+        * Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py:
+        (LayoutTestRunner._annotate_results_with_additional_failures):
+        * Scripts/webkitpy/layout_tests/models/test_expectations.py:
+        (TestExpectations):
+
 2018-09-06  Wenson Hsieh  <wenson_hsieh@apple.com>
 
         [macOS] [WK2] Support changing attributes for selected text (text shadow, underline, strike-through)

diff --git a/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py b/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py
index 506bd06..884d78d 100644 (file)
--- a/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py
+++ b/Tools/Scripts/webkitpy/layout_tests/controllers/layout_test_runner.py
@@ -202,6 +202,11 @@ class LayoutTestRunner(object):
                 expectations = self._expectations.filtered_expectations_for_test(new_result.test_name, self._options.pixel_tests or bool(new_result.reftest_type), self._options.world_leaks)
                 was_expected = self._expectations.matches_an_expected_result(new_result.test_name, existing_result.type, expectations)
                 now_expected = self._expectations.matches_an_expected_result(new_result.test_name, new_result.type, expectations)
+                if was_expected != now_expected:
+                    # When annotation is not just about leaks, this logging should be changed.
+                    _log.warning('  %s -> changed by leak detection from a %s (%s) to a %s (%s)' % (new_result.test_name,
+                        TestExpectations.EXPECTATION_DESCRIPTION[existing_result.type], 'expected' if was_expected else 'unexpected',
+                        TestExpectations.EXPECTATION_DESCRIPTION[new_result.type], 'expected' if now_expected else 'unexpected'))
                 run_results.change_result_to_failure(existing_result, new_result, was_expected, now_expected)
 
     def start_servers(self):

diff --git a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py
index e70c65c..46b14cb 100644 (file)
--- a/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py
+++ b/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py
@@ -815,6 +815,19 @@ class TestExpectations(object):
                     'leak': LEAK,
                     'skip': SKIP}
 
+    # Singulars
+    EXPECTATION_DESCRIPTION = {SKIP: 'skipped',
+                                PASS: 'pass',
+                                FAIL: 'failure',
+                                IMAGE: 'image-only failure',
+                                TEXT: 'text-only failure',
+                                IMAGE_PLUS_TEXT: 'image and text failure',
+                                AUDIO: 'audio failure',
+                                CRASH: 'crash',
+                                TIMEOUT: 'timeout',
+                                MISSING: 'missing',
+                                LEAK: 'leak'}
+
     # (aggregated by category, pass/fail/skip, type)
     EXPECTATION_DESCRIPTIONS = {SKIP: 'skipped',
                                 PASS: 'passes',

diff --git a/Tools/Scripts/webkitpy/layout_tests/views/printing.py b/Tools/Scripts/webkitpy/layout_tests/views/printing.py
index 0dec9c8..1d12371 100644 (file)
--- a/Tools/Scripts/webkitpy/layout_tests/views/printing.py
+++ b/Tools/Scripts/webkitpy/layout_tests/views/printing.py
@@ -321,7 +321,7 @@ class Printer(object):
     def print_finished_test(self, result, expected, exp_str, got_str):
         test_name = result.test_name
 
-        result_message = self._result_message(result.type, result.failures, expected, self._options.verbose)
+        result_message = self._result_message(result.type, result.failures, expected, exp_str, self._options.verbose)
 
         if self._options.details:
             self._print_test_trace(result, exp_str, got_str)
@@ -340,8 +340,11 @@ class Printer(object):
             self._completed_tests = []
         self._running_tests.remove(test_name)
 
-    def _result_message(self, result_type, failures, expected, verbose):
-        exp_string = ' unexpectedly' if not expected else ''
+    def _result_message(self, result_type, failures, expected, exp_str, verbose):
+        exp_string = ''
+        if not expected:
+            exp_string = ' (leak detection is pending)' if 'LEAK' in exp_str else ' unexpectedly'
+
         if result_type == test_expectations.PASS:
             return ' passed%s' % exp_string
         else: