webkitpy: Layout tests which have no output will succeed
authorjbedard@apple.com <jbedard@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 19 May 2017 16:23:15 +0000 (16:23 +0000)
committerjbedard@apple.com <jbedard@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 19 May 2017 16:23:15 +0000 (16:23 +0000)
https://bugs.webkit.org/show_bug.cgi?id=172322
<rdar://problem/32258350>

Reviewed by David Kilzer.

* Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
(SingleTestRunner._compare_text): Add FailreNotTested to failures if test has no output.
* Scripts/webkitpy/layout_tests/models/test_failures.py:
(determine_result_type): FailureNotTested means that the test could not be loaded. This
is classified as a MISSING failure.
(FailureNotTested): Added.
(FailureNotTested.message):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@217130 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py
Tools/Scripts/webkitpy/layout_tests/models/test_failures.py

index baea30c..f2f3698 100644 (file)
@@ -1,3 +1,19 @@
+2017-05-19  Jonathan Bedard  <jbedard@apple.com>
+
+        webkitpy: Layout tests which have no output will succeed
+        https://bugs.webkit.org/show_bug.cgi?id=172322
+        <rdar://problem/32258350>
+
+        Reviewed by David Kilzer.
+
+        * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
+        (SingleTestRunner._compare_text): Add FailreNotTested to failures if test has no output.
+        * Scripts/webkitpy/layout_tests/models/test_failures.py:
+        (determine_result_type): FailureNotTested means that the test could not be loaded. This
+        is classified as a MISSING failure.
+        (FailureNotTested): Added.
+        (FailureNotTested.message):
+
 2017-05-19  Yusuke Suzuki  <utatane.tea@gmail.com>
 
         Add SixSpeed benchmark to PerformanceTests
index 682a928..c8a119c 100644 (file)
@@ -240,6 +240,8 @@ class SingleTestRunner(object):
             failures.append(test_failures.FailureTextMismatch())
         elif actual_text and not expected_text:
             failures.append(test_failures.FailureMissingResult())
+        elif not actual_text and expected_text:
+            failures.append(test_failures.FailureNotTested())
         return failures
 
     def _compare_audio(self, expected_audio, actual_audio):
index 98723f4..bd811db 100644 (file)
@@ -62,7 +62,8 @@ def determine_result_type(failure_list):
     elif (FailureMissingResult in failure_types or
           FailureMissingImage in failure_types or
           FailureMissingImageHash in failure_types or
-          FailureMissingAudio in failure_types):
+          FailureMissingAudio in failure_types or
+          FailureNotTested in failure_types):
         return test_expectations.MISSING
     else:
         is_text_failure = FailureTextMismatch in failure_types
@@ -164,6 +165,10 @@ class FailureMissingResult(FailureText):
         return "-expected.txt was missing"
 
 
+class FailureNotTested(FailureText):
+    def message(self):
+        return 'test was not run'
+
 class FailureTextMismatch(FailureText):
     def message(self):
         return "text diff"
@@ -265,7 +270,7 @@ class FailureEarlyExit(TestFailure):
 
 # Convenient collection of all failure classes for anything that might
 # need to enumerate over them all.
-ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult,
+ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult, FailureNotTested,
                        FailureTextMismatch, FailureMissingImageHash,
                        FailureMissingImage, FailureImageHashMismatch,
                        FailureImageHashIncorrect, FailureReftestMismatch,