New flakiness dashboard shouldn't treat tests with right expectations as failing

https://bugs.webkit.org/show_bug.cgi?id=123385

Reviewed by Alexey Proskuryakov.

We define failing tests to be tests with wrong expectations whose actual results are not PASS
since tests with TEXT, IMAGE, etc... failures do not turn the bots red as long as the expectation
of the same type is specified in TestExpectation files.

* public/include/test-results.php:
(FailingResultsJSONWriter): Inherit from WrongExpectationsResultsJSONWriter.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@158093 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/Websites/test-results/ChangeLog b/Websites/test-results/ChangeLog
index 3b23d75750b62026c8fcfe0c99957873ee873216..f86081beb3a6e71d764381debc04fe564ab8c201 100644 (file)
--- a/Websites/test-results/ChangeLog
+++ b/Websites/test-results/ChangeLog
@@ -1,3 +1,17 @@
+2013-10-26  Ryosuke Niwa  <rniwa@webkit.org>
+
+        New flakiness dashboard shouldn't treat tests with right expectations as failing
+        https://bugs.webkit.org/show_bug.cgi?id=123385
+
+        Reviewed by Alexey Proskuryakov.
+
+        We define failing tests to be tests with wrong expectations whose actual results are not PASS
+        since tests with TEXT, IMAGE, etc... failures do not turn the bots red as long as the expectation
+        of the same type is specified in TestExpectation files.
+
+        * public/include/test-results.php:
+        (FailingResultsJSONWriter): Inherit from WrongExpectationsResultsJSONWriter.
+
 2013-10-25  Ryosuke Niwa  <rniwa@webkit.org>
 
         Build fix. The query results weren't sorted by the latest commit time,

diff --git a/Websites/test-results/public/include/test-results.php b/Websites/test-results/public/include/test-results.php
index 77a1c2f833e51e69f877187ded8aca651fce0f6b..77bfc1df0e6e510df6315e610603d9e1c9f4af64 100644 (file)
--- a/Websites/test-results/public/include/test-results.php
+++ b/Websites/test-results/public/include/test-results.php
@@ -137,13 +137,6 @@ abstract class ResultsJSONWriter {
     abstract protected function pass_for_failure_type(&$results);
 }
 
-class FailingResultsJSONWriter extends ResultsJSONWriter {
-    public function __construct($fp) { parent::__construct($fp); }
-    protected function pass_for_failure_type(&$results) {
-        return $results[0]['actual'] == 'PASS';
-    }
-}
-
 class FlakyResultsJSONWriter extends ResultsJSONWriter {
     public function __construct($fp) { parent::__construct($fp); }
     protected function pass_for_failure_type(&$results) {
@@ -175,6 +168,13 @@ class WrongExpectationsResultsJSONWriter extends ResultsJSONWriter {
     }
 }
 
+class FailingResultsJSONWriter extends WrongExpectationsResultsJSONWriter {
+    public function __construct($fp) { parent::__construct($fp); }
+    protected function pass_for_failure_type(&$results) {
+        return $results[0]['actual'] == 'PASS' || parent::pass_for_failure_type($results);
+    }
+}
+
 class ResultsJSONGenerator {
     private $db;
     private $builder_id;