garden-o-matic should not report flaking in expected failures as failures.
authordglazkov@chromium.org <dglazkov@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 18 Oct 2011 17:17:50 +0000 (17:17 +0000)
committerdglazkov@chromium.org <dglazkov@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 18 Oct 2011 17:17:50 +0000 (17:17 +0000)
https://bugs.webkit.org/show_bug.cgi?id=70279

Reviewed by Adam Barth.

On the bots, flakes that are already expected to fail aren't reported as failures (they keep the bot green).
For example, if TEXT failure is expected, a TEXT TIMEOUT result will not cause the redness. We should do the
same for garden-o-matic.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results.js: Spuriously changed to use Array iterators rather than jQuery APIs,
    rewrote the logic for determining expected or unexpected failures as a central analysis tool.
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results_unittests.js: Added unit tests.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@97770 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results_unittests.js
Tools/ChangeLog

index f1d6b8e..d566e5e 100644 (file)
@@ -76,16 +76,6 @@ results.kImageType = 'image'
 results.kTextType = 'text'
 // FIXME: There are more types of tests.
 
-function isFailure(result)
-{
-    return kFailingResults.indexOf(result) != -1;
-}
-
-function isSuccess(result)
-{
-    return result === PASS;
-}
-
 function possibleSuffixListFor(failureTypeList)
 {
     var suffixList = [];
@@ -201,59 +191,60 @@ var g_resultsCache = new base.AsynchronousCache(function (key, callback) {
     net.jsonp(key, callback);
 });
 
-function anyIsFailure(resultsList)
-{
-    return $.grep(resultsList, isFailure).length > 0;
-}
-
-function anyIsSuccess(resultsList)
-{
-    return $.grep(resultsList, isSuccess).length > 0;
-}
-
-function addImpliedExpectations(resultsList)
-{
-    if (resultsList.indexOf('FAIL') == -1)
-        return resultsList;
-    return resultsList.concat(kFailingResults);
-}
-
-results.unexpectedResults = function(resultNode)
-{
-    var actualResults = results.failureTypeList(resultNode.actual);
-    var expectedResults = addImpliedExpectations(results.failureTypeList(resultNode.expected))
-
-    return $.grep(actualResults, function(result) {
-        return expectedResults.indexOf(result) == -1;
-    });
-};
-
-function isExpectedOrUnexpectedFailure(resultNode)
-{
-    if (!resultNode)
-        return false;
-    var actualResults = results.failureTypeList(resultNode.actual);
-    if (anyIsSuccess(actualResults))
-        return false;
-    return anyIsFailure(actualResults);
-}
+results.ResultAnalyzer = base.extends(Object, {
+    init: function(resultNode)
+    {
+        this._actual = resultNode ? results.failureTypeList(resultNode.actual) : [];
+        this._expected = resultNode ? this._addImpliedExpectations(results.failureTypeList(resultNode.expected)) : [];
+    },
+    _addImpliedExpectations: function(resultsList)
+    {
+        if (resultsList.indexOf('FAIL') == -1)
+            return resultsList;
+        return resultsList.concat(kFailingResults);
+    },
+    _hasPass: function(results)
+    {
+        return results.indexOf(PASS) != -1;
+    },
+    unexpectedResults: function()
+    {
+        return this._actual.filter(function(result) {
+            return this._expected.indexOf(result) == -1;
+        }, this);
+    },
+    succeeded: function()
+    {
+        return this._hasPass(this._actual);
+    },
+    expectedToSucceed: function()
+    {
+        return this._hasPass(this._expected);
+    },
+    flaky: function()
+    {
+        return this._actual.length > 1;
+    },
+    hasUnexpectedFailures: function()
+    {
+        var difference = {};
+        this._actual.forEach(function(actual) {
+            difference[actual] = actual !== PASS;
+        });
+        this._expected.forEach(function(expected) {
+            if (expected !== PASS)
+                delete difference[expected];
+        });
+        return Object.keys(difference).some(function(key) {
+            return difference[key];
+        });
+    }
+})
 
 function isUnexpectedFailure(resultNode)
 {
-    if (!resultNode)
-        return false;
-    if (anyIsSuccess(results.failureTypeList(resultNode.actual)))
-        return false;
-    return anyIsFailure(results.unexpectedResults(resultNode));
-}
-
-function isUnexpectedSuccesses(resultNode)
-{
-    if (!resultNode)
-        return false;
-    if (anyIsFailure(results.failureTypeList(resultNode.actual)))
-        return false;
-    return anyIsSuccess(results.unexpectedResults(resultNode));
+    var analyzer = new results.ResultAnalyzer(resultNode);
+    return analyzer.hasUnexpectedFailures() && !analyzer.succeeded() && !analyzer.flaky();
 }
 
 function isResultNode(node)
@@ -263,7 +254,9 @@ function isResultNode(node)
 
 results.expectedOrUnexpectedFailures = function(resultsTree)
 {
-    return base.filterTree(resultsTree.tests, isResultNode, isExpectedOrUnexpectedFailure);
+    return base.filterTree(resultsTree.tests, isResultNode, function(resultNode) {
+        return !(new results.ResultAnalyzer(resultNode).succeeded());
+    });
 };
 
 results.unexpectedFailures = function(resultsTree)
@@ -273,7 +266,10 @@ results.unexpectedFailures = function(resultsTree)
 
 results.unexpectedSuccesses = function(resultsTree)
 {
-    return base.filterTree(resultsTree.tests, isResultNode, isUnexpectedSuccesses);
+    return base.filterTree(resultsTree.tests, isResultNode, function(resultNode) {
+        var analyzer = new results.ResultAnalyzer(resultNode);
+        return !analyzer.expectedToSucceed() && analyzer.succeeded() && !analyzer.flaky();
+    });
 };
 
 function resultsByTest(resultsByBuilder, filter)
@@ -318,7 +314,8 @@ results.collectUnexpectedResults = function(dictionaryOfResultNodes)
 {
     var collectedResults = [];
     $.each(dictionaryOfResultNodes, function(key, resultNode) {
-        collectedResults = collectedResults.concat(results.unexpectedResults(resultNode));
+        var analyzer = new results.ResultAnalyzer(resultNode);
+        collectedResults = collectedResults.concat(analyzer.unexpectedResults());
     });
     return base.uniquifyArray(collectedResults);
 };
index 137aac5..56e8e64 100644 (file)
@@ -40,6 +40,10 @@ unittest.kExampleResultsJSON = {
                 "expected": "PASS",
                 "actual": "PASS TEXT"
             },
+            "unexpected-failing-flaky-scrollbar.html": {
+                "expected": "TEXT",
+                "actual": "TIMEOUT TEXT"
+            },
             "unexpected-pass.html": {
                 "expected": "FAIL",
                 "actual": "PASS"
@@ -70,6 +74,59 @@ unittest.kExampleResultsJSON = {
     "revision": "90430"
 };
 
+test("ResultAnalyzer", 35, function() {
+    var analyzer;
+
+    analyzer = new results.ResultAnalyzer({expected: 'PASS', actual: 'TEXT'});
+    ok(analyzer.expectedToSucceed());
+    ok(analyzer.hasUnexpectedFailures());
+    deepEqual(analyzer.unexpectedResults(), ['TEXT']);
+    ok(!analyzer.succeeded());
+    ok(!analyzer.flaky());
+
+    analyzer = new results.ResultAnalyzer({expected: 'PASS TIMEOUT', actual: 'TEXT'});
+    ok(analyzer.expectedToSucceed());
+    ok(analyzer.hasUnexpectedFailures());
+    deepEqual(analyzer.unexpectedResults(), ['TEXT']);
+    ok(!analyzer.succeeded());
+    ok(!analyzer.flaky());
+
+    analyzer = new results.ResultAnalyzer({expected: 'TEXT', actual: 'TEXT TIMEOUT'});
+    ok(!analyzer.expectedToSucceed());
+    ok(analyzer.hasUnexpectedFailures());
+    deepEqual(analyzer.unexpectedResults(), ['TIMEOUT']);
+    ok(!analyzer.succeeded());
+    ok(analyzer.flaky());
+
+    analyzer = new results.ResultAnalyzer({expected: 'PASS', actual: 'TEXT TIMEOUT'});
+    ok(analyzer.expectedToSucceed());
+    ok(analyzer.hasUnexpectedFailures());
+    deepEqual(analyzer.unexpectedResults(), ['TEXT', 'TIMEOUT']);
+    ok(!analyzer.succeeded());
+    ok(analyzer.flaky());
+
+    analyzer = new results.ResultAnalyzer({expected: 'PASS TIMEOUT', actual: 'PASS TIMEOUT'});
+    ok(analyzer.expectedToSucceed());
+    ok(!analyzer.hasUnexpectedFailures());
+    deepEqual(analyzer.unexpectedResults(), []);
+    ok(analyzer.succeeded());
+    ok(analyzer.flaky());
+
+    analyzer = new results.ResultAnalyzer({expected: 'PASS TIMEOUT', actual: 'TIMEOUT PASS'});
+    ok(analyzer.expectedToSucceed());
+    ok(!analyzer.hasUnexpectedFailures());
+    deepEqual(analyzer.unexpectedResults(), []);
+    ok(analyzer.succeeded());
+    ok(analyzer.flaky());
+
+    analyzer = new results.ResultAnalyzer({expected: 'FAIL', actual: 'TIMEOUT'});
+    ok(!analyzer.expectedToSucceed());
+    ok(!analyzer.hasUnexpectedFailures());
+    deepEqual(analyzer.unexpectedResults(), []);
+    ok(!analyzer.succeeded());
+    ok(!analyzer.flaky());
+});
+
 test("unexpectedFailures", 1, function() {
     var unexpectedFailures = results.unexpectedFailures(unittest.kExampleResultsJSON);
     deepEqual(unexpectedFailures, {
index e73b8c4..fde753d 100644 (file)
@@ -1,3 +1,18 @@
+2011-10-18  Dimitri Glazkov  <dglazkov@chromium.org>
+
+        garden-o-matic should not report flaking in expected failures as failures.
+        https://bugs.webkit.org/show_bug.cgi?id=70279
+
+        Reviewed by Adam Barth.
+
+        On the bots, flakes that are already expected to fail aren't reported as failures (they keep the bot green).
+        For example, if TEXT failure is expected, a TEXT TIMEOUT result will not cause the redness. We should do the
+        same for garden-o-matic.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results.js: Spuriously changed to use Array iterators rather than jQuery APIs,
+            rewrote the logic for determining expected or unexpected failures as a central analysis tool.
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/results_unittests.js: Added unit tests.
+
 2011-10-18  Zan Dobersek  <zandobersek@gmail.com>
 
         [Gtk] Support for client-based geolocation