Make TestFailures show every time a possibly-flaky test failed, but hide it by default
authoraroben@apple.com <aroben@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 13 Jul 2011 16:30:22 +0000 (16:30 +0000)
committeraroben@apple.com <aroben@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 13 Jul 2011 16:30:22 +0000 (16:30 +0000)
It's useful to be able to see every time a flaky test failed to see whether it failed the
same way every time. But doing so takes a lot of space, so the list of failures is now
collapsed by default and can be revealed using a disclosure triangle.

Fixes <http://webkit.org/b/64455> TestFailures page doesn't show as much information for
flaky tests as I would like, even though the page is already so long

Reviewed by Dimitri Glazkov.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector.js:
(FlakyLayoutTestDetector.prototype.allFailures): Replaced flakinessExamples with this
function. Now returns all failures for the given test.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js:
Added. This just contains some simple tests of the FlakyLayoutTestDetector class. We'll add
more over time.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestHistoryAnalyzer.js:
(LayoutTestHistoryAnalyzer.prototype.start): Updated the documentation comment to reflect
that we no longer return passing builds for possibly-flaky tests.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/TestFailures.css:
(.existing-bugs-list, .suspect-revisions-list, .flakiness-examples-list): Make the list of
flakiness examples small, too, since it can get quite long.

(.disclosure-triangle):
(.expanded > .disclosure-triangle):
Simple styles for the disclosure triangle.

(.flakiness-examples-list): Collapse the list by default.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/ViewController.js:
(ViewController.prototype._displayBuilder): Pass the total number of builds analyzed to
_domForPossiblyFlakyTests.
(ViewController.prototype._domForPossiblyFlakyTests): Put a disclosure triangle to the left
of each test name, and the number of failures to the right. When the disclosure triangle is
clicked for the first time, we build up the list of failures and expand the element. After
that we just collapse or expand the element on subsequent clicks.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html:
Pulled in new tests.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@90922 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js [new file with mode: 0644]
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestHistoryAnalyzer.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/TestFailures.css
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/ViewController.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html
Tools/ChangeLog

index 6e15048..9280b79 100644 (file)
@@ -70,43 +70,11 @@ FlakyLayoutTestDetector.prototype = {
         return newFlakyTests;
     },
 
-    flakinessExamples: function(testName) {
-        if (!(testName in this._tests) || this._tests[testName].state !== this._states.PossiblyFlaky)
+    allFailures: function(testName) {
+        if (!(testName in this._tests))
             return null;
 
-        var history = this._tests[testName].history;
-
-        var examples = [];
-        for (var i = 0; i < history.length - 1; ++i) {
-            var thisIsPassing = history[i].result.failureType === 'pass';
-            var nextIsPassing = history[i + 1].result.failureType === 'pass';
-            if (thisIsPassing === nextIsPassing)
-                continue;
-            var last = examples.last();
-            if (!last || last.build !== history[i].build)
-                examples.push(history[i]);
-            examples.push(history[i + 1]);
-        }
-
-        // The list of examples can get quite long. Instead of showing the whole list, we abbreviate
-        // by replacing the middle items with a separator.
-        const startAndEndAbbreviatedExamplesCount = 3;
-        console.assert(startAndEndAbbreviatedExamplesCount > 1);
-        var abbreviatedExamplesToShow = 2 * startAndEndAbbreviatedExamplesCount;
-        if (examples.length > abbreviatedExamplesToShow) {
-            var examplesBeforeSeparator = examples.slice(0, startAndEndAbbreviatedExamplesCount);
-            var examplesAfterSeparator = examples.slice(-startAndEndAbbreviatedExamplesCount);
-
-            // There's no real use in having two "pass" examples in a row immediately next to the
-            // separator.
-            if (examplesBeforeSeparator[examplesBeforeSeparator.length - 1].result.failureType === 'pass' && examplesBeforeSeparator[examplesBeforeSeparator.length - 2].result.failureType === 'pass')
-                examplesBeforeSeparator.splice(examplesBeforeSeparator.length - 1, 1);
-            if (examplesAfterSeparator[0].result.failureType === 'pass' && examplesAfterSeparator[1].result.failureType === 'pass')
-                examplesAfterSeparator.splice(0, 1);
-            examples = examplesBeforeSeparator.concat({ isSeparator: true }, examplesAfterSeparator);
-        }
-
-        return examples;
+        return this._tests[testName].history.filter(function(historyItem) { return historyItem.result.failureType !== 'pass' });
     },
 
     get possiblyFlakyTests() {
diff --git a/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js b/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js
new file mode 100644 (file)
index 0000000..cf74bee
--- /dev/null
@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2011 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+(function() {
+
+module('FlakyLayoutTestDetector');
+
+const runs = [
+    {
+        buildName: 'build6',
+        failingTests: {
+            b: 'fail',
+        },
+        tooManyFailures: false,
+    },
+    {
+        buildName: 'build5',
+        failingTests: {
+            a: 'fail',
+            b: 'fail',
+        },
+        tooManyFailures: false,
+    },
+    {
+        buildName: 'build4',
+        failingTests: {
+            b: 'fail',
+        },
+        tooManyFailures: false,
+    },
+    {
+        buildName: 'build3',
+        failingTests: {
+            a: 'crash',
+            b: 'fail',
+        },
+        tooManyFailures: false,
+    },
+    {
+        buildName: 'build2',
+        failingTests: {
+            a: 'fail',
+        },
+        tooManyFailures: false,
+    },
+    {
+        buildName: 'build1',
+        failingTests: {},
+        tooManyFailures: false,
+    },
+];
+
+test('possiblyFlakyTests', 1, function() {
+    var detector = new FlakyLayoutTestDetector();
+    runs.forEach(function(run) { detector.incorporateTestResults(run.buildName, run.failingTests, run.tooManyFailures) });
+
+    deepEqual(detector.possiblyFlakyTests, ['a']);
+});
+
+test('allFailures', 3, function() {
+    var detector = new FlakyLayoutTestDetector();
+    runs.forEach(function(run) { detector.incorporateTestResults(run.buildName, run.failingTests, run.tooManyFailures) });
+
+    deepEqual(detector.allFailures('a'), [
+        { build: 'build5', result: 'fail' },
+        { build: 'build3', result: 'crash' },
+        { build: 'build2', result: 'fail' },
+    ]);
+
+    deepEqual(detector.allFailures('b'), [
+        { build: 'build6', result: 'fail' },
+        { build: 'build5', result: 'fail' },
+        { build: 'build4', result: 'fail' },
+        { build: 'build3', result: 'fail' },
+    ]);
+
+    equal(detector.allFailures('c'), null);
+});
+
+})();
index cbcadbd..a6628d7 100644 (file)
@@ -56,10 +56,8 @@ LayoutTestHistoryAnalyzer.prototype = {
      *     },
      *     'possiblyFlaky': {
      *         'fast/workers/worker-test.html': [
-     *             { 'build': 'r12345 (679)', 'result': 'pass' },
      *             { 'build': 'r12344 (678)', 'result': 'fail' },
-     *             { 'build': 'r12340 (676)', 'result': 'fail' },
-     *             { 'build': 'r12338 (675)', 'result': 'pass' },
+     *             { 'build': 'r12340 (676)', 'result': 'crash' },
      *         ],
      *     },
      * }
@@ -86,7 +84,7 @@ LayoutTestHistoryAnalyzer.prototype = {
                     possiblyFlaky: {},
                 };
                 self._flakinessDetector.possiblyFlakyTests.forEach(function(testName) {
-                    data.possiblyFlaky[testName] = self._flakinessDetector.flakinessExamples(testName);
+                    data.possiblyFlaky[testName] = self._flakinessDetector.allFailures(testName);
                 });
 
                 var nextIndex = buildIndex + 1;
index 62697cd..ca39b21 100644 (file)
@@ -52,7 +52,7 @@ dt::after {
     margin-bottom: 0;
 }
 
-.existing-bugs-list, .suspect-revisions-list {
+.existing-bugs-list, .suspect-revisions-list, .flakiness-examples-list {
     font-size: smaller;
 }
 
@@ -64,3 +64,21 @@ dt::after {
     padding-left: 5em;
     font-size: larger;
 }
+
+.disclosure-triangle {
+    -webkit-transition: -webkit-transform 0.1s;
+    -webkit-user-select: none;
+    cursor: default;
+    display: inline-block;
+    padding: 0px 3px;
+}
+
+.expanded > .disclosure-triangle {
+    -webkit-transform: rotateZ(90deg);
+}
+
+.flakiness-examples-list {
+    -webkit-transition: height 0.25s;
+    height: 0px;
+    overflow: hidden;
+}
index 069ca32..f6b1a24 100644 (file)
@@ -70,7 +70,9 @@ ViewController.prototype = {
 
             var list = document.createElement('ol');
             list.id = 'failure-history';
-            Object.keys(data.history).forEach(function(buildName, buildIndex, buildNameArray) {
+
+            var buildNames = Object.keys(data.history)
+            buildNames.forEach(function(buildName, buildIndex, buildNameArray) {
                 var failingTestNames = Object.keys(data.history[buildName].tests);
                 if (!failingTestNames.length)
                     return;
@@ -107,7 +109,7 @@ ViewController.prototype = {
 
             self._mainContentElement.removeAllChildren();
             self._mainContentElement.appendChild(list);
-            self._mainContentElement.appendChild(self._domForPossiblyFlakyTests(builder, data.possiblyFlaky));
+            self._mainContentElement.appendChild(self._domForPossiblyFlakyTests(builder, data.possiblyFlaky, buildNames.length));
 
             if (!stillFetchingData)
                 PersistentCache.prune();
@@ -398,7 +400,7 @@ ViewController.prototype = {
         return result;
     },
 
-    _domForPossiblyFlakyTests: function(builder, possiblyFlakyTestData) {
+    _domForPossiblyFlakyTests: function(builder, possiblyFlakyTestData, buildCount) {
         var result = document.createDocumentFragment();
         var flakyTests = Object.keys(possiblyFlakyTestData);
         if (!flakyTests.length)
@@ -416,22 +418,46 @@ ViewController.prototype = {
         var self = this;
         flakyList.appendChildren(sorted(flakyTests).map(function(testName) {
             var item = document.createElement('li');
-            item.appendChild(document.createTextNode(testName));
-            var historyList = document.createElement('ol');
-            item.appendChild(historyList);
-            historyList.appendChildren(possiblyFlakyTestData[testName].map(function(historyItem) {
-                var item = document.createElement('li');
-                if (historyItem.isSeparator) {
-                    const verticalEllipsis = '\u22ee';
-                    item.appendChild(document.createTextNode(verticalEllipsis));
-                    item.className = 'flakiness-example-separator';
-                    return item;
+
+            var disclosureTriangle = document.createElement('span');
+            item.appendChild(disclosureTriangle);
+
+            disclosureTriangle.className = 'disclosure-triangle';
+            const blackRightPointingSmallTriangle = '\u25b8';
+            disclosureTriangle.appendChild(document.createTextNode(blackRightPointingSmallTriangle));
+
+            var failures = possiblyFlakyTestData[testName];
+
+            item.appendChild(document.createTextNode(testName + ' (failed ' + failures.length + ' out of ' + buildCount + ' times)'));
+
+            var failureList = document.createElement('ol');
+            item.appendChild(failureList);
+
+            failureList.className = 'flakiness-examples-list';
+
+            disclosureTriangle.addEventListener('click', function() {
+                item.classList.toggle('expanded');
+                if (!item.classList.contains('expanded')) {
+                    failureList.style.height = '';
+                    return;
                 }
-                item.appendChild(self._domForBuildName(builder, historyItem.build));
-                item.appendChild(document.createTextNode(': '));
-                item.appendChild(self._domForFailureDiagnosis(builder, historyItem.build, testName, historyItem.result));
-                return item;
-            }));
+
+                if (!failureList.firstChild) {
+                    failureList.appendChildren(failures.map(function(historyItem) {
+                        var item = document.createElement('li');
+                        item.appendChild(self._domForBuildName(builder, historyItem.build));
+                        item.appendChild(document.createTextNode(': '));
+                        item.appendChild(self._domForFailureDiagnosis(builder, historyItem.build, testName, historyItem.result));
+                        return item;
+                    }));
+                }
+
+                // CSS transitions can't transition to a value of 'auto', so we find out the actual
+                // value using getComputedStyle and transition to that.
+                failureList.style.height = 'auto';
+                failureList.style.height = getComputedStyle(failureList).height;
+            });
+
             return item;
         }));
 
index 1d4125f..36c3756 100644 (file)
@@ -19,6 +19,8 @@
 <script src="Buildbot_unittests.js"></script>
 <script src="Builder.js"></script>
 <script src="Builder_unittests.js"></script>
+<script src="FlakyLayoutTestDetector.js"></script>
+<script src="FlakyLayoutTestDetector_unittests.js"></script>
 <script src="NewBugForm.js"></script>
 <script src="NewBugForm_unittests.js"></script>
 <script src="TestFailureBugForm.js"></script>
index 1a224a4..3bb6b71 100644 (file)
@@ -1,4 +1,50 @@
 2011-07-13  Adam Roben  <aroben@apple.com>
+
+        Make TestFailures show every time a possibly-flaky test failed, but hide it by default
+
+        It's useful to be able to see every time a flaky test failed to see whether it failed the
+        same way every time. But doing so takes a lot of space, so the list of failures is now
+        collapsed by default and can be revealed using a disclosure triangle.
+
+        Fixes <http://webkit.org/b/64455> TestFailures page doesn't show as much information for
+        flaky tests as I would like, even though the page is already so long
+
+        Reviewed by Dimitri Glazkov.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector.js:
+        (FlakyLayoutTestDetector.prototype.allFailures): Replaced flakinessExamples with this
+        function. Now returns all failures for the given test.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js:
+        Added. This just contains some simple tests of the FlakyLayoutTestDetector class. We'll add
+        more over time.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestHistoryAnalyzer.js:
+        (LayoutTestHistoryAnalyzer.prototype.start): Updated the documentation comment to reflect
+        that we no longer return passing builds for possibly-flaky tests.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/TestFailures.css:
+        (.existing-bugs-list, .suspect-revisions-list, .flakiness-examples-list): Make the list of
+        flakiness examples small, too, since it can get quite long.
+
+        (.disclosure-triangle):
+        (.expanded > .disclosure-triangle):
+        Simple styles for the disclosure triangle.
+
+        (.flakiness-examples-list): Collapse the list by default.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/ViewController.js:
+        (ViewController.prototype._displayBuilder): Pass the total number of builds analyzed to
+        _domForPossiblyFlakyTests.
+        (ViewController.prototype._domForPossiblyFlakyTests): Put a disclosure triangle to the left
+        of each test name, and the number of failures to the right. When the disclosure triangle is
+        clicked for the first time, we build up the list of failures and expand the element. After
+        that we just collapse or expand the element on subsequent clicks.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html:
+        Pulled in new tests.
+
+2011-07-13  Adam Roben  <aroben@apple.com>
         
         Teach TestFailures to understand NRWT's output when it exits early due to too many failures