Make TestFailures show every time a possibly-flaky test failed, but hide it by default
authoraroben@apple.com <aroben@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 13 Jul 2011 16:30:22 +0000 (16:30 +0000)
committeraroben@apple.com <aroben@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 13 Jul 2011 16:30:22 +0000 (16:30 +0000)
It's useful to be able to see every time a flaky test failed to see whether it failed the
same way every time. But doing so takes a lot of space, so the list of failures is now
collapsed by default and can be revealed using a disclosure triangle.

Fixes <http://webkit.org/b/64455> TestFailures page doesn't show as much information for
flaky tests as I would like, even though the page is already so long

Reviewed by Dimitri Glazkov.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector.js:
(FlakyLayoutTestDetector.prototype.allFailures): Replaced flakinessExamples with this
function. Now returns all failures for the given test.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js:
Added. This just contains some simple tests of the FlakyLayoutTestDetector class. We'll add
more over time.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestHistoryAnalyzer.js:
(LayoutTestHistoryAnalyzer.prototype.start): Updated the documentation comment to reflect
that we no longer return passing builds for possibly-flaky tests.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/TestFailures.css:
(.existing-bugs-list, .suspect-revisions-list, .flakiness-examples-list): Make the list of
flakiness examples small, too, since it can get quite long.

(.disclosure-triangle):
(.expanded > .disclosure-triangle):
Simple styles for the disclosure triangle.

(.flakiness-examples-list): Collapse the list by default.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/ViewController.js:
(ViewController.prototype._displayBuilder): Pass the total number of builds analyzed to
_domForPossiblyFlakyTests.
(ViewController.prototype._domForPossiblyFlakyTests): Put a disclosure triangle to the left
of each test name, and the number of failures to the right. When the disclosure triangle is
clicked for the first time, we build up the list of failures and expand the element. After
that we just collapse or expand the element on subsequent clicks.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html:
Pulled in new tests.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@90922 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js [new file with mode: 0644]
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestHistoryAnalyzer.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/TestFailures.css
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/ViewController.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html
Tools/ChangeLog

index 6e1504846bf856eb1269eb3655d38fd124fce95e..9280b79986a60066e1705c225f5cba74b284df32 100644 (file)
@@ -70,43 +70,11 @@ FlakyLayoutTestDetector.prototype = {
         return newFlakyTests;
     },
 
-    flakinessExamples: function(testName) {
-        if (!(testName in this._tests) || this._tests[testName].state !== this._states.PossiblyFlaky)
+    allFailures: function(testName) {
+        if (!(testName in this._tests))
             return null;
 
-        var history = this._tests[testName].history;
-
-        var examples = [];
-        for (var i = 0; i < history.length - 1; ++i) {
-            var thisIsPassing = history[i].result.failureType === 'pass';
-            var nextIsPassing = history[i + 1].result.failureType === 'pass';
-            if (thisIsPassing === nextIsPassing)
-                continue;
-            var last = examples.last();
-            if (!last || last.build !== history[i].build)
-                examples.push(history[i]);
-            examples.push(history[i + 1]);
-        }
-
-        // The list of examples can get quite long. Instead of showing the whole list, we abbreviate
-        // by replacing the middle items with a separator.
-        const startAndEndAbbreviatedExamplesCount = 3;
-        console.assert(startAndEndAbbreviatedExamplesCount > 1);
-        var abbreviatedExamplesToShow = 2 * startAndEndAbbreviatedExamplesCount;
-        if (examples.length > abbreviatedExamplesToShow) {
-            var examplesBeforeSeparator = examples.slice(0, startAndEndAbbreviatedExamplesCount);
-            var examplesAfterSeparator = examples.slice(-startAndEndAbbreviatedExamplesCount);
-
-            // There's no real use in having two "pass" examples in a row immediately next to the
-            // separator.
-            if (examplesBeforeSeparator[examplesBeforeSeparator.length - 1].result.failureType === 'pass' && examplesBeforeSeparator[examplesBeforeSeparator.length - 2].result.failureType === 'pass')
-                examplesBeforeSeparator.splice(examplesBeforeSeparator.length - 1, 1);
-            if (examplesAfterSeparator[0].result.failureType === 'pass' && examplesAfterSeparator[1].result.failureType === 'pass')
-                examplesAfterSeparator.splice(0, 1);
-            examples = examplesBeforeSeparator.concat({ isSeparator: true }, examplesAfterSeparator);
-        }
-
-        return examples;
+        return this._tests[testName].history.filter(function(historyItem) { return historyItem.result.failureType !== 'pass' });
     },
 
     get possiblyFlakyTests() {
diff --git a/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js b/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js
new file mode 100644 (file)
index 0000000..cf74bee
--- /dev/null
@@ -0,0 +1,102 @@
+/*
+ * Copyright (C) 2011 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+(function() {
+
+module('FlakyLayoutTestDetector');
+
+const runs = [
+    {
+        buildName: 'build6',
+        failingTests: {
+            b: 'fail',
+        },
+        tooManyFailures: false,
+    },
+    {
+        buildName: 'build5',
+        failingTests: {
+            a: 'fail',
+            b: 'fail',
+        },
+        tooManyFailures: false,
+    },
+    {
+        buildName: 'build4',
+        failingTests: {
+            b: 'fail',
+        },
+        tooManyFailures: false,
+    },
+    {
+        buildName: 'build3',
+        failingTests: {
+            a: 'crash',
+            b: 'fail',
+        },
+        tooManyFailures: false,
+    },
+    {
+        buildName: 'build2',
+        failingTests: {
+            a: 'fail',
+        },
+        tooManyFailures: false,
+    },
+    {
+        buildName: 'build1',
+        failingTests: {},
+        tooManyFailures: false,
+    },
+];
+
+test('possiblyFlakyTests', 1, function() {
+    var detector = new FlakyLayoutTestDetector();
+    runs.forEach(function(run) { detector.incorporateTestResults(run.buildName, run.failingTests, run.tooManyFailures) });
+
+    deepEqual(detector.possiblyFlakyTests, ['a']);
+});
+
+test('allFailures', 3, function() {
+    var detector = new FlakyLayoutTestDetector();
+    runs.forEach(function(run) { detector.incorporateTestResults(run.buildName, run.failingTests, run.tooManyFailures) });
+
+    deepEqual(detector.allFailures('a'), [
+        { build: 'build5', result: 'fail' },
+        { build: 'build3', result: 'crash' },
+        { build: 'build2', result: 'fail' },
+    ]);
+
+    deepEqual(detector.allFailures('b'), [
+        { build: 'build6', result: 'fail' },
+        { build: 'build5', result: 'fail' },
+        { build: 'build4', result: 'fail' },
+        { build: 'build3', result: 'fail' },
+    ]);
+
+    equal(detector.allFailures('c'), null);
+});
+
+})();
index cbcadbd528bef1644012bbddf5e7957cf97289f2..a6628d7b5f9be9be32256c1bed51a14a5c15a935 100644 (file)
@@ -56,10 +56,8 @@ LayoutTestHistoryAnalyzer.prototype = {
      *     },
      *     'possiblyFlaky': {
      *         'fast/workers/worker-test.html': [
-     *             { 'build': 'r12345 (679)', 'result': 'pass' },
      *             { 'build': 'r12344 (678)', 'result': 'fail' },
-     *             { 'build': 'r12340 (676)', 'result': 'fail' },
-     *             { 'build': 'r12338 (675)', 'result': 'pass' },
+     *             { 'build': 'r12340 (676)', 'result': 'crash' },
      *         ],
      *     },
      * }
@@ -86,7 +84,7 @@ LayoutTestHistoryAnalyzer.prototype = {
                     possiblyFlaky: {},
                 };
                 self._flakinessDetector.possiblyFlakyTests.forEach(function(testName) {
-                    data.possiblyFlaky[testName] = self._flakinessDetector.flakinessExamples(testName);
+                    data.possiblyFlaky[testName] = self._flakinessDetector.allFailures(testName);
                 });
 
                 var nextIndex = buildIndex + 1;
index 62697cd844db843480266e17ed69f9431afed404..ca39b21c9e8070f63c73792dc1276b07ee9ffb23 100644 (file)
@@ -52,7 +52,7 @@ dt::after {
     margin-bottom: 0;
 }
 
-.existing-bugs-list, .suspect-revisions-list {
+.existing-bugs-list, .suspect-revisions-list, .flakiness-examples-list {
     font-size: smaller;
 }
 
@@ -64,3 +64,21 @@ dt::after {
     padding-left: 5em;
     font-size: larger;
 }
+
+.disclosure-triangle {
+    -webkit-transition: -webkit-transform 0.1s;
+    -webkit-user-select: none;
+    cursor: default;
+    display: inline-block;
+    padding: 0px 3px;
+}
+
+.expanded > .disclosure-triangle {
+    -webkit-transform: rotateZ(90deg);
+}
+
+.flakiness-examples-list {
+    -webkit-transition: height 0.25s;
+    height: 0px;
+    overflow: hidden;
+}
index 069ca32bf62c4365446e8ebbfb82b2b5d5f601fa..f6b1a24ef5be7c8d25c4a77059be15e8607f86d8 100644 (file)
@@ -70,7 +70,9 @@ ViewController.prototype = {
 
             var list = document.createElement('ol');
             list.id = 'failure-history';
-            Object.keys(data.history).forEach(function(buildName, buildIndex, buildNameArray) {
+
+            var buildNames = Object.keys(data.history)
+            buildNames.forEach(function(buildName, buildIndex, buildNameArray) {
                 var failingTestNames = Object.keys(data.history[buildName].tests);
                 if (!failingTestNames.length)
                     return;
@@ -107,7 +109,7 @@ ViewController.prototype = {
 
             self._mainContentElement.removeAllChildren();
             self._mainContentElement.appendChild(list);
-            self._mainContentElement.appendChild(self._domForPossiblyFlakyTests(builder, data.possiblyFlaky));
+            self._mainContentElement.appendChild(self._domForPossiblyFlakyTests(builder, data.possiblyFlaky, buildNames.length));
 
             if (!stillFetchingData)
                 PersistentCache.prune();
@@ -398,7 +400,7 @@ ViewController.prototype = {
         return result;
     },
 
-    _domForPossiblyFlakyTests: function(builder, possiblyFlakyTestData) {
+    _domForPossiblyFlakyTests: function(builder, possiblyFlakyTestData, buildCount) {
         var result = document.createDocumentFragment();
         var flakyTests = Object.keys(possiblyFlakyTestData);
         if (!flakyTests.length)
@@ -416,22 +418,46 @@ ViewController.prototype = {
         var self = this;
         flakyList.appendChildren(sorted(flakyTests).map(function(testName) {
             var item = document.createElement('li');
-            item.appendChild(document.createTextNode(testName));
-            var historyList = document.createElement('ol');
-            item.appendChild(historyList);
-            historyList.appendChildren(possiblyFlakyTestData[testName].map(function(historyItem) {
-                var item = document.createElement('li');
-                if (historyItem.isSeparator) {
-                    const verticalEllipsis = '\u22ee';
-                    item.appendChild(document.createTextNode(verticalEllipsis));
-                    item.className = 'flakiness-example-separator';
-                    return item;
+
+            var disclosureTriangle = document.createElement('span');
+            item.appendChild(disclosureTriangle);
+
+            disclosureTriangle.className = 'disclosure-triangle';
+            const blackRightPointingSmallTriangle = '\u25b8';
+            disclosureTriangle.appendChild(document.createTextNode(blackRightPointingSmallTriangle));
+
+            var failures = possiblyFlakyTestData[testName];
+
+            item.appendChild(document.createTextNode(testName + ' (failed ' + failures.length + ' out of ' + buildCount + ' times)'));
+
+            var failureList = document.createElement('ol');
+            item.appendChild(failureList);
+
+            failureList.className = 'flakiness-examples-list';
+
+            disclosureTriangle.addEventListener('click', function() {
+                item.classList.toggle('expanded');
+                if (!item.classList.contains('expanded')) {
+                    failureList.style.height = '';
+                    return;
                 }
-                item.appendChild(self._domForBuildName(builder, historyItem.build));
-                item.appendChild(document.createTextNode(': '));
-                item.appendChild(self._domForFailureDiagnosis(builder, historyItem.build, testName, historyItem.result));
-                return item;
-            }));
+
+                if (!failureList.firstChild) {
+                    failureList.appendChildren(failures.map(function(historyItem) {
+                        var item = document.createElement('li');
+                        item.appendChild(self._domForBuildName(builder, historyItem.build));
+                        item.appendChild(document.createTextNode(': '));
+                        item.appendChild(self._domForFailureDiagnosis(builder, historyItem.build, testName, historyItem.result));
+                        return item;
+                    }));
+                }
+
+                // CSS transitions can't transition to a value of 'auto', so we find out the actual
+                // value using getComputedStyle and transition to that.
+                failureList.style.height = 'auto';
+                failureList.style.height = getComputedStyle(failureList).height;
+            });
+
             return item;
         }));
 
index 1d4125f638dfb260ea4f33c351ff32e860739abd..36c3756160a1d870e0b77d644c631fce77d4dc23 100644 (file)
@@ -19,6 +19,8 @@
 <script src="Buildbot_unittests.js"></script>
 <script src="Builder.js"></script>
 <script src="Builder_unittests.js"></script>
+<script src="FlakyLayoutTestDetector.js"></script>
+<script src="FlakyLayoutTestDetector_unittests.js"></script>
 <script src="NewBugForm.js"></script>
 <script src="NewBugForm_unittests.js"></script>
 <script src="TestFailureBugForm.js"></script>
index 1a224a4fd271ae6cd0107a8e5cedcace4aeada31..3bb6b710963afe521da9f8e0f77d8546fe5b31dd 100644 (file)
@@ -1,3 +1,49 @@
+2011-07-13  Adam Roben  <aroben@apple.com>
+
+        Make TestFailures show every time a possibly-flaky test failed, but hide it by default
+
+        It's useful to be able to see every time a flaky test failed to see whether it failed the
+        same way every time. But doing so takes a lot of space, so the list of failures is now
+        collapsed by default and can be revealed using a disclosure triangle.
+
+        Fixes <http://webkit.org/b/64455> TestFailures page doesn't show as much information for
+        flaky tests as I would like, even though the page is already so long
+
+        Reviewed by Dimitri Glazkov.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector.js:
+        (FlakyLayoutTestDetector.prototype.allFailures): Replaced flakinessExamples with this
+        function. Now returns all failures for the given test.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/FlakyLayoutTestDetector_unittests.js:
+        Added. This just contains some simple tests of the FlakyLayoutTestDetector class. We'll add
+        more over time.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestHistoryAnalyzer.js:
+        (LayoutTestHistoryAnalyzer.prototype.start): Updated the documentation comment to reflect
+        that we no longer return passing builds for possibly-flaky tests.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/TestFailures.css:
+        (.existing-bugs-list, .suspect-revisions-list, .flakiness-examples-list): Make the list of
+        flakiness examples small, too, since it can get quite long.
+
+        (.disclosure-triangle):
+        (.expanded > .disclosure-triangle):
+        Simple styles for the disclosure triangle.
+
+        (.flakiness-examples-list): Collapse the list by default.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/ViewController.js:
+        (ViewController.prototype._displayBuilder): Pass the total number of builds analyzed to
+        _domForPossiblyFlakyTests.
+        (ViewController.prototype._domForPossiblyFlakyTests): Put a disclosure triangle to the left
+        of each test name, and the number of failures to the right. When the disclosure triangle is
+        clicked for the first time, we build up the list of failures and expand the element. After
+        that we just collapse or expand the element on subsequent clicks.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html:
+        Pulled in new tests.
+
 2011-07-13  Adam Roben  <aroben@apple.com>
         
         Teach TestFailures to understand NRWT's output when it exits early due to too many failures