Change "losing test coverage" to cover all non-layout test problems in garden-o-matic
authorojan@chromium.org <ojan@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 10 May 2012 03:08:02 +0000 (03:08 +0000)
committerojan@chromium.org <ojan@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 10 May 2012 03:08:02 +0000 (03:08 +0000)
https://bugs.webkit.org/show_bug.cgi?id=86043

Reviewed by Adam Barth.

-Get rid of the gtest iframe.
-Cover all non-layout test problems. Before we wouldn't catch compile failures
or gtest failures since they were not on the webkit testing bots.
-Include the list of failing steps next to the builder name. This is a lot less
cluttered than you'd expect and already helped me find a compile failure early.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/builders.js:
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/builders_unittests.js:
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/config.js:
Delete unused map.
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/controllers.js:
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/garden-o-matic.js:
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui.js:
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/failures.js:
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/failures_unittests.js:
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/notifications.js:
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/notifications_unittests.js:
* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui_unittests.js:

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@116599 268f45cc-cd09-0410-ab3c-d52691b4dbfc

12 files changed:
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/builders.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/builders_unittests.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/config.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/controllers.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/garden-o-matic.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/failures.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/failures_unittests.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/notifications.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/notifications_unittests.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui_unittests.js
Tools/ChangeLog

index 9d4eff4..5a6211a 100644 (file)
@@ -41,32 +41,18 @@ function urlForBuildInfo(builderName, buildNumber)
     return kChromiumBuildBotURL + '/json/builders/' + encodeURIComponent(builderName) + '/builds/' + encodeURIComponent(buildNumber);
 }
 
-function isStepRequredForTestCoverage(step)
-{
-    switch(step.name) {
-    case kUpdateStepName:
-    case kUpdateScriptsStepName:
-    case kCompileStepName:
-    case kWebKitTestsStepName:
-        return true;
-    default:
-        return false;
-    }
-}
-
 function didFail(step)
 {
     if (step.name == kWebKitTestsStepName) {
         // run-webkit-tests fails to generate test coverage when it crashes or hangs.
         return step.text.indexOf(kCrashedOrHungOutputMarker) != -1;
     }
-    // FIXME: Is this the correct way to test for failure?
     return step.results[0] > 0;
 }
 
-function didFailStepRequredForTestCoverage(buildInfo)
+function failingSteps(buildInfo)
 {
-    return buildInfo.steps.filter(isStepRequredForTestCoverage).filter(didFail).length > 0;
+    return buildInfo.steps.filter(didFail);
 }
 
 function mostRecentCompletedBuildNumber(individualBuilderStatus)
@@ -90,11 +76,19 @@ var g_buildInfoCache = new base.AsynchronousCache(function(key, callback) {
 
 function fetchMostRecentBuildInfoByBuilder(callback)
 {
-    var buildInfoByBuilder = {};
-    var builderNames = Object.keys(config.kBuilders);
-    var requestTracker = new base.RequestTracker(builderNames.length, callback, [buildInfoByBuilder]);
     net.get(kChromiumBuildBotURL + '/json/builders', function(builderStatus) {
-        $.each(builderNames, function(index, builderName) {
+        var buildInfoByBuilder = {};
+        var builderNames = Object.keys(builderStatus);
+        var requestTracker = new base.RequestTracker(builderNames.length, callback, [buildInfoByBuilder]);
+        builderNames.forEach(function(builderName) {
+            // FIXME: Should garden-o-matic show these? I can imagine showing the deps bots being useful at least so
+            // that the gardener only need to look at garden-o-matic and never at the waterfall. Not really sure who
+            // watches the GPU bots.
+            if (builderName.indexOf('GPU') != -1 || builderName.indexOf('deps') != -1) {
+                requestTracker.requestComplete();
+                return;
+            }
+
             var buildNumber = mostRecentCompletedBuildNumber(builderStatus[builderName]);
             if (!buildNumber) {
                 buildInfoByBuilder[builderName] = null;
@@ -110,17 +104,18 @@ function fetchMostRecentBuildInfoByBuilder(callback)
     });
 }
 
-builders.buildersFailingStepRequredForTestCoverage = function(callback)
+builders.buildersFailingNonLayoutTests = function(callback)
 {
     fetchMostRecentBuildInfoByBuilder(function(buildInfoByBuilder) {
-        var builderNameList = [];
+        var failureList = {};
         $.each(buildInfoByBuilder, function(builderName, buildInfo) {
             if (!buildInfo)
                 return;
-            if (didFailStepRequredForTestCoverage(buildInfo))
-                builderNameList.push(builderName);
+            var failures = failingSteps(buildInfo);
+            if (failures.length)
+                failureList[builderName] = failures.map(function(failure) { return failure.name; });
         });
-        callback(builderNameList);
+        callback(failureList);
     });
 };
 
index c3c15c9..2f96be7 100644 (file)
@@ -582,7 +582,7 @@ var kExampleBuildInfoWithWebKitTestCrashJSON = {
     "times": [1318364210.066524, 1318366408.0732119]
 };
 
-test("buildersFailingStepRequredForTestCoverage", 3, function() {
+test("buildersFailing", 3, function() {
     var simulator = new NetworkSimulator();
 
     var failingBuildInfoJSON = JSON.parse(JSON.stringify(kExampleBuildInfoJSON));
@@ -608,8 +608,16 @@ test("buildersFailingStepRequredForTestCoverage", 3, function() {
     };
 
     simulator.runTest(function() {
-        builders.buildersFailingStepRequredForTestCoverage(function(builderNameList) {
-            deepEqual(builderNameList, ["Webkit Mac10.6"]);
+        builders.buildersFailingNonLayoutTests(function(builderNameList) {
+            deepEqual(builderNameList, {
+                "Webkit Linux": [
+                    "webkit_gpu_tests"
+                ],
+                "Webkit Mac10.6": [
+                    "compile",
+                    "webkit_gpu_tests"
+                ]
+            });
         });
     });
 
@@ -620,7 +628,7 @@ test("buildersFailingStepRequredForTestCoverage", 3, function() {
     ]);
 });
 
-test("buildersFailingStepRequredForTestCoverage (run-webkit-tests crash)", 3, function() {
+test("buildersFailing (run-webkit-tests crash)", 3, function() {
     var simulator = new NetworkSimulator();
 
     var builderStatusJSON = JSON.parse(JSON.stringify(kExampleBuilderStatusJSON));
@@ -648,8 +656,16 @@ test("buildersFailingStepRequredForTestCoverage (run-webkit-tests crash)", 3, fu
     };
 
     simulator.runTest(function() {
-        builders.buildersFailingStepRequredForTestCoverage(function(builderNameList) {
-            deepEqual(builderNameList, ["Webkit Linux"]);
+        builders.buildersFailingNonLayoutTests(function(builderNameList) {
+            deepEqual(builderNameList, {
+                "Webkit Linux": [
+                    "extract_build",
+                    "webkit_tests",
+                    "archive_webkit_tests_results",
+                    "webkit_gpu_tests",
+                    "archive_webkit_tests_gpu_results"
+                ]
+            });
         });
     });
 
index 8f5059c..39ae713 100644 (file)
@@ -44,15 +44,6 @@ config.kBuilders = {
     'Webkit Mac10.7': {version: 'lion'},
 };
 
-config.kBuildersThatOnlyCompile = [
-    'Webkit Win Builder',
-    'Webkit Win Builder (dbg)',
-    // FIXME: Where is the Linux Builder?
-    'Webkit Mac Builder',
-    'Webkit Mac Builder (dbg)',
-    'Win Builder',
-];
-
 config.kTracURL = 'http://trac.webkit.org';
 config.kBugzillaURL = 'https://bugs.webkit.org';
 config.kLocalServerURL = 'http://127.0.0.1:8127';
index 4f95f57..2b98add 100644 (file)
@@ -287,9 +287,9 @@ controllers.FailingBuilders = base.extends(Object, {
         this._message = message;
         this._notification = null;
     },
-    update: function(builderNameList)
+    update: function(failuresList)
     {
-        if (builderNameList.length == 0) {
+        if (Object.keys(failuresList).length == 0) {
             if (this._notification) {
                 this._notification.dismiss();
                 this._notification = null;
@@ -302,7 +302,7 @@ controllers.FailingBuilders = base.extends(Object, {
         }
         // FIXME: We should provide regression ranges for the failing builders.
         // This doesn't seem to happen often enough to worry too much about that, however.
-        this._notification.setFailingBuilders(builderNameList);
+        this._notification.setFailingBuilders(failuresList);
     }
 });
 
index 028f3dd..ab382e1 100644 (file)
@@ -34,7 +34,7 @@ var g_buildersFailing = null;
 var g_unexpectedFailuresController = null;
 var g_failuresController = null;
 
-var g_losingTestCoverageBuilders = null;
+var g_nonLayoutTestFailureBuilders = null;
 
 function update()
 {
@@ -51,7 +51,7 @@ function update()
 
     g_info.add(updating);
 
-    builders.buildersFailingStepRequredForTestCoverage(g_losingTestCoverageBuilders.update.bind(g_losingTestCoverageBuilders));
+    builders.buildersFailingNonLayoutTests(g_nonLayoutTestFailureBuilders.update.bind(g_nonLayoutTestFailureBuilders));
 
     base.callInParallel([model.updateRecentCommits, model.updateResultsByBuilder], function() {
         if (g_failuresController)
@@ -90,7 +90,6 @@ $(document).ready(function() {
         showResults: function(resultsView)
         {
             var resultsContainer = onebar.results();
-            console.log(resultsContainer);
             $(resultsContainer).empty().append(resultsView);
             onebar.select('results');
         }
@@ -100,7 +99,7 @@ $(document).ready(function() {
     g_unexpectedFailuresController = new controllers.UnexpectedFailures(model.state, unexpectedFailuresView, onebarController);
 
     g_info = new ui.notifications.Stream();
-    g_losingTestCoverageBuilders = new controllers.FailingBuilders(g_info, 'Losing test coverage');
+    g_nonLayoutTestFailureBuilders = new controllers.FailingBuilders(g_info, 'Non-layout test failures');
 
     // FIXME: This should be an Action object.
     var updateButton = document.body.insertBefore(document.createElement('button'), document.body.firstChild);
index 404a069..a4101d7 100644 (file)
@@ -55,23 +55,6 @@ ui.urlForEmbeddedFlakinessDashboard = function(opt_testNameList)
     return ui.urlForFlakinessDashboard(opt_testNameList) + '&showChrome=false';
 }
 
-ui.urlForChromiumGtestSummary = function()
-{
-    return 'http://build.chromium.org/p/chromium.webkit/horizontal_one_box_per_builder?' +
-        'builder=Win%20Builder' +
-        '&builder=Win%20Reliability%20Builder' +
-        '&builder=Vista%20Tests&builder=Win%20Reliability' +
-        '&builder=Win+(dbg)' +
-        '&builder=Win%20Shared%20Builder%20%28dbg%29' +
-        '&builder=Mac10.6%20Tests' +
-        '&builder=Mac+Builder+(dbg)' +
-        '&builder=Linux%20Tests' +
-        '&builder=Linux%20Valgrind' +
-        '&builder=Vista%20Perf' +
-        '&builder=Linux%20Perf' +
-        '&builder=Mac10.6%20Perf';
-}
-
 ui.rolloutReasonForTestNameList = function(testNameList)
 {
     return 'Broke:\n' + testNameList.map(function(testName) {
@@ -91,15 +74,13 @@ ui.onebar = base.extends('div', {
             '</ul>' +
             '<div id="unexpected"></div>' +
             '<div id="expected"></div>' +
-            '<div id="results"></div>' +
-            '<div id="chromium-gtests"><span id="gtest-label">Chromium gtest bots:</span>' +
-                '<iframe scrolling="no" src="' + ui.urlForChromiumGtestSummary() + '"></iframe>' +
-            '</div>';
+            '<div id="results"></div>';
         this._tabNames = [
             'unexpected',
             'expected',
             'results',
         ]
+
         this._tabIndexToSavedScrollOffset = {};
         this._tabs = $(this).tabs({
             disabled: [2],
index d3d631a..2c53bfc 100644 (file)
@@ -31,17 +31,23 @@ ui.failures = ui.failures || {};
 var kBuildingResult = 'BUILDING';
 
 ui.failures.Builder = base.extends('a', {
-    init: function(builderName)
+    init: function(builderName, failures)
     {
         var configuration = config.kBuilders[builderName];
-        if (configuration.version)
-            this._addSpan('version', configuration.version);
-        if (configuration.is64bit)
-            this._addSpan('architecture', '64-bit');
-        this._configuration = configuration;
+        if (configuration) {
+            if (configuration.version)
+                this._addSpan('version', configuration.version);
+            if (configuration.is64bit)
+                this._addSpan('architecture', '64-bit');
+            this._configuration = configuration;
+        } else
+            this._addSpan('version', builderName);
+
         this.className = 'failing-builder';
         this.target = '_blank';
         this.href = ui.displayURLForBuilder(builderName);
+        if (failures)
+            this._addSpan('failures', ' ' + failures.join(', '));
     },
     _addSpan: function(className, text)
     {
@@ -51,7 +57,7 @@ ui.failures.Builder = base.extends('a', {
     },
     equals: function(configuration)
     {
-        return this._configuration.is64bit == configuration.is64bit && this._configuration.version == configuration.version; 
+        return this._configuration && this._configuration.is64bit == configuration.is64bit && this._configuration.version == configuration.version; 
     }
 });
 
index 4a96613..7e5197e 100644 (file)
 
 module('ui.failures');
 
-test('Builder', 7, function() {
-    raises(function() {
-        new ui.failures.Builder();
-    });
-
+test('Builder', 6, function() {
     var configuration;
-    configuration = new ui.failures.Builder("Webkit Linux");
+    configuration = new ui.failures.Builder("Webkit Linux", ["update", "webkit_tests"]);
     deepEqual(Object.getOwnPropertyNames(configuration.__proto__).sort(), [
         '_addSpan',
         'equals',
         'init',
     ]);
-    equal(configuration.outerHTML, '<a class="failing-builder" target="_blank" href="http://build.chromium.org/p/chromium.webkit/waterfall?builder=Webkit+Linux"><span class="version">lucid</span><span class="architecture">64-bit</span></a>');
+    equal(configuration.outerHTML, '<a class="failing-builder" target="_blank" href="http://build.chromium.org/p/chromium.webkit/waterfall?builder=Webkit+Linux"><span class="version">lucid</span><span class="architecture">64-bit</span><span class="failures"> update, webkit_tests</span></a>');
     configuration = new ui.failures.Builder("Webkit Win");
     equal(configuration.outerHTML, '<a class="failing-builder" target="_blank" href="http://build.chromium.org/p/chromium.webkit/waterfall?builder=Webkit+Win"><span class="version">xp</span></a>');
     configuration._addSpan('foo', 'bar');
index ef713c2..572082e 100644 (file)
@@ -243,12 +243,12 @@ ui.notifications.BuildersFailing = base.extends(ui.notifications.Failure, {
     {
         this._problem.insertBefore(document.createTextNode(message + ':'), this._problem.firstChild);
     },
-    setFailingBuilders: function(builderNameList)
+    setFailingBuilders: function(failuresList)
     {
-        $(this._effects).empty().append(builderNameList.map(function(builderName) {
+        $(this._effects).empty().append(Object.keys(failuresList).map(function(builderName) {
             var effect = document.createElement('li');
             effect.className = 'builder';
-            effect.appendChild(new ui.failures.Builder(builderName));
+            effect.appendChild(new ui.failures.Builder(builderName, failuresList[builderName]));
             return effect;
         }));
     }
index 241085f..6c6a18c 100644 (file)
@@ -318,7 +318,7 @@ test('FailingTestsSummary (grouping)', 1, function() {
 
 test('BuildersFailing', 1, function() {
     var builderFailing = new ui.notifications.BuildersFailing('Disasterifying');
-    builderFailing.setFailingBuilders(['Webkit Linux', 'Webkit Vista']);
+    builderFailing.setFailingBuilders({'Webkit Linux': ['compile'], 'Webkit Vista': ['webkit_tests', 'update']);
     equal(builderFailing.innerHTML,
         '<div class="how">' +
             '<time class="relative"></time>' +
index 1c80fff..1b815b8 100644 (file)
@@ -61,23 +61,7 @@ test("ui.onebar", 3, function() {
         '</ul>' +
         '<div id="unexpected" class="ui-tabs-panel ui-widget-content ui-corner-bottom"></div>' +
         '<div id="expected" class="ui-tabs-panel ui-widget-content ui-corner-bottom ui-tabs-hide"></div>' +
-        '<div id="results" class="ui-tabs-panel ui-widget-content ui-corner-bottom ui-tabs-hide"></div>' +
-        '<div id="chromium-gtests"><span id="gtest-label">Chromium gtest bots:</span>' +
-            '<iframe scrolling="no" src="http://build.chromium.org/p/chromium.webkit/horizontal_one_box_per_builder?' +
-                'builder=Win%20Builder' +
-                '&amp;builder=Win%20Reliability%20Builder' +
-                '&amp;builder=Vista%20Tests' +
-                '&amp;builder=Win%20Reliability' +
-                '&amp;builder=Win+(dbg)' +
-                '&amp;builder=Win%20Shared%20Builder%20%28dbg%29' +
-                '&amp;builder=Mac10.6%20Tests' +
-                '&amp;builder=Mac+Builder+(dbg)' +
-                '&amp;builder=Linux%20Tests' +
-                '&amp;builder=Linux%20Valgrind' +
-                '&amp;builder=Vista%20Perf' +
-                '&amp;builder=Linux%20Perf' +
-                '&amp;builder=Mac10.6%20Perf"></iframe>' +
-        '</div>');
+        '<div id="results" class="ui-tabs-panel ui-widget-content ui-corner-bottom ui-tabs-hide"></div>');
 
     onebar.select('expected');
     equal(window.location.hash, '#expected');
index 708869f..8ff1e38 100644 (file)
@@ -1,3 +1,29 @@
+2012-05-09  Ojan Vafai  <ojan@chromium.org>
+
+        Change "losing test coverage" to cover all non-layout test problems in garden-o-matic
+        https://bugs.webkit.org/show_bug.cgi?id=86043
+
+        Reviewed by Adam Barth.
+
+        -Get rid of the gtest iframe.
+        -Cover all non-layout test problems. Before we wouldn't catch compile failures
+        or gtest failures since they were not on the webkit testing bots.
+        -Include the list of failing steps next to the builder name. This is a lot less
+        cluttered than you'd expect and already helped me find a compile failure early.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/builders.js:
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/builders_unittests.js:
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/config.js:
+        Delete unused map.
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/controllers.js:
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/garden-o-matic.js:
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui.js:
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/failures.js:
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/failures_unittests.js:
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/notifications.js:
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui/notifications_unittests.js:
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/scripts/ui_unittests.js:
+
 2012-05-09  Jessie Berlin  <jberlin@apple.com>
 
         Crash using the new WKBundleDOMWindowExtensions APIs.