Teach TestFailures how to load, parse, and interpret NRWT test results
authoraroben@apple.com <aroben@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 6 Jul 2011 20:42:14 +0000 (20:42 +0000)
committeraroben@apple.com <aroben@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 6 Jul 2011 20:42:14 +0000 (20:42 +0000)
Fixes <http://webkit.org/b/61877> TestFailures page doesn't show testers that use
new-run-webkit-tests

Reviewed by Adam Barth.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js:
(Builder.prototype.failureDiagnosisTextAndURL): Added support for the new 'flaky' failure
type. For now we don't account for image-only flakes (but TestFailures doesn't deal with
pixel tests at all currently).
(Builder.prototype.getNumberOfFailingTests): Relaxed the regex used for parsing the number
of failing tests from the buildbot output. Make sure not to count "new passes" (a new
category introduced by NRWT) as failures.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js:
(LayoutTestResultsLoader.prototype.start): Moved a bunch of code from here to
_fetchAndParseORWTResults. This function now attempts to load NRWT results, then falls back
to loading ORWT results.
(LayoutTestResultsLoader.prototype._fetchAndParseNRWTResults): Added. Tries to load and
parse the full_results.json file for the given build.
(LayoutTestResultsLoader.prototype._fetchAndParseORWTResults): Added. Code came from start.
Fixed a bug along the way where we were sometimes calling the error callback instead of the
success callback when all tests passed.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/NRWTResultsParser.js: Added.
(NRWTResultsParser): Do-nothing constructor.
(NRWTResultsParser.prototype.parse): Uses eval() (eww!) to get the test results out of the
JS string, then iterates over all the tests in the results data and builds up a data
structure matching what ORWTResultsParser returns.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Utilities.js:
(Array.prototype.contains):
(String.prototype.contains):
Added these simple helper functions.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/index.html: Pull in
NRWTResultsParser.js.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@90489 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/NRWTResultsParser.js [new file with mode: 0644]
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Utilities.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/index.html
Tools/ChangeLog

index abac1b67f37e59d7a660809b6a57f82b9bc0d43d..b35480e1694bf39d6653cb053ce203d757841c70 100644 (file)
@@ -41,6 +41,10 @@ Builder.prototype = {
                 text: 'pretty diff',
                 url: urlStem + '-pretty-diff.html',
             },
+            flaky: {
+                text: 'pretty diff (flaky)',
+                url: urlStem + '-pretty-diff.html',
+            },
             timeout: {
                 text: 'timed out',
             },
@@ -136,11 +140,11 @@ Builder.prototype = {
                 result.tooManyFailures = true;
 
             result.failureCount = layoutTestStep.results[1].reduce(function(sum, outputLine) {
-                var match = /^(\d+) test case/.exec(outputLine);
+                var match = /^(\d+)/.exec(outputLine);
                 if (!match)
                     return sum;
-                // Don't count new tests as failures.
-                if (outputLine.indexOf('were new') >= 0)
+                // Don't count new tests or passes as failures.
+                if (outputLine.contains('were new') || outputLine.contains('new passes'))
                     return sum;
                 return sum + parseInt(match[1], 10);
             }, 0);
index b7982e397a5cd54bad0e4e1d7b25ac3f4b52dabe..48bb3877502c2751372688c90a35f98eb3dde810 100644 (file)
@@ -41,32 +41,60 @@ LayoutTestResultsLoader.prototype = {
 
         var result = { tests: {}, tooManyFailures: false, version: currentCachedDataVersion };
 
+        function cacheParseResultsAndCallCallback(parseResult) {
+            result.tests = parseResult.tests;
+            result.tooManyFailures = parseResult.tooManyFailures;
+
+            PersistentCache.set(cacheKey, result);
+            callback(result.tests, result.tooManyFailures);
+        }
+
+        var self = this;
+        self._fetchAndParseNRWTResults(buildName, cacheParseResultsAndCallCallback, function() {
+            self._fetchAndParseORWTResults(buildName, cacheParseResultsAndCallCallback, function() {
+                // We couldn't fetch results for this build.
+                PersistentCache.set(cacheKey, result);
+                errorCallback(result.tests, result.tooManyFailures);
+            });
+        });
+    },
+
+    _fetchAndParseNRWTResults: function(buildName, successCallback, errorCallback) {
+        getResource(this._builder.resultsDirectoryURL(buildName) + 'full_results.json', function(xhr) {
+            successCallback((new NRWTResultsParser()).parse(xhr.responseText));
+        },
+        function(xhr) {
+            errorCallback();
+        });
+    },
+
+    _fetchAndParseORWTResults: function(buildName, successCallback, errorCallback) {
         var parsedBuildName = this._builder.buildbot.parseBuildName(buildName);
 
         // http://webkit.org/b/62380 was fixed in r89610.
         var resultsHTMLSupportsTooManyFailuresInfo = parsedBuildName.revision >= 89610;
 
+        var result = { tests: {}, tooManyFailures: false };
+
         var self = this;
 
-        function fetchAndParseResultsHTMLAndCallCallback(callback) {
+        function fetchAndParseResultsHTMLAndCallCallback() {
             getResource(self._builder.resultsPageURL(buildName), function(xhr) {
                 var parseResult = (new ORWTResultsParser()).parse(xhr.responseText);
                 result.tests = parseResult.tests;
                 if (resultsHTMLSupportsTooManyFailuresInfo)
                     result.tooManyFailures = parseResult.tooManyFailures;
 
-                PersistentCache.set(cacheKey, result);
-                callback(result.tests, result.tooManyFailures);
+                successCallback(result);
             },
             function(xhr) {
                 // We failed to fetch results.html. run-webkit-tests must have aborted early.
-                PersistentCache.set(cacheKey, result);
-                errorCallback(result.tests, result.tooManyFailures);
+                errorCallback();
             });
         }
 
         if (resultsHTMLSupportsTooManyFailuresInfo) {
-            fetchAndParseResultsHTMLAndCallCallback(callback);
+            fetchAndParseResultsHTMLAndCallCallback();
             return;
         }
 
@@ -75,20 +103,18 @@ LayoutTestResultsLoader.prototype = {
 
             if (failingTestCount < 0) {
                 // The number of failing tests couldn't be determined.
-                PersistentCache.set(cacheKey, result);
-                errorCallback(result.tests, result.tooManyFailures);
+                errorCallback();
                 return;
             }
 
             if (!failingTestCount) {
                 // All tests passed.
-                PersistentCache.set(cacheKey, result);
-                errorCallback(result.tests, result.tooManyFailures);
+                successCallback(result);
                 return;
             }
 
             // Find out which tests failed.
-            fetchAndParseResultsHTMLAndCallCallback(callback);
+            fetchAndParseResultsHTMLAndCallCallback();
         });
     },
 };
diff --git a/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/NRWTResultsParser.js b/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/NRWTResultsParser.js
new file mode 100644 (file)
index 0000000..b02e06c
--- /dev/null
@@ -0,0 +1,99 @@
+/*
+ * Copyright (C) 2011 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
+ * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+ * THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+function NRWTResultsParser() {
+}
+
+NRWTResultsParser.prototype = {
+    parse: function(unexpectedResultsJS) {
+        var data;
+        function ADD_RESULTS(x) {
+            data = x;
+        }
+
+        eval(unexpectedResultsJS);
+        console.assert(data);
+
+        var result = { tests: {}, tooManyFailures: data.interrupted };
+
+        function forEachTest(tree, handler, opt_prefix) {
+            var prefix = opt_prefix || '';
+
+            for (var key in tree) {
+                var newPrefix = prefix ? (prefix + '/' + key) : key;
+                if ('actual' in tree[key]) {
+                    var testObject = tree[key];
+                    testObject.name = newPrefix;
+                    handler(testObject);
+                } else
+                    forEachTest(tree[key], handler, newPrefix);
+            }
+        }
+
+        function isFailureExpected(expected, actual) {
+            if (actual === 'SKIP')
+                return true;
+
+            var expectedArray = expected.split(' ');
+            var actualArray = actual.split(' ');
+            for (var i = 0; i < actualArray.length; i++) {
+                var actualValue = actualArray[i];
+                if (expectedArray.contains(actualValue))
+                    continue;
+                if (expectedArray.contains('FAIL') && ['IMAGE', 'TEXT', 'IMAGE+TEXT'].contains(actualValue))
+                    continue;
+                return false;
+            }
+
+            return true;
+        }
+
+        function convertNRWTResultString(nrwtResult) {
+            const translations = {
+                CRASH: 'crash',
+                'IMAGE+TEXT': 'fail',
+                IMAGE: 'fail',
+                TEXT: 'fail',
+                TIMEOUT: 'timeout',
+            };
+
+            if (nrwtResult in translations)
+                return translations[nrwtResult];
+
+            if (nrwtResult.contains(' '))
+                return 'flaky';
+
+            return 'unknown failure type ' + nrwtResult;
+        }
+
+        forEachTest(data.tests, function(test) {
+            if (isFailureExpected(test.expected, test.actual))
+                return;
+            result.tests[test.name] = { failureType: convertNRWTResultString(test.actual) };
+        });
+
+        return result;
+    },
+};
index f9e165e48f3608fabef123f2a9290bca35cd9951..2c960e6e2683ff22f25cbd9719cba88390345cf4 100644 (file)
@@ -114,6 +114,10 @@ function sorted(array, sortFunction) {
     return newArray;
 }
 
+Array.prototype.contains = function(value) {
+    return this.indexOf(value) >= 0;
+}
+
 Array.prototype.findFirst = function(predicate) {
     for (var i = 0; i < this.length; ++i) {
         if (predicate(this[i]))
@@ -145,3 +149,7 @@ Node.prototype.removeAllChildren = function() {
     while (this.firstChild)
         this.removeChild(this.firstChild);
 }
+
+String.prototype.contains = function(substring) {
+    return this.indexOf(substring) >= 0;
+}
index 070309d3a0e22c0ea85bc5db9b1db156cc13e9fc..12478122d104ede8c93dac0228bd94cdc2b691da 100644 (file)
@@ -33,6 +33,7 @@ THE POSSIBILITY OF SUCH DAMAGE.
     <script src="FlakyLayoutTestDetector.js"></script>
     <script src="LayoutTestHistoryAnalyzer.js"></script>
     <script src="LayoutTestResultsLoader.js"></script>
+    <script src="NRWTResultsParser.js"></script>
     <script src="ORWTResultsParser.js"></script>
     <script src="PersistentCache.js"></script>
     <script src="Trac.js"></script>
index 790665ea92498283657d3c9ef437b0740d9494e8..1c3ba1dc90f0df6ca793f54e755731dd7a996220 100644 (file)
@@ -1,3 +1,44 @@
+2011-07-06  Adam Roben  <aroben@apple.com>
+
+        Teach TestFailures how to load, parse, and interpret NRWT test results
+
+        Fixes <http://webkit.org/b/61877> TestFailures page doesn't show testers that use
+        new-run-webkit-tests
+
+        Reviewed by Adam Barth.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js:
+        (Builder.prototype.failureDiagnosisTextAndURL): Added support for the new 'flaky' failure
+        type. For now we don't account for image-only flakes (but TestFailures doesn't deal with
+        pixel tests at all currently).
+        (Builder.prototype.getNumberOfFailingTests): Relaxed the regex used for parsing the number
+        of failing tests from the buildbot output. Make sure not to count "new passes" (a new
+        category introduced by NRWT) as failures.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js:
+        (LayoutTestResultsLoader.prototype.start): Moved a bunch of code from here to
+        _fetchAndParseORWTResults. This function now attempts to load NRWT results, then falls back
+        to loading ORWT results.
+        (LayoutTestResultsLoader.prototype._fetchAndParseNRWTResults): Added. Tries to load and
+        parse the full_results.json file for the given build.
+        (LayoutTestResultsLoader.prototype._fetchAndParseORWTResults): Added. Code came from start.
+        Fixed a bug along the way where we were sometimes calling the error callback instead of the
+        success callback when all tests passed.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/NRWTResultsParser.js: Added.
+        (NRWTResultsParser): Do-nothing constructor.
+        (NRWTResultsParser.prototype.parse): Uses eval() (eww!) to get the test results out of the
+        JS string, then iterates over all the tests in the results data and builds up a data
+        structure matching what ORWTResultsParser returns.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Utilities.js:
+        (Array.prototype.contains):
+        (String.prototype.contains):
+        Added these simple helper functions.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/index.html: Pull in
+        NRWTResultsParser.js.
+
 2011-07-06  Adam Roben  <aroben@apple.com>
 
         Extract code to parse ORWT's results.html file into its own class