Teach TestFailures how to interpret unfinished test runs
authoraroben@apple.com <aroben@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 16 Aug 2011 16:27:54 +0000 (16:27 +0000)
committeraroben@apple.com <aroben@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 16 Aug 2011 16:27:54 +0000 (16:27 +0000)
Fixes <http://webkit.org/b/66309> TestFailures thinks all tests passed in
http://build.webkit.org/builders/Lion%20Intel%20Debug%20(Tests)/builds/136

Reviewed by David Kilzer.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js:
(Builder.prototype.getNumberOfFailingTests): Look for "isFinished" in the layout-test step
rather than "isStarted" so that we don't count builds for which the test run never finished.
Bumped the cache version to evict old, buggy cached data.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js:
Added a test for the above, and made a somewhat synthetic test more realistic by including
more of the actual JSON data from build.webkit.org.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js:
(LayoutTestResultsLoader.prototype.start): Bumped the cache version to evict old, buggy
cached data.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@93111 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js
Tools/ChangeLog

index a750b3c..82215be 100644 (file)
@@ -100,7 +100,7 @@ Builder.prototype = {
 
     getNumberOfFailingTests: function(buildNumber, callback) {
         var cacheKey = this.name + '_getNumberOfFailingTests_' + buildNumber;
-        const currentCachedDataVersion = 1;
+        const currentCachedDataVersion = 2;
         if (PersistentCache.contains(cacheKey)) {
             var cachedData = PersistentCache.get(cacheKey);
             if (cachedData.version === currentCachedDataVersion) {
@@ -120,8 +120,8 @@ Builder.prototype = {
                 return;
             }
 
-            if (!('isStarted' in layoutTestStep)) {
-                // run-webkit-tests never even ran.
+            if (!('isFinished' in layoutTestStep)) {
+                // run-webkit-tests never even ran, or didn't finish running.
                 PersistentCache.set(cacheKey, result);
                 callback(result.failureCount, result.tooManyFailures);
                 return;
index 44b3c57..0d15e31 100644 (file)
@@ -63,8 +63,9 @@ test("getNumberOfFailingTests shouldn't include leaks", 4, function() {
     const jsonData = {
         steps: [
             {
-                name: 'layout-test',
+                isFinished: true,
                 isStarted: true,
+                name: 'layout-test',
                 results: [
                     2,
                     [
@@ -179,4 +180,28 @@ test("getNumberOfFailingTests treats build step interruptions as errors", 4, fun
     });
 });
 
+test("getNumberOfFailingTests treats unfinished test runs as errors", 4, function() {
+    const jsonData = {
+        steps: [
+            {
+                isStarted: true, 
+                name: "layout-test", 
+                step_number: 5, 
+                text: [
+                    "layout-tests running"
+                ], 
+                times: [
+                    1312989295.518481, 
+                    null
+                ]
+            }, 
+        ],
+    };
+
+    runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailures) {
+        equal(failureCount, -1);
+        equal(tooManyFailures, false);
+    });
+});
+
 })();
index c7ffb20..70f0c51 100644 (file)
@@ -30,7 +30,7 @@ function LayoutTestResultsLoader(builder) {
 LayoutTestResultsLoader.prototype = {
     start: function(buildName, callback, errorCallback) {
         var cacheKey = 'LayoutTestResultsLoader.' + this._builder.name + '.' + buildName;
-        const currentCachedDataVersion = 6;
+        const currentCachedDataVersion = 7;
         if (PersistentCache.contains(cacheKey)) {
             var cachedData = PersistentCache.get(cacheKey);
             if (cachedData.version === currentCachedDataVersion) {
index c9eada4..954f593 100644 (file)
@@ -1,5 +1,27 @@
 2011-08-16  Adam Roben  <aroben@apple.com>
 
+        Teach TestFailures how to interpret unfinished test runs
+
+        Fixes <http://webkit.org/b/66309> TestFailures thinks all tests passed in
+        http://build.webkit.org/builders/Lion%20Intel%20Debug%20(Tests)/builds/136
+
+        Reviewed by David Kilzer.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js:
+        (Builder.prototype.getNumberOfFailingTests): Look for "isFinished" in the layout-test step
+        rather than "isStarted" so that we don't count builds for which the test run never finished.
+        Bumped the cache version to evict old, buggy cached data.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js:
+        Added a test for the above, and made a somewhat synthetic test more realistic by including
+        more of the actual JSON data from build.webkit.org.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js:
+        (LayoutTestResultsLoader.prototype.start): Bumped the cache version to evict old, buggy
+        cached data.
+
+2011-08-16  Adam Roben  <aroben@apple.com>
+
         Make Apple's Windows port fall back to Lion results instead of SnowLeopard
 
         Apple's Windows port now uses Lion-era versions of CoreFoundation, ICU, etc., so in theory