Don't count leaks as test failures on TestFailures's front page
authoraroben@apple.com <aroben@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 11 Jul 2011 19:20:49 +0000 (19:20 +0000)
committeraroben@apple.com <aroben@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 11 Jul 2011 19:20:49 +0000 (19:20 +0000)
As a bonus, this patch adds our first unit test.

Fixes <http://webkit.org/b/64303> REGRESSION (r90489): TestFailures page says far more tests
are failing on the Leaks bot than actually are

Reviewed by Adam Barth.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js:
(Builder.prototype.getNumberOfFailingTests): Exclude lines containing "leak" when summing
failure counts.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js:
Added. Contains a simple test to show that leaks aren't counted as test failures.

* BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html:
Added. This is the test harness.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@90778 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js [new file with mode: 0644]
Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html [new file with mode: 0644]
Tools/ChangeLog

index bbd29210c54365536acd8b6126937a06880e74f1..751b1529a998a6f6cf654ffc09287e7aa208b637 100644 (file)
@@ -142,8 +142,8 @@ Builder.prototype = {
                 var match = /^(\d+)/.exec(outputLine);
                 if (!match)
                     return sum;
-                // Don't count new tests or passes as failures.
-                if (outputLine.contains('were new') || outputLine.contains('new passes'))
+                // Don't count leaks, new tests, or passes as failures.
+                if (outputLine.contains('leak') || outputLine.contains('were new') || outputLine.contains('new passes'))
                     return sum;
                 return sum + parseInt(match[1], 10);
             }, 0);
diff --git a/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js b/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js
new file mode 100644 (file)
index 0000000..8d4ef94
--- /dev/null
@@ -0,0 +1,50 @@
+module("Builder");
+
+test("getNumberOfFailingTests from Leaks bot", 4, function() {
+    var mockBuildbot = {};
+    mockBuildbot.baseURL = 'http://example.com/';
+
+    var builder = new Builder('test builder', mockBuildbot);
+
+    var realGetResource = window.getResource;
+    window.getResource = function(url, successCallback, errorCallback) {
+        var mockXHR = {};
+        mockXHR.responseText = JSON.stringify({
+            steps: [
+                {
+                    name: 'layout-test',
+                    isStarted: true,
+                    results: [
+                        2,
+                        [
+                            "2178 total leaks found!", 
+                            "2 test cases (<1%) had incorrect layout", 
+                            "2 test cases (<1%) crashed",
+                        ],
+                    ],
+                },
+            ],
+        });
+
+        successCallback(mockXHR);
+    };
+
+    // FIXME: It's lame to be modifying singletons like this. We should get rid
+    // of our singleton usage entirely!
+    var realPersistentCache = window.PersistentCache;
+    window.PersistentCache = {
+        contains: function() { return false; },
+        set: function() { },
+        get: function() { },
+    };
+
+    builder.getNumberOfFailingTests(1, function(failureCount, tooManyFailures) {
+        equals(failureCount, 4);
+        equals(tooManyFailures, false);
+    });
+
+    window.getResource = realGetResource;
+    equals(window.getResource, realGetResource);
+    window.PersistentCache = realPersistentCache;
+    equals(window.PersistentCache, realPersistentCache);
+});
diff --git a/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html b/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html
new file mode 100644 (file)
index 0000000..28c934f
--- /dev/null
@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html>
+<head>
+<link rel="stylesheet" href="../../../../../Source/ThirdParty/qunit/qunit/qunit.css">
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.4.2/jquery.min.js"></script> 
+<script src="../../../../../Source/ThirdParty/qunit/qunit/qunit.js"></script>
+</head>
+<body>
+<h1 id="qunit-header">TestFailures JavaScript Unit Tests</h1>
+<h2 id="qunit-banner"></h2>
+<div id="qunit-testrunner-toolbar"></div>
+<h2 id="qunit-userAgent"></h2>
+<ol id="qunit-tests"></ol>
+<script src="Utilities.js"></script>
+
+<script src="Builder.js"></script>
+<script src="Builder_unittests.js"></script>
+</body>
+</html>
index a6ef2f5b50721f55e56cfaa45d8f0c0fc6591538..57282ee60a4a19a542429a831d7806b8337d1c58 100644 (file)
@@ -1,3 +1,24 @@
+2011-07-11  Adam Roben  <aroben@apple.com>
+
+        Don't count leaks as test failures on TestFailures's front page
+
+        As a bonus, this patch adds our first unit test.
+
+        Fixes <http://webkit.org/b/64303> REGRESSION (r90489): TestFailures page says far more tests
+        are failing on the Leaks bot than actually are
+
+        Reviewed by Adam Barth.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js:
+        (Builder.prototype.getNumberOfFailingTests): Exclude lines containing "leak" when summing
+        failure counts.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js:
+        Added. Contains a simple test to show that leaks aren't counted as test failures.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/run-unittests.html:
+        Added. This is the test harness.
+
 2011-06-14  Mike Stegeman  <mrstegeman@gmail.com>
 
         [GTK] Add API to allow setting local storage database path