Web Inspector: TestSuite test cases should have their own timeout to ensure tests...
authordrousso@apple.com <drousso@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 5 Apr 2019 23:39:16 +0000 (23:39 +0000)
committerdrousso@apple.com <drousso@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 5 Apr 2019 23:39:16 +0000 (23:39 +0000)
https://bugs.webkit.org/show_bug.cgi?id=162814
<rdar://problem/28574102>

Reviewed by Brian Burg.

Source/WebInspectorUI:

A 10s timer is started for every test case added to an async suite. The timer is cleared
when the test finishes, but if the timer fires, the test is forcibly ended with an error.

This timer can be configured by setting a `timeout` value when adding the test case. Values
are expected to be in milliseconds. The value `-1` will prevent a timer from being set.

This change also relaxes the expectation that any individual test case failure will stop the
rest of the suite from running. Since timers are set per test case, it is possible to
recover from a "bad" test case to still run the remaining test cases.

NOTE: there may be unexpected behaviour if a test times out, as the timer doesn't actually
stop the execution of the test, so it may still run and log information, which may appear
"out of nowhere" in the middle of other tests.

* UserInterface/Test/TestSuite.js:
(TestSuite.prototype.get passCount):
(AsyncTestSuite.prototype.runTestCases):
(SyncTestSuite.prototype.runTestCases):

LayoutTests:

* inspector/unit-tests/async-test-suite.html:
* inspector/unit-tests/async-test-suite-expected.txt:
* inspector/unit-tests/sync-test-suite.html:
* inspector/unit-tests/sync-test-suite-expected.txt:

* http/tests/inspector/network/set-resource-caching-disabled-disk-cache-expected.txt:
* inspector/canvas/recording-2d.html:
* inspector/canvas/recording-webgl-snapshots.html:
* inspector/canvas/recording-webgl.html:
* inspector/canvas/resources/shaderProgram-utilities.js:
(TestPage.registerInitializer.whenProgramAdded): Added.
(TestPage.registerInitializer.whenProgramRemoved): Added.
(TestPage.registerInitializer.window.initializeTestSuite):
(TestPage.registerInitializer.window.addSimpleTestCase):
(TestPage.registerInitializer.window.addParentCanvasRemovedTestCase):
(TestPage.registerInitializer.awaitProgramAdded): Added.
(TestPage.registerInitializer.awaitProgramRemoved): Added.
* inspector/console/command-line-api-expected.txt:
* inspector/console/heap-snapshot.html:
* inspector/debugger/async-stack-trace-truncate-expected.txt:
* inspector/debugger/pause-for-internal-scripts-expected.txt:
* inspector/formatting/resources/utilities.js:
(TestPage.registerInitializer.window.addFormattingTests):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@243953 268f45cc-cd09-0410-ab3c-d52691b4dbfc

17 files changed:
LayoutTests/ChangeLog
LayoutTests/http/tests/inspector/network/set-resource-caching-disabled-disk-cache-expected.txt
LayoutTests/inspector/canvas/recording-2d.html
LayoutTests/inspector/canvas/recording-webgl-snapshots.html
LayoutTests/inspector/canvas/recording-webgl.html
LayoutTests/inspector/canvas/resources/shaderProgram-utilities.js
LayoutTests/inspector/console/command-line-api-expected.txt
LayoutTests/inspector/console/heapSnapshot.html
LayoutTests/inspector/debugger/async-stack-trace-truncate-expected.txt
LayoutTests/inspector/debugger/pause-for-internal-scripts-expected.txt
LayoutTests/inspector/formatting/resources/utilities.js
LayoutTests/inspector/unit-tests/async-test-suite-expected.txt
LayoutTests/inspector/unit-tests/async-test-suite.html
LayoutTests/inspector/unit-tests/sync-test-suite-expected.txt
LayoutTests/inspector/unit-tests/sync-test-suite.html
Source/WebInspectorUI/ChangeLog
Source/WebInspectorUI/UserInterface/Test/TestSuite.js

index 5cf21bd..e13d677 100644 (file)
@@ -1,3 +1,35 @@
+2019-04-05  Devin Rousso  <drousso@apple.com>
+
+        Web Inspector: TestSuite test cases should have their own timeout to ensure tests fail with output instead of timeout by test runner
+        https://bugs.webkit.org/show_bug.cgi?id=162814
+        <rdar://problem/28574102>
+
+        Reviewed by Brian Burg.
+
+        * inspector/unit-tests/async-test-suite.html:
+        * inspector/unit-tests/async-test-suite-expected.txt:
+        * inspector/unit-tests/sync-test-suite.html:
+        * inspector/unit-tests/sync-test-suite-expected.txt:
+
+        * http/tests/inspector/network/set-resource-caching-disabled-disk-cache-expected.txt:
+        * inspector/canvas/recording-2d.html:
+        * inspector/canvas/recording-webgl-snapshots.html:
+        * inspector/canvas/recording-webgl.html:
+        * inspector/canvas/resources/shaderProgram-utilities.js:
+        (TestPage.registerInitializer.whenProgramAdded): Added.
+        (TestPage.registerInitializer.whenProgramRemoved): Added.
+        (TestPage.registerInitializer.window.initializeTestSuite):
+        (TestPage.registerInitializer.window.addSimpleTestCase):
+        (TestPage.registerInitializer.window.addParentCanvasRemovedTestCase):
+        (TestPage.registerInitializer.awaitProgramAdded): Added.
+        (TestPage.registerInitializer.awaitProgramRemoved): Added.
+        * inspector/console/command-line-api-expected.txt:
+        * inspector/console/heap-snapshot.html:
+        * inspector/debugger/async-stack-trace-truncate-expected.txt:
+        * inspector/debugger/pause-for-internal-scripts-expected.txt:
+        * inspector/formatting/resources/utilities.js:
+        (TestPage.registerInitializer.window.addFormattingTests):
+
 2019-04-05  Ryan Haddad  <ryanhaddad@apple.com>
 
         [Mac WK2 iOS Sim] Layout Test imported/w3c/web-platform-tests/webrtc/RTCRtpReceiver-getSynchronizationSources.https.html is a flaky failure
index 1e87a5c..174f1a1 100644 (file)
@@ -6,8 +6,8 @@ Test for `Network.setResourceCachingDisabled` for disk cache loads.
 -- Running test case: PossibleNetworkLoad
 PASS: Resource should be created.
 PASS: Resource should receive a Response.
--- Running test setup.
 
+-- Running test setup.
 -- Running test case: SetResourceCachingDisabled.DiskCache
 PASS: Resource should be created.
 PASS: Resource should receive a Response.
index df20f0b..a412029 100644 (file)
@@ -444,6 +444,7 @@ function test() {
         test(resolve, reject) {
             startRecording(WI.Canvas.ContextType.Canvas2D, resolve, reject, {frameCount: 1});
         },
+        timeout: -1,
     });
 
     suite.addTestCase({
@@ -452,6 +453,7 @@ function test() {
         test(resolve, reject) {
             startRecording(WI.Canvas.ContextType.Canvas2D, resolve, reject);
         },
+        timeout: -1,
     });
 
     suite.addTestCase({
@@ -460,6 +462,7 @@ function test() {
         test(resolve, reject) {
             startRecording(WI.Canvas.ContextType.Canvas2D, resolve, reject, {memoryLimit: 10});
         },
+        timeout: -1,
     });
 
     suite.addTestCase({
index 6afc61e..b70c7c5 100644 (file)
@@ -102,6 +102,7 @@ function test() {
         test(resolve, reject) {
             startRecording(WI.Canvas.ContextType.WebGL, resolve, reject, {frameCount: 1, checkForContentChange: true});
         },
+        timeout: -1,
     });
 
     suite.runTestCasesAndFinish();
index 7672d76..9e81095 100644 (file)
@@ -526,6 +526,7 @@ function test() {
         test(resolve, reject) {
             startRecording(WI.Canvas.ContextType.WebGL, resolve, reject, {frameCount: 1});
         },
+        timeout: -1,
     });
 
     suite.addTestCase({
@@ -534,6 +535,7 @@ function test() {
         test(resolve, reject) {
             startRecording(WI.Canvas.ContextType.WebGL, resolve, reject);
         },
+        timeout: -1,
     });
 
     suite.addTestCase({
@@ -542,6 +544,7 @@ function test() {
         test(resolve, reject) {
             startRecording(WI.Canvas.ContextType.WebGL, resolve, reject, {memoryLimit: 10});
         },
+        timeout: -1,
     });
 
     suite.addTestCase({
index 74ea3ea..5275c44 100644 (file)
@@ -46,25 +46,23 @@ function deleteContext() {
 TestPage.registerInitializer(() => {
     let suite = null;
 
-    function awaitProgramAdded() {
+    function whenProgramAdded(callback) {
         InspectorTest.assert(WI.canvasManager.canvases.length === 1, "There should only be one canvas.");
-        return WI.canvasManager.canvases[0].shaderProgramCollection.awaitEvent(WI.Collection.Event.ItemAdded)
-        .then((event) => {
+        WI.canvasManager.canvases[0].shaderProgramCollection.singleFireEventListener(WI.Collection.Event.ItemAdded, (event) => {
             let program = event.data.item;
             InspectorTest.expectThat(program instanceof WI.ShaderProgram, "Added ShaderProgram.");
             InspectorTest.expectThat(program.canvas instanceof WI.Canvas, "ShaderProgram should have a parent Canvas.");
-            return program;
+            callback(program);
         });
     }
 
-    function awaitProgramRemoved() {
+    function whenProgramRemoved(callback) {
         InspectorTest.assert(WI.canvasManager.canvases.length === 1, "There should only be one canvas.");
-        return WI.canvasManager.canvases[0].shaderProgramCollection.awaitEvent(WI.Collection.Event.ItemRemoved)
-        .then((event) => {
+        WI.canvasManager.canvases[0].shaderProgramCollection.singleFireEventListener(WI.Collection.Event.ItemRemoved, (event) => {
             let program = event.data.item;
             InspectorTest.expectThat(program instanceof WI.ShaderProgram, "Removed ShaderProgram.");
             InspectorTest.expectThat(program.canvas instanceof WI.Canvas, "ShaderProgram should have a parent Canvas.");
-            return program;
+            callback(program);
         });
     }
 
@@ -77,7 +75,9 @@ TestPage.registerInitializer(() => {
             test(resolve, reject) {
                 // This can't use `awaitEvent` since the promise resolution happens on the next tick.
                 WI.canvasManager.singleFireEventListener(WI.CanvasManager.Event.CanvasAdded, (event) => {
-                    awaitProgramAdded().then(resolve, reject);
+                    whenProgramAdded((program) => {
+                        resolve();
+                    });
                 });
 
                 InspectorTest.reloadPage();
@@ -92,13 +92,11 @@ TestPage.registerInitializer(() => {
             name: `${suite.name}.ShaderProgramAdded`,
             description: "Check that added/removed events are sent.",
             test(resolve, reject) {
-                awaitProgramAdded()
-                .then((addedProgram) => {
-                    awaitProgramRemoved()
-                    .then((removedProgram) => {
+                whenProgramAdded((addedProgram) => {
+                    whenProgramRemoved((removedProgram) => {
                         InspectorTest.expectEqual(removedProgram, addedProgram, "Removed the previously added ShaderProgram.");
-                    })
-                    .then(resolve, reject);
+                        resolve();
+                    });
 
                     InspectorTest.evaluateInPage(`deleteProgram()`);
                 });
@@ -113,16 +111,16 @@ TestPage.registerInitializer(() => {
             name: `${suite.name}.ParentCanvasRemoved`,
             description: "Check that the ShaderProgram is removed before it's parent Canvas.",
             test(resolve, reject) {
-                Promise.race([
-                    awaitProgramRemoved()
-                    .then(() => {
-                        InspectorTest.pass("Removed ShaderProgram before Canvas.");
-                        resolve();
-                    }),
-                    WI.canvasManager.awaitEvent(WI.CanvasManager.Event.CanvasRemoved)
-                    .then(reject)
-                ])
-                .catch(() => { InspectorTest.fail("Removed Canvas before ShaderProgram."); });
+                let canvasRemoved = false;
+
+                WI.canvasManager.singleFireEventListener(WI.CanvasManager.Event.CanvasRemoved, (event) => {
+                    canvasRemoved = true;
+                });
+
+                whenProgramRemoved((program) => {
+                    InspectorTest.expectFalse(canvasRemoved, "Removed ShaderProgram before Canvas.");
+                    resolve();
+                });
 
                 InspectorTest.evaluateInPage(`createProgram("${contextType}")`);
                 InspectorTest.evaluateInPage(`deleteContext()`);
index ed414cd..d3365b5 100644 (file)
@@ -10,13 +10,13 @@ Output: 0,1
 -- Running test case: EvaluateArrayValues
 Input: values([3,4])
 Output: 3,4
--- Running test setup.
 
+-- Running test setup.
 -- Running test case: EvaluateDollarZero
 Input: $0
 Output: [object HTMLParagraphElement]
--- Running test setup.
 
+-- Running test setup.
 -- Running test case: EvaluateInvalidSelector
 Input: $('foo')
 CONSOLE: The console function $() has changed from $=getElementById(id) to $=querySelector(selector). You might try $("#%s")
index 92be69d..c20e606 100644 (file)
@@ -27,7 +27,8 @@ function test()
             });
 
             ProtocolTest.evaluateInPage("triggerHeapSnapshotNoTitle()");
-        }
+        },
+        timeout: -1,
     });
 
     suite.addTestCase({
@@ -42,7 +43,8 @@ function test()
             });
 
             ProtocolTest.evaluateInPage("triggerHeapSnapshotWithTitle()");
-        }
+        },
+        timeout: -1,
     });
 
     suite.runTestCasesAndFinish();
index 771c8b8..032f1b2 100644 (file)
@@ -10,9 +10,9 @@ CALL STACK:
 0: [F] handleAnimationFrame
 PASS: Async stack trace should be null.
 -- Running test teardown.
+
 -- Running test setup.
 Set maximum stack trace depth = 10.
-
 -- Running test case: AsyncStackTrace.CheckTruncated
 PAUSED
 CALL STACK:
@@ -31,9 +31,9 @@ ASYNC CALL STACK:
 (remaining call frames truncated)
 PASS: Async stack trace should be truncated.
 -- Running test teardown.
+
 -- Running test setup.
 Set maximum stack trace depth = 10.
-
 -- Running test case: AsyncStackTrace.CheckNotTruncated
 PAUSED
 CALL STACK:
index 92eee8b..53c63e7 100644 (file)
@@ -65,8 +65,8 @@ PAUSE AT entryConsoleLog:12:2
 ACTION: resume
 RESUMED
 PASS: Should have used all steps.
--- Running test setup.
 
+-- Running test setup.
 -- Running test case: Debugger.setPauseForInternalScripts.Enabled
 EXPRESSION: setTimeout(entryConsoleLog)
 STEPS: over, in, in, in, resume
index 644cebc..9b4e326 100644 (file)
@@ -43,7 +43,7 @@ TestPage.registerInitializer(function() {
 
     window.addFormattingTests = function(suite, mode, tests) {
         let testPageURL = WI.networkManager.mainFrame.mainResource.url;
-        let testPageResourcesURL = testPageURL.substring(0, testPageURL.lastIndexOf("/"));            
+        let testPageResourcesURL = testPageURL.substring(0, testPageURL.lastIndexOf("/"));
 
         for (let test of tests) {
             let testName = test.substring(test.lastIndexOf("/") + 1);
@@ -52,7 +52,8 @@ TestPage.registerInitializer(function() {
                 name: suite.name + "." + testName,
                 test(resolve, reject) {
                     runFormattingTest(mode, testName, testURL).then(resolve).catch(reject);
-                }
+                },
+                timeout: -1,
             });
         }
     };
index 8b9beb0..be3f96e 100644 (file)
-PASS: instantiating AsyncTestSuite requires name argument.
-PASS: instantiating AsyncTestSuite requires string name argument.
-PASS: instantiating AsyncTestSuite requires non-whitespace name argument.
-PASS: instantiating AsyncTestSuite requires test harness argument.
-PASS: should not be able to add empty test case.
-PASS: should not be able to add non-object test case.
-PASS: test case should require string name.
-PASS: test case should require non-whitespace name.
-PASS: test case should require test function.
-PASS: should not be able to specify non-Function `setup` parameter.
-PASS: should not be able to specify non-Function `setup` parameter.
-PASS: should not be able to specify non-Function `setup` parameter.
-PASS: should not be able to specify non-Function `teardown` parameter.
-PASS: should not be able to specify non-Function `teardown` parameter.
-PASS: should not be able to specify non-Function `teardown` parameter.
+PASS: Should produce an exception.
+Error: Must pass the test's harness as the first argument.
+PASS: Should produce an exception.
+Error: Must pass the test's harness as the first argument.
+PASS: Should produce an exception.
+Error: Must pass the test's harness as the first argument.
+PASS: Should produce an exception.
+Error: Must pass the test's harness as the first argument.
+PASS: Should produce an exception.
+Error: Tried to add non-object test case.
+PASS: Should produce an exception.
+Error: Tried to add non-object test case.
+PASS: Should produce an exception.
+Error: Tried to add test case without a name.
+PASS: Should produce an exception.
+Error: Tried to add test case without a name.
+PASS: Should produce an exception.
+Error: Tried to add test case without `test` function.
+PASS: Should produce an exception.
+Error: Tried to add test case with invalid `setup` parameter (must be a function).
+PASS: Should produce an exception.
+Error: Tried to add test case with invalid `setup` parameter (must be a function).
+PASS: Should produce an exception.
+Error: Tried to add test case with invalid `setup` parameter (must be a function).
+PASS: Should produce an exception.
+Error: Tried to add test case with invalid `teardown` parameter (must be a function).
+PASS: Should produce an exception.
+Error: Tried to add test case with invalid `teardown` parameter (must be a function).
+PASS: Should produce an exception.
+Error: Tried to add test case with invalid `teardown` parameter (must be a function).
 PASS: should not be able to run empty test suite.
 
 == Running test suite: AsyncTestSuite.RunTwiceSuite
-PASS: should not be able to run a test suite twice.
+PASS: Should produce an exception.
+Error: Tried to call runTestCases() more than once.
 -- Running test case: DummyTest0
+PASS: DummyTest0 should only run once.
 
-== Running test suite: AsyncTestSuite.SequentialExecution
-PASS: AsyncTestSuite.RunTestCases() should return a Promise.
--- Running test case: DummyTest1
--- Running test case: DummyTest2
--- Running test case: DummyTest3
--- Running test case: FailingTest4
-!! EXCEPTION: [object Object]
-Stack Trace: (suppressed)
-PASS: Promise from sequentialExecutionSuite.runTestCases() should reject when a test case fails.
-PASS: Promise from sequentialExecutionSuite.runTestCases() should reject without altering its result value.
-PASS: sequentialExecutionSuite should have executed four tests.
-PASS: sequentialExecutionSuite should have passed three tests.
-PASS: sequentialExecutionSuite should have failed 1 test.
-PASS: sequentialExecutionSuite should have skipped zero tests.
-
-== Running test suite: AsyncTestSuite.AbortOnFailure
--- Running test case: PassingTest5
--- Running test case: FailingTest6
-!! EXCEPTION: {"token":666}
-Stack Trace: (suppressed)
-PASS: Promise from abortOnFailureSuite.runTestCases() should reject when a test case fails.
-PASS: Promise from abortOnFailureSuite.runTestCases() should reject without altering its result value.
-PASS: abortOnFailureSuite should have executed two tests.
-PASS: abortOnFailureSuite should have passed one test.
-PASS: abortOnFailureSuite should have failed one test.
-PASS: abortOnFailureSuite should have skipped one test.
-
-== Running test suite: AsyncTestSuite.SetupAndTeardown
+== Running test suite: AsyncTestSuite.PromiseFunctionSuccess
+-- Running test case: PromiseFunctionSuccess
+
+PASS: Promise from AsyncTestSuite.PromiseFunctionSuccess.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.PromiseFunctionSuccess should have executed 1 tests.
+PASS: AsyncTestSuite.PromiseFunctionSuccess should have passed 1 tests.
+PASS: AsyncTestSuite.PromiseFunctionSuccess should have failed 0 tests.
+PASS: AsyncTestSuite.PromiseFunctionSuccess should have skipped 0 tests.
+
+== Running test suite: PromiseTestSuite.PromiseFunctionException
+-- Running test case: PromiseFunctionException
+Throwing...
+!! EXCEPTION: PromiseFunctionException throw
+Stack Trace: (suppressed)
+
+PASS: Promise from PromiseTestSuite.PromiseFunctionException.runTestCases() should resolve even if a test case fails.
+PASS: PromiseTestSuite.PromiseFunctionException should have executed 1 tests.
+PASS: PromiseTestSuite.PromiseFunctionException should have passed 0 tests.
+PASS: PromiseTestSuite.PromiseFunctionException should have failed 1 tests.
+PASS: PromiseTestSuite.PromiseFunctionException should have skipped 0 tests.
+
+== Running test suite: PromiseTestSuite.PromiseFunctionFailure
+-- Running test case: PromiseFunctionFailure
+Rejecting...
+!! EXCEPTION: PromiseFunctionFailure reject
+Stack Trace: (suppressed)
+
+PASS: Promise from PromiseTestSuite.PromiseFunctionFailure.runTestCases() should resolve even if a test case fails.
+PASS: PromiseTestSuite.PromiseFunctionFailure should have executed 1 tests.
+PASS: PromiseTestSuite.PromiseFunctionFailure should have passed 0 tests.
+PASS: PromiseTestSuite.PromiseFunctionFailure should have failed 1 tests.
+PASS: PromiseTestSuite.PromiseFunctionFailure should have skipped 0 tests.
+
+== Running test suite: AsyncTestSuite.PromiseSequentialExecution
+-- Running test case: 1 (Pass)
+-- Running test case: 2 (Pass)
+-- Running test case: 3 (Pass)
+-- Running test case: 4 (Pass)
+
+PASS: Promise from AsyncTestSuite.PromiseSequentialExecution.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.PromiseSequentialExecution should have executed 4 tests.
+PASS: AsyncTestSuite.PromiseSequentialExecution should have passed 4 tests.
+PASS: AsyncTestSuite.PromiseSequentialExecution should have failed 0 tests.
+PASS: AsyncTestSuite.PromiseSequentialExecution should have skipped 0 tests.
+
+== Running test suite: AsyncTestSuite.PromiseContinueOnFailure
+-- Running test case: 1 (Pass)
+-- Running test case: 2 (Fail)
+Throwing...
+!! EXCEPTION: {"x":"PromiseContinueOnFailure throw"}
+Stack Trace: (suppressed)
+
+-- Running test case: 3 (Pass)
+-- Running test case: 4 (Fail)
+Rejecting...
+!! EXCEPTION: {"x":"PromiseContinueOnFailure reject"}
+Stack Trace: (suppressed)
+
+PASS: Promise from AsyncTestSuite.PromiseContinueOnFailure.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.PromiseContinueOnFailure should have executed 4 tests.
+PASS: AsyncTestSuite.PromiseContinueOnFailure should have passed 2 tests.
+PASS: AsyncTestSuite.PromiseContinueOnFailure should have failed 2 tests.
+PASS: AsyncTestSuite.PromiseContinueOnFailure should have skipped 0 tests.
+
+== Running test suite: AsyncTestSuite.PromiseSetupAndTeardown
 -- Running test setup.
 -- Running test case: TestWithSetupAndTeardown
-PASS: Test should see side effects of running setup() action.
 -- Running test teardown.
-PASS: Teardown should see side effects of running setup() action.
-
 -- Running test case: TestRunningAfterTeardown
-PASS: Test should see side effects of previous test's teardown() action.
-PASS: Promise from setupAndTeardownTestSuite.runTestCases() should resolve.
 
-== Running test suite: AsyncTestSuite.SetupException
+PASS: Promise from AsyncTestSuite.PromiseSetupAndTeardown.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.PromiseSetupAndTeardown should have executed 2 tests.
+PASS: AsyncTestSuite.PromiseSetupAndTeardown should have passed 2 tests.
+PASS: AsyncTestSuite.PromiseSetupAndTeardown should have failed 0 tests.
+PASS: AsyncTestSuite.PromiseSetupAndTeardown should have skipped 0 tests.
+
+== Running test suite: AsyncTestSuite.PromiseSetupException
 -- Running test setup.
-!! EXCEPTION: 
+Throwing...
+!! EXCEPTION: PromiseSetupException throw
 Stack Trace: (suppressed)
-PASS: Promise from setupExceptionTestSuite.runTestCases() should reject.
 
-== Running test suite: AsyncTestSuite.SetupFailure
 -- Running test setup.
-!! EXCEPTION: undefined
+PASS: Setup action should still execute if previous test's setup action threw an exception.
+-- Running test case: TestAfterSetupException
+PASS: Test should still execute if previous test's setup action threw an exception.
+-- Running test teardown.
+PASS: Teardown action should still execute if previous test's setup action threw an exception.
+
+PASS: Promise from AsyncTestSuite.PromiseSetupException.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.PromiseSetupException should have executed 1 tests.
+PASS: AsyncTestSuite.PromiseSetupException should have passed 1 tests.
+PASS: AsyncTestSuite.PromiseSetupException should have failed 1 tests.
+PASS: AsyncTestSuite.PromiseSetupException should have skipped 1 tests.
+
+== Running test suite: AsyncTestSuite.PromiseSetupFailure
+-- Running test setup.
+Rejecting...
+!! EXCEPTION: PromiseSetupFailure reject
 Stack Trace: (suppressed)
-PASS: Promise from setupFailureTestSuite.runTestCases() should reject.
 
-== Running test suite: AsyncTestSuite.TeardownException
+-- Running test setup.
+PASS: Setup action should still execute if previous test's setup action failed.
+-- Running test case: TestAfterSetupException
+PASS: Test should still execute if previous test's setup action failed.
+-- Running test teardown.
+PASS: Teardown action should still execute if previous test's setup action failed.
+
+PASS: Promise from AsyncTestSuite.PromiseSetupFailure.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.PromiseSetupFailure should have executed 1 tests.
+PASS: AsyncTestSuite.PromiseSetupFailure should have passed 1 tests.
+PASS: AsyncTestSuite.PromiseSetupFailure should have failed 1 tests.
+PASS: AsyncTestSuite.PromiseSetupFailure should have skipped 1 tests.
+
+== Running test suite: AsyncTestSuite.PromiseTeardownException
 -- Running test case: TestWithExceptionDuringTeardown
 -- Running test teardown.
-!! EXCEPTION: 
+Throwing...
+!! EXCEPTION: PromiseTeardownException throw
 Stack Trace: (suppressed)
-PASS: Promise from teardownExceptionTestSuite.runTestCases() should reject.
 
-== Running test suite: AsyncTestSuite.TeardownFailure
+-- Running test setup.
+PASS: Setup action should still execute if previous test's teardown action threw an exception.
+-- Running test case: TestAfterTeardownException
+PASS: Test should still execute if previous test's teardown action threw an exception.
+-- Running test teardown.
+PASS: Teardown action should still execute if previous test's teardown action threw an exception.
+
+PASS: Promise from AsyncTestSuite.PromiseTeardownException.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.PromiseTeardownException should have executed 2 tests.
+PASS: AsyncTestSuite.PromiseTeardownException should have passed 1 tests.
+PASS: AsyncTestSuite.PromiseTeardownException should have failed 1 tests.
+PASS: AsyncTestSuite.PromiseTeardownException should have skipped 0 tests.
+
+== Running test suite: AsyncTestSuite.PromiseTeardownFailure
 -- Running test case: TestWithExceptionDuringTeardown
 -- Running test teardown.
-!! EXCEPTION: undefined
+Rejecting...
+!! EXCEPTION: PromiseTeardownFailure reject
 Stack Trace: (suppressed)
-PASS: Promise from teardownFailureTestSuite.runTestCases() should reject.
+
+-- Running test setup.
+PASS: Setup action should still execute if previous test's teardown action failed.
+-- Running test case: TestAfterTeardownException
+PASS: Test should still execute if previous test's teardown action failed.
+-- Running test teardown.
+PASS: Teardown action should still execute if previous test's teardown action failed.
+
+PASS: Promise from AsyncTestSuite.PromiseTeardownFailure.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.PromiseTeardownFailure should have executed 2 tests.
+PASS: AsyncTestSuite.PromiseTeardownFailure should have passed 1 tests.
+PASS: AsyncTestSuite.PromiseTeardownFailure should have failed 1 tests.
+PASS: AsyncTestSuite.PromiseTeardownFailure should have skipped 0 tests.
+
+== Running test suite: AsyncTestSuite.PromiseTimeout
+-- Running test case: PromiseTestWithTimeout
+Timeout...
+!! TIMEOUT: took longer than 5ms
+
+-- Running test setup.
+PASS: Setup action should still execute if previous test timed out.
+-- Running test case: PromiseTestAfterTimeout
+PASS: Test should still execute if previous test timed out.
+-- Running test teardown.
+PASS: Teardown action should still execute if previous test timed out.
+
+PASS: Promise from AsyncTestSuite.PromiseTimeout.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.PromiseTimeout should have executed 2 tests.
+PASS: AsyncTestSuite.PromiseTimeout should have passed 1 tests.
+PASS: AsyncTestSuite.PromiseTimeout should have failed 1 tests.
+PASS: AsyncTestSuite.PromiseTimeout should have skipped 0 tests.
 
 == Running test suite: AsyncTestSuite.AsyncFunctionSuccess
 -- Running test case: AsyncFunctionSuccess
-PASS: Promise from asyncFunctionSuccessTestSuite.runTestCases() should succeed.
-PASS: Promise did evaluate the async test function.
-PASS: Resolved value should be 42.
 
-== Running test suite: AsyncTestSuite.AsyncFunctionExplicitFailure
--- Running test case: AsyncFunctionFailure
-!! EXCEPTION: AsyncFunctionFailure Exception Message
+PASS: Promise from AsyncTestSuite.AsyncFunctionSuccess.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.AsyncFunctionSuccess should have executed 1 tests.
+PASS: AsyncTestSuite.AsyncFunctionSuccess should have passed 1 tests.
+PASS: AsyncTestSuite.AsyncFunctionSuccess should have failed 0 tests.
+PASS: AsyncTestSuite.AsyncFunctionSuccess should have skipped 0 tests.
+
+== Running test suite: AsyncTestSuite.AsyncFunctionExplicitException
+-- Running test case: AsyncFunctionExplicitException
+Throwing...
+!! EXCEPTION: AsyncFunctionExplicitException throw
+Stack Trace: (suppressed)
+
+PASS: Promise from AsyncTestSuite.AsyncFunctionExplicitException.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.AsyncFunctionExplicitException should have executed 1 tests.
+PASS: AsyncTestSuite.AsyncFunctionExplicitException should have passed 0 tests.
+PASS: AsyncTestSuite.AsyncFunctionExplicitException should have failed 1 tests.
+PASS: AsyncTestSuite.AsyncFunctionExplicitException should have skipped 0 tests.
+
+== Running test suite: AsyncTestSuite.AsyncFunctionRuntimeException
+-- Running test case: AsyncFunctionRuntimeException
+Throwing...
+!! EXCEPTION: undefined is not an object (evaluating '({}).x.x')
+Stack Trace: (suppressed)
+
+PASS: Promise from AsyncTestSuite.AsyncFunctionRuntimeException.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.AsyncFunctionRuntimeException should have executed 1 tests.
+PASS: AsyncTestSuite.AsyncFunctionRuntimeException should have passed 0 tests.
+PASS: AsyncTestSuite.AsyncFunctionRuntimeException should have failed 1 tests.
+PASS: AsyncTestSuite.AsyncFunctionRuntimeException should have skipped 0 tests.
+
+== Running test suite: AsyncTestSuite.AsyncFunctionFailure
+-- Running test case: AsyncFunctionException
+Rejecting...
+!! EXCEPTION: AsyncFunctionFailure reject
 Stack Trace: (suppressed)
-PASS: Promise from asyncFunctionExplicitFailureTestSuite.runTestCases() should reject.
-PASS: Promise did evaluate the async test function.
-PASS: Rejected value should be thrown exception.
 
-== Running test suite: AsyncTestSuite.AsyncFunctionRuntimeFailure
--- Running test case: AsyncFunctionFailure
+PASS: Promise from AsyncTestSuite.AsyncFunctionFailure.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.AsyncFunctionFailure should have executed 1 tests.
+PASS: AsyncTestSuite.AsyncFunctionFailure should have passed 0 tests.
+PASS: AsyncTestSuite.AsyncFunctionFailure should have failed 1 tests.
+PASS: AsyncTestSuite.AsyncFunctionFailure should have skipped 0 tests.
+
+== Running test suite: AsyncTestSuite.AsyncSequentialExecution
+-- Running test case: 1 (Pass)
+-- Running test case: 2 (Pass)
+-- Running test case: 3 (Pass)
+-- Running test case: 4 (Pass)
+
+PASS: Promise from AsyncTestSuite.AsyncSequentialExecution.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.AsyncSequentialExecution should have executed 4 tests.
+PASS: AsyncTestSuite.AsyncSequentialExecution should have passed 4 tests.
+PASS: AsyncTestSuite.AsyncSequentialExecution should have failed 0 tests.
+PASS: AsyncTestSuite.AsyncSequentialExecution should have skipped 0 tests.
+
+== Running test suite: AsyncTestSuite.AsyncContinueOnFailure
+-- Running test case: 1 (Pass)
+-- Running test case: 2 (Fail)
+Throwing...
+!! EXCEPTION: {"x":"AsyncContinueOnFailure throw"}
+Stack Trace: (suppressed)
+
+-- Running test case: 3 (Pass)
+-- Running test case: 4 (Fail)
+Throwing...
 !! EXCEPTION: undefined is not an object (evaluating '({}).x.x')
 Stack Trace: (suppressed)
-PASS: Promise from asyncFunctionRuntimeFailureTestSuite.runTestCases() should reject.
-PASS: Promise did evaluate the async test function.
-PASS: Rejected value should be a runtime exception.
 
-== Running test suite: AsyncTestSuite.AsyncSetupAndAsyncTeardown
+-- Running test case: 5 (Pass)
+-- Running test case: 6 (Fail)
+Rejecting...
+!! EXCEPTION: AsyncContinueOnFailure reject
+Stack Trace: (suppressed)
+
+PASS: Promise from AsyncTestSuite.AsyncContinueOnFailure.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.AsyncContinueOnFailure should have executed 6 tests.
+PASS: AsyncTestSuite.AsyncContinueOnFailure should have passed 3 tests.
+PASS: AsyncTestSuite.AsyncContinueOnFailure should have failed 3 tests.
+PASS: AsyncTestSuite.AsyncContinueOnFailure should have skipped 0 tests.
+
+== Running test suite: AsyncTestSuite.AsyncSetupAndTeardown
 -- Running test setup.
--- Running test case: TestWithSetupAndTeardown
-PASS: Test should see side effects of running setup() action.
+-- Running test case: AsyncTestWithSetupAndTeardown
 -- Running test teardown.
-PASS: Teardown should see side effects of running setup() action.
+-- Running test case: AsyncTestRunningAfterTeardown
 
--- Running test case: TestRunningAfterTeardown
-PASS: Test should see side effects of previous test's teardown() action.
-PASS: Promise from asyncSetupAndAsyncTeardownTestSuite.runTestCases() should resolve.
+PASS: Promise from AsyncTestSuite.AsyncSetupAndTeardown.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.AsyncSetupAndTeardown should have executed 2 tests.
+PASS: AsyncTestSuite.AsyncSetupAndTeardown should have passed 2 tests.
+PASS: AsyncTestSuite.AsyncSetupAndTeardown should have failed 0 tests.
+PASS: AsyncTestSuite.AsyncSetupAndTeardown should have skipped 0 tests.
 
-== Running test suite: AsyncTestSuite.AsyncSetupExplicitFailure
--- Running test case: AsyncFunctionFailure
-!! EXCEPTION: AsyncFunctionFailure Exception Message
+== Running test suite: AsyncTestSuite.AsyncSetupExplicitException
+-- Running test setup.
+Throwing...
+!! EXCEPTION: AsyncSetupExplicitException throw
 Stack Trace: (suppressed)
-PASS: Promise from asyncSetupExplicitFailureTestSuite.runTestCases() should reject.
-PASS: Promise did evaluate the async setup function.
-PASS: Rejected value should be thrown exception.
 
-== Running test suite: AsyncTestSuite.AsyncSetupRuntimeFailure
 -- Running test setup.
+PASS: Setup action should still execute if previous test's setup action threw an exception.
+-- Running test case: AsyncTestAfterSetupExplicitException
+PASS: Test should still execute if previous test's setup action threw an exception.
+-- Running test teardown.
+PASS: Teardown action should still execute if previous test's setup action threw an exception.
+
+PASS: Promise from AsyncTestSuite.AsyncSetupExplicitException.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.AsyncSetupExplicitException should have executed 1 tests.
+PASS: AsyncTestSuite.AsyncSetupExplicitException should have passed 1 tests.
+PASS: AsyncTestSuite.AsyncSetupExplicitException should have failed 1 tests.
+PASS: AsyncTestSuite.AsyncSetupExplicitException should have skipped 1 tests.
+
+== Running test suite: AsyncTestSuite.AsyncSetupRuntimeException
+-- Running test setup.
+Throwing...
 !! EXCEPTION: undefined is not an object (evaluating '({}).x.x')
 Stack Trace: (suppressed)
-PASS: Promise from asyncSetupRuntimeFailureTestSuite.runTestCases() should reject.
-PASS: Promise did evaluate the async setup function.
-PASS: Rejected value should be a runtime exception.
 
-== Running test suite: AsyncTestSuite.AsyncTeardownExplicitFailure
--- Running test case: AsyncFunctionFailure
+-- Running test setup.
+PASS: Setup action should still execute if previous test's setup action threw an exception.
+-- Running test case: AsyncTestAfterSetupRuntimeException
+PASS: Test should still execute if previous test's setup action threw an exception.
+-- Running test teardown.
+PASS: Teardown action should still execute if previous test's setup action threw an exception.
+
+PASS: Promise from AsyncTestSuite.AsyncSetupRuntimeException.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.AsyncSetupRuntimeException should have executed 1 tests.
+PASS: AsyncTestSuite.AsyncSetupRuntimeException should have passed 1 tests.
+PASS: AsyncTestSuite.AsyncSetupRuntimeException should have failed 1 tests.
+PASS: AsyncTestSuite.AsyncSetupRuntimeException should have skipped 1 tests.
+
+== Running test suite: AsyncTestSuite.AsyncSetupFailure
+-- Running test setup.
+Rejecting...
+!! EXCEPTION: AsyncSetupFailure reject
+Stack Trace: (suppressed)
+
+-- Running test setup.
+PASS: Setup action should still execute if previous test's setup action failed.
+-- Running test case: AsyncTestAfterSetupFailure
+PASS: Test should still execute if previous test's setup action failed.
+-- Running test teardown.
+PASS: Setup action should still execute if previous test's setup action failed.
+
+PASS: Promise from AsyncTestSuite.AsyncSetupFailure.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.AsyncSetupFailure should have executed 1 tests.
+PASS: AsyncTestSuite.AsyncSetupFailure should have passed 1 tests.
+PASS: AsyncTestSuite.AsyncSetupFailure should have failed 1 tests.
+PASS: AsyncTestSuite.AsyncSetupFailure should have skipped 1 tests.
+
+== Running test suite: AsyncTestSuite.AsyncTeardownExplicitException
+-- Running test case: AsyncTestWithExplicitExceptionDuringTeardown
 -- Running test teardown.
-!! EXCEPTION: AsyncFunctionFailure Exception Message
+Throwing...
+!! EXCEPTION: AsyncTeardownExplicitException throw
 Stack Trace: (suppressed)
-PASS: Promise from asyncTeardownExplicitFailureTestSuite.runTestCases() should reject.
-PASS: Promise did evaluate the async teardown function.
-PASS: Rejected value should be thrown exception.
 
-== Running test suite: AsyncTestSuite.AsyncTeardownRuntimeFailure
--- Running test case: AsyncFunctionFailure
+-- Running test setup.
+PASS: Setup action should still execute if previous test's teardown action threw an exception.
+-- Running test case: AsyncTestAfterTeardownExplicitException
+PASS: Test should still execute if previous test's teardown action threw an exception.
+-- Running test teardown.
+PASS: Teardown action should still execute if previous test's teardown action threw an exception.
+
+PASS: Promise from AsyncTestSuite.AsyncTeardownExplicitException.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.AsyncTeardownExplicitException should have executed 2 tests.
+PASS: AsyncTestSuite.AsyncTeardownExplicitException should have passed 1 tests.
+PASS: AsyncTestSuite.AsyncTeardownExplicitException should have failed 1 tests.
+PASS: AsyncTestSuite.AsyncTeardownExplicitException should have skipped 0 tests.
+
+== Running test suite: AsyncTestSuite.AsyncTeardownRuntimeException
+-- Running test case: AsyncTestWithRuntimeExceptionDuringTeardown
 -- Running test teardown.
+Throwing...
 !! EXCEPTION: undefined is not an object (evaluating '({}).x.x')
 Stack Trace: (suppressed)
-PASS: Promise from asyncTeardownRuntimeFailureTestSuite.runTestCases() should reject.
-PASS: Promise did evaluate the async teardown function.
-PASS: Rejected value should be a runtime exception.
+
+-- Running test setup.
+PASS: Setup action should still execute if previous test's teardown action threw an exception.
+-- Running test case: AsyncTestAfterTeardownRuntimeException
+PASS: Test should still execute if previous test's teardown action threw an exception.
+-- Running test teardown.
+PASS: Teardown action should still execute if previous test's teardown action threw an exception.
+
+PASS: Promise from AsyncTestSuite.AsyncTeardownRuntimeException.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.AsyncTeardownRuntimeException should have executed 2 tests.
+PASS: AsyncTestSuite.AsyncTeardownRuntimeException should have passed 1 tests.
+PASS: AsyncTestSuite.AsyncTeardownRuntimeException should have failed 1 tests.
+PASS: AsyncTestSuite.AsyncTeardownRuntimeException should have skipped 0 tests.
+
+== Running test suite: AsyncTestSuite.AsyncTeardownFailure
+-- Running test case: AsyncTestWithFailureDuringTeardown
+-- Running test teardown.
+Rejecting...
+!! EXCEPTION: AsyncTeardownFailure reject
+Stack Trace: (suppressed)
+
+-- Running test setup.
+PASS: Setup action should still execute if previous test's teardown action failed.
+-- Running test case: AsyncTestAfterTeardownFailure
+PASS: Test should still execute if previous test's teardown action failed.
+-- Running test teardown.
+PASS: Teardown action should still execute if previous test's teardown action failed.
+
+PASS: Promise from AsyncTestSuite.AsyncTeardownFailure.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.AsyncTeardownFailure should have executed 2 tests.
+PASS: AsyncTestSuite.AsyncTeardownFailure should have passed 1 tests.
+PASS: AsyncTestSuite.AsyncTeardownFailure should have failed 1 tests.
+PASS: AsyncTestSuite.AsyncTeardownFailure should have skipped 0 tests.
+
+== Running test suite: AsyncTestSuite.AsyncTimeout
+-- Running test case: AsyncTestWithTimeout
+Timeout...
+!! TIMEOUT: took longer than 5ms
+
+-- Running test setup.
+PASS: Setup action should still execute if previous test timed out.
+-- Running test case: AsyncTestAfterTimeout
+PASS: Test should still execute if previous test timed out.
+-- Running test teardown.
+PASS: Teardown action should still execute if previous test timed out.
+
+PASS: Promise from AsyncTestSuite.AsyncTimeout.runTestCases() should resolve even if a test case fails.
+PASS: AsyncTestSuite.AsyncTimeout should have executed 2 tests.
+PASS: AsyncTestSuite.AsyncTimeout should have passed 1 tests.
+PASS: AsyncTestSuite.AsyncTimeout should have failed 1 tests.
+PASS: AsyncTestSuite.AsyncTimeout should have skipped 0 tests.
 
index 7794d10..5a03db7 100644 (file)
@@ -1,4 +1,4 @@
-<!doctype html>
+<!DOCTYPE html>
 <html>
 <head>
 <script src="../../http/tests/inspector/resources/protocol-test.js"></script>
@@ -7,134 +7,97 @@ function test()
 {
     ProtocolTest.suppressStackTraces = true;
 
-    try {
-        let result = new AsyncTestSuite(this);
-        ProtocolTest.fail("instantiating AsyncTestSuite requires name argument.");
-    } catch (e) {
-        ProtocolTest.pass("instantiating AsyncTestSuite requires name argument.");
-    }
+    ProtocolTest.expectException(() => {
+        new AsyncTestSuite(this);
+    });
 
-    try {
-        let result = new AsyncTestSuite(this, {});
-        ProtocolTest.fail("instantiating AsyncTestSuite requires string name argument.");
-    } catch (e) {
-        ProtocolTest.pass("instantiating AsyncTestSuite requires string name argument.");
-    }
+    ProtocolTest.expectException(() => {
+        new AsyncTestSuite(this, {});
+    });
 
-    try {
-        let result = new AsyncTestSuite(this, "      ");
-        ProtocolTest.fail("instantiating AsyncTestSuite requires non-whitespace name argument.");
-    } catch (e) {
-        ProtocolTest.pass("instantiating AsyncTestSuite requires non-whitespace name argument.");
-    }
+    ProtocolTest.expectException(() => {
+        new AsyncTestSuite(this, "      ");
+    });
 
-    try {
-        let result = new AsyncTestSuite("something", {});
-        ProtocolTest.fail("instantiating AsyncTestSuite requires test harness argument.");
-    } catch (e) {
-        ProtocolTest.pass("instantiating AsyncTestSuite requires test harness argument.");
-    }
+    ProtocolTest.expectException(() => {
+        new AsyncTestSuite("something", {});
+    });
 
     let badArgsSuite = ProtocolTest.createAsyncSuite("dummy");
-    try {
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase();
-        ProtocolTest.fail("should not be able to add empty test case.");
-    } catch (e) {
-        ProtocolTest.pass("should not be able to add empty test case.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase("string");
-        ProtocolTest.fail("should not be able to add non-object test case.");
-    } catch (e) {
-        ProtocolTest.pass("should not be able to add non-object test case.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: {},
-            test() {},
+            test() {
+            },
         });
-        ProtocolTest.fail("test case should require string name.");
-    } catch (e) {
-        ProtocolTest.pass("test case should require string name.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "        ",
-            test() {},
+            test() {
+            },
         });
-        ProtocolTest.fail("test case should require non-whitespace name.");
-    } catch (e) {
-        ProtocolTest.pass("test case should require non-whitespace name.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
             test: null,
         });
-        ProtocolTest.fail("test case should require test function.");
-    } catch (e) {
-        ProtocolTest.pass("test case should require test function.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
-            test() {},
-            setup: "astd"
+            test() {
+            },
+            setup: "astd",
         });
-        ProtocolTest.fail("should not be able to specify non-Function `setup` parameter.");
-    } catch (e) {
-        ProtocolTest.pass("should not be able to specify non-Function `setup` parameter.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
-            test() {},
-            setup: 123
+            test() {
+            },
+            setup: 123,
         });
-        ProtocolTest.fail("should not be able to specify non-Function `setup` parameter.");
-    } catch (e) {
-        ProtocolTest.pass("should not be able to specify non-Function `setup` parameter.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
-            test() {},
-            setup: {}
+            test() {
+            },
+            setup: {},
         });
-        ProtocolTest.fail("should not be able to specify non-Function `setup` parameter.");
-    } catch (e) {
-        ProtocolTest.pass("should not be able to specify non-Function `setup` parameter.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
-            test() {},
-            teardown: "astd"
+            test() {
+            },
+            teardown: "astd",
         });
-        ProtocolTest.fail("should not be able to specify non-Function `teardown` parameter.");
-    } catch (e) {
-        ProtocolTest.pass("should not be able to specify non-Function `teardown` parameter.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
-            test() {},
-            teardown: 123
+            test() {
+            },
+            teardown: 123,
         });
-        ProtocolTest.fail("should not be able to specify non-Function `teardown` parameter.");
-    } catch (e) {
-        ProtocolTest.pass("should not be able to specify non-Function `teardown` parameter.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
-            test() {},
-            teardown: {}
+            test() {
+            },
+            teardown: {},
         });
-        ProtocolTest.fail("should not be able to specify non-Function `teardown` parameter.");
-    } catch (e) {
-        ProtocolTest.pass("should not be able to specify non-Function `teardown` parameter.");
-    }
+    });
 
     let runEmptySuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.RunEmptySuite");
     try {
@@ -144,459 +107,765 @@ function test()
         ProtocolTest.pass("should not be able to run empty test suite.");
     }
 
+    let runTwiceSuiteRunCount = 0;
     let runTwiceSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.RunTwiceSuite");
     runTwiceSuite.addTestCase({
         name: "DummyTest0",
-        description: "Check that a suite can't run more than once.",
-        test(resolve, reject) { resolve(); }
+        test(resolve, reject) {
+            ProtocolTest.expectEqual(++runTwiceSuiteRunCount, 1, "DummyTest0 should only run once.");
+            resolve();
+        },
     });
 
     let result = runTwiceSuite.runTestCases();
-    try {
+    ProtocolTest.expectException(() => {
         // Test cases won't run in this event loop; this call should still throw.
         // Later tests are chained to this suite to avoid nondeterminism.
         runTwiceSuite.runTestCases();
-        ProtocolTest.fail("should not be able to run a test suite twice.");
-    } catch (e) {
-        ProtocolTest.pass("should not be able to run a test suite twice.");
+    });
+
+    function checkResult(suite, expectedCounts) {
+        result = result.then(() => suite.runTestCases());
+
+        let message = `Promise from ${suite.name}.runTestCases() should resolve even if a test case fails.`;
+
+        result = result.then(function resolved() {
+            ProtocolTest.log("");
+            ProtocolTest.pass(message);
+
+            ProtocolTest.expectEqual(expectedCounts.runCount, suite.runCount, `${suite.name} should have executed ${expectedCounts.runCount} tests.`);
+            ProtocolTest.expectEqual(expectedCounts.passCount, suite.passCount, `${suite.name} should have passed ${expectedCounts.passCount} tests.`);
+            ProtocolTest.expectEqual(expectedCounts.failCount, suite.failCount, `${suite.name} should have failed ${expectedCounts.failCount} tests.`);
+            ProtocolTest.expectEqual(expectedCounts.skipCount, suite.skipCount, `${suite.name} should have skipped ${expectedCounts.skipCount} tests.`);
+        }, function rejected(e) {
+            ProtocolTest.log("");
+            ProtocolTest.fail(message);
+
+            ProtocolTest.log(e.message);
+        });
     }
 
-    let rejectToken = {"token": 666};
-    let thrownError = new Error(rejectToken);
+    // Promise test functions.
 
-    let sequentialExecutionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.SequentialExecution");
-    sequentialExecutionSuite.addTestCase({
-        name: "DummyTest1",
-        description: "Check test case execution order.",
-        test(resolve, reject) { resolve(); }
+    let promiseFunctionSuccessSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseFunctionSuccess");
+    promiseFunctionSuccessSuite.addTestCase({
+        name: "PromiseFunctionSuccess",
+        test(resolve, reject) {
+            resolve();
+        },
     });
-    sequentialExecutionSuite.addTestCase({
-        name: "DummyTest2",
-        description: "Check test case execution order.",
-        test(resolve, reject) { resolve(); }
+    checkResult(promiseFunctionSuccessSuite, {
+        runCount: 1,
+        passCount: 1,
+        failCount: 0,
+        skipCount: 0,
     });
-    sequentialExecutionSuite.addTestCase({
-        name: "DummyTest3",
-        description: "Check test case execution order.",
-        test(resolve, reject) { resolve(); }
+
+    let promiseFunctionExceptionSuite = ProtocolTest.createAsyncSuite("PromiseTestSuite.PromiseFunctionException");
+    promiseFunctionExceptionSuite.addTestCase({
+        name: "PromiseFunctionException",
+        test(resolve, reject) {
+            ProtocolTest.log("Throwing...");
+            throw "PromiseFunctionException throw";
+        },
     });
-    sequentialExecutionSuite.addTestCase({
-        name: "FailingTest4",
-        description: "Check that test fails by throwing an Error instance.",
-        test(resolve, reject) { throw thrownError; }
+    checkResult(promiseFunctionExceptionSuite, {
+        runCount: 1,
+        passCount: 0,
+        failCount: 1,
+        skipCount: 0,
     });
 
-    let abortOnFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AbortOnFailure");
-    abortOnFailureSuite.addTestCase({
-        name: "PassingTest5",
-        description: "This test is a dummy.",
-        test(resolve, reject) { resolve(); }
-    });
-    abortOnFailureSuite.addTestCase({
-        name: "FailingTest6",
-        description: "This test should fail by explicitly calling the `reject` callback.",
-        test(resolve, reject) { reject(rejectToken); }
+    let promiseFunctionFailureSuite = ProtocolTest.createAsyncSuite("PromiseTestSuite.PromiseFunctionFailure");
+    promiseFunctionFailureSuite.addTestCase({
+        name: "PromiseFunctionFailure",
+        test(resolve, reject) {
+            ProtocolTest.log("Rejecting...");
+            reject("PromiseFunctionFailure reject");
+        },
     });
-    abortOnFailureSuite.addTestCase({
-        name: "PassingTest7",
-        description: "This test should not executed when the preceding test fails.",
-        test(resolve, reject) { resolve(); }
+    checkResult(promiseFunctionFailureSuite, {
+        runCount: 1,
+        passCount: 0,
+        failCount: 1,
+        skipCount: 0,
     });
 
+    let promiseSequentialExecutionPhase = 0;
+    let promiseSequentialExecutionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseSequentialExecution");
+    promiseSequentialExecutionSuite.addTestCase({
+        name: "1 (Pass)",
+        test(resolve, reject) {
+            ProtocolTest.assert(promiseSequentialExecutionPhase === 0);
+            promiseSequentialExecutionPhase = 1;
+            resolve();
+        },
+    });
+    promiseSequentialExecutionSuite.addTestCase({
+        name: "2 (Pass)",
+        test(resolve, reject) {
+            ProtocolTest.assert(promiseSequentialExecutionPhase === 1);
+            promiseSequentialExecutionPhase = 2;
+            resolve();
+        },
+    });
+    promiseSequentialExecutionSuite.addTestCase({
+        name: "3 (Pass)",
+        test(resolve, reject) {
+            ProtocolTest.assert(promiseSequentialExecutionPhase === 2);
+            promiseSequentialExecutionPhase = 3;
+            resolve();
+        },
+    });
+    promiseSequentialExecutionSuite.addTestCase({
+        name: "4 (Pass)",
+        test(resolve, reject) {
+            ProtocolTest.assert(promiseSequentialExecutionPhase === 3);
+            promiseSequentialExecutionPhase = 4;
+            resolve();
+        },
+    });
+    checkResult(promiseSequentialExecutionSuite, {
+        runCount: 4,
+        passCount: 4,
+        failCount: 0,
+        skipCount: 0,
+    });
     result = result.then(() => {
-        let promise = sequentialExecutionSuite.runTestCases();
-        ProtocolTest.expectThat(result instanceof Promise, "AsyncTestSuite.RunTestCases() should return a Promise.");
-        return promise;
+        ProtocolTest.assert(promiseSequentialExecutionPhase === 4);
     });
-    result = result.then(function resolved() {
-        ProtocolTest.fail("Promise from sequentialExecutionSuite.runTestCases() should reject when a test case fails.");
-        return Promise.resolve(); // Continue this test.
-    }, function rejected(e) {
-        ProtocolTest.pass("Promise from sequentialExecutionSuite.runTestCases() should reject when a test case fails.");
-        ProtocolTest.expectThat(e === thrownError, "Promise from sequentialExecutionSuite.runTestCases() should reject without altering its result value.");
 
-        ProtocolTest.expectThat(sequentialExecutionSuite.runCount === 4, "sequentialExecutionSuite should have executed four tests.");
-        ProtocolTest.expectThat(sequentialExecutionSuite.passCount === 3, "sequentialExecutionSuite should have passed three tests.");
-        ProtocolTest.expectThat(sequentialExecutionSuite.failCount === 1, "sequentialExecutionSuite should have failed 1 test.");
-        ProtocolTest.expectThat(sequentialExecutionSuite.skipCount === 0, "sequentialExecutionSuite should have skipped zero tests.");
-        return Promise.resolve(); // Continue this test.
+    let promiseContinueOnFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseContinueOnFailure");
+    promiseContinueOnFailureSuite.addTestCase({
+        name: "1 (Pass)",
+        test(resolve, reject) {
+            resolve();
+        },
+    });
+    promiseContinueOnFailureSuite.addTestCase({
+        name: "2 (Fail)",
+        test(resolve, reject) {
+            ProtocolTest.log("Throwing...");
+            throw {x: "PromiseContinueOnFailure throw"};
+        },
+    });
+    promiseContinueOnFailureSuite.addTestCase({
+        name: "3 (Pass)",
+        test(resolve, reject) {
+            resolve();
+        },
+    });
+    promiseContinueOnFailureSuite.addTestCase({
+        name: "4 (Fail)",
+        test(resolve, reject) {
+            ProtocolTest.log("Rejecting...");
+            reject({x: "PromiseContinueOnFailure reject"});
+        },
+    });
+    checkResult(promiseContinueOnFailureSuite, {
+        runCount: 4,
+        passCount: 2,
+        failCount: 2,
+        skipCount: 0,
     });
 
-    result = result.then(() => {
-        return abortOnFailureSuite.runTestCases();
-    }).then(function resolved() {
-        ProtocolTest.fail("Promise from abortOnFailureSuite.runTestCases() should reject when a test case fails.");
-        return Promise.resolve(); // Continue this test.
-    }, function rejected(e) {
-        ProtocolTest.pass("Promise from abortOnFailureSuite.runTestCases() should reject when a test case fails.");
-        ProtocolTest.expectThat(e === rejectToken, "Promise from abortOnFailureSuite.runTestCases() should reject without altering its result value.");
-        ProtocolTest.expectThat(abortOnFailureSuite.runCount === 2, "abortOnFailureSuite should have executed two tests.");
-        ProtocolTest.expectThat(abortOnFailureSuite.passCount === 1, "abortOnFailureSuite should have passed one test.");
-        ProtocolTest.expectThat(abortOnFailureSuite.failCount === 1, "abortOnFailureSuite should have failed one test.");
-        ProtocolTest.expectThat(abortOnFailureSuite.skipCount === 1, "abortOnFailureSuite should have skipped one test.");
-
-        return Promise.resolve(); // Continue this test.
-    });
-
-    var setupAndTeardownSymbol = Symbol("async-suite-setup-and-teardown-token");
-    window[setupAndTeardownSymbol] = 0;
-
-    let setupAndTeardownTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.SetupAndTeardown");
-    setupAndTeardownTestSuite.addTestCase({
+    let promiseSetupAndTeardownPhase = 0;
+    let promiseSetupAndTeardownSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseSetupAndTeardown");
+    promiseSetupAndTeardownSuite.addTestCase({
         name: "TestWithSetupAndTeardown",
-        description: "Check execution order for setup and teardown actions.",
-        setup: (resolve, reject) => {
-            window[setupAndTeardownSymbol] = 1;
+        setup(resolve, reject) {
+            ProtocolTest.assert(promiseSetupAndTeardownPhase === 0);
+            promiseSetupAndTeardownPhase = 1;
             resolve();
         },
         test(resolve, reject) {
-            ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 1, "Test should see side effects of running setup() action.");
-            window[setupAndTeardownSymbol] = 2;
+            ProtocolTest.assert(promiseSetupAndTeardownPhase === 1);
+            promiseSetupAndTeardownPhase = 2;
             resolve();
         },
-        teardown: (resolve, reject) => {
-            ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 2, "Teardown should see side effects of running setup() action.");
-            window[setupAndTeardownSymbol] = 3;
+        teardown(resolve, reject) {
+            ProtocolTest.assert(promiseSetupAndTeardownPhase === 2);
+            promiseSetupAndTeardownPhase = 3;
             resolve();
-        }
+        },
     });
-    setupAndTeardownTestSuite.addTestCase({
+    promiseSetupAndTeardownSuite.addTestCase({
         name: "TestRunningAfterTeardown",
-        description: "Check execution order for test after a teardown action.",
         test(resolve, reject) {
-            ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 3, "Test should see side effects of previous test's teardown() action.");
+            ProtocolTest.assert(promiseSetupAndTeardownPhase === 3);
             resolve();
         },
     });
-
-    result = result.then(() => {
-        return setupAndTeardownTestSuite.runTestCases();
-    }).then(function resolved() {
-        ProtocolTest.pass("Promise from setupAndTeardownTestSuite.runTestCases() should resolve.");
-        return Promise.resolve(); // Continue this test.
-    }, function rejected(e) {
-        ProtocolTest.fail("Promise from setupAndTeardownTestSuite.runTestCases() should resolve.");
-        return Promise.resolve(); // Continue this test.
+    checkResult(promiseSetupAndTeardownSuite, {
+        runCount: 2,
+        passCount: 2,
+        failCount: 0,
+        skipCount: 0,
     });
 
-    let setupExceptionTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.SetupException");
-    setupExceptionTestSuite.addTestCase({
+    let promiseSetupExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseSetupException");
+    promiseSetupExceptionSuite.addTestCase({
         name: "TestWithExceptionDuringSetup",
-        description: "Check execution order for setup action that throws an exception.",
-        setup: (resolve, reject) => { throw new Error() },
+        setup(resolve, reject) {
+            ProtocolTest.log("Throwing...");
+            throw "PromiseSetupException throw";
+        },
         test(resolve, reject) {
-            ProtocolTest.assert(false, "Test should not execute if its setup action threw an exception.");
+            ProtocolTest.fail("Test should not execute if its setup action threw an exception.");
             reject();
         },
-        teardown: (resolve, reject) => {
-            ProtocolTest.assert(false, "Teardown action should not execute if its setup action threw an exception.");
+        teardown(resolve, reject) {
+            ProtocolTest.fail("Teardown action should not execute if its setup action threw an exception.");
             reject();
-        }
+        },
     });
-
-    result = result.then(() => {
-        return setupExceptionTestSuite.runTestCases();
-    }).then(function resolved() {
-        ProtocolTest.fail("Promise from setupExceptionTestSuite.runTestCases() should reject.");
-        return Promise.resolve(); // Continue this test.
-    }, function rejected(e) {
-        ProtocolTest.pass("Promise from setupExceptionTestSuite.runTestCases() should reject.");
-        return Promise.resolve(); // Continue this test.
+    promiseSetupExceptionSuite.addTestCase({
+        name: "TestAfterSetupException",
+        setup(resolve, reject) {
+            ProtocolTest.pass("Setup action should still execute if previous test's setup action threw an exception.");
+            resolve();
+        },
+        test(resolve, reject) {
+            ProtocolTest.pass("Test should still execute if previous test's setup action threw an exception.");
+            resolve();
+        },
+        teardown(resolve, reject) {
+            ProtocolTest.pass("Teardown action should still execute if previous test's setup action threw an exception.");
+            resolve();
+        },
+    });
+    checkResult(promiseSetupExceptionSuite, {
+        runCount: 1,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 1,
     });
 
-    let setupFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.SetupFailure");
-    setupFailureTestSuite.addTestCase({
+    let promiseSetupFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseSetupFailure");
+    promiseSetupFailureSuite.addTestCase({
         name: "TestWithFailureDuringSetup",
-        description: "Check execution order for setup action that has failed.",
-        setup: (resolve, reject) => { reject(); },
+        setup(resolve, reject) {
+            ProtocolTest.log("Rejecting...");
+            reject("PromiseSetupFailure reject");
+        },
         test(resolve, reject) {
-            ProtocolTest.assert(false, "Test should not execute if its setup action failed.")
+            ProtocolTest.fail("Test should not execute if its setup action failed.");
             reject();
         },
-        teardown: (resolve, reject) => {
-            ProtocolTest.assert(false, "Teardown action should not execute if its setup action failed.")
+        teardown(resolve, reject) {
+            ProtocolTest.fail("Teardown action should not execute if its setup action failed.");
             reject();
-        }
+        },
     });
-
-    result = result.then(() => {
-        return setupFailureTestSuite.runTestCases();
-    }).then(function resolved() {
-        ProtocolTest.fail("Promise from setupFailureTestSuite.runTestCases() should reject.");
-        return Promise.resolve(); // Continue this test.
-    }, function rejected(e) {
-        ProtocolTest.pass("Promise from setupFailureTestSuite.runTestCases() should reject.");
-        return Promise.resolve(); // Continue this test.
+    promiseSetupFailureSuite.addTestCase({
+        name: "TestAfterSetupException",
+        setup(resolve, reject) {
+            ProtocolTest.pass("Setup action should still execute if previous test's setup action failed.");
+            resolve();
+        },
+        test(resolve, reject) {
+            ProtocolTest.pass("Test should still execute if previous test's setup action failed.");
+            resolve();
+        },
+        teardown(resolve, reject) {
+            ProtocolTest.pass("Teardown action should still execute if previous test's setup action failed.");
+            resolve();
+        },
+    });
+    checkResult(promiseSetupFailureSuite, {
+        runCount: 1,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 1,
     });
 
-    let teardownExceptionTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.TeardownException");
-    teardownExceptionTestSuite.addTestCase({
+    let promiseTeardownExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseTeardownException");
+    promiseTeardownExceptionSuite.addTestCase({
         name: "TestWithExceptionDuringTeardown",
-        description: "Check execution order for teardown action that throws an exception.",
-        test(resolve, reject) { resolve(); },
-        teardown: (resolve, reject) => { throw new Error() }
+        test(resolve, reject) {
+            resolve();
+        },
+        teardown(resolve, reject) {
+            ProtocolTest.log("Throwing...");
+            throw "PromiseTeardownException throw";
+        },
     });
-    teardownExceptionTestSuite.addTestCase({
+    promiseTeardownExceptionSuite.addTestCase({
         name: "TestAfterTeardownException",
-        descrption: "Check execution order for test after previous test's teardown throws an exception.",
-        setup: (resolve, reject) => {
-            ProtocolTest.assert(false, "Setup action should not execute if previous test's teardown action threw an exception.");
-            reject();
+        setup(resolve, reject) {
+            ProtocolTest.pass("Setup action should still execute if previous test's teardown action threw an exception.");
+            resolve();
         },
         test(resolve, reject) {
-            ProtocolTest.assert(false, "Test should not execute if previous test's teardown action threw an exception.");
-            reject();
-        }
+            ProtocolTest.pass("Test should still execute if previous test's teardown action threw an exception.");
+            resolve();
+        },
+        teardown(resolve, reject) {
+            ProtocolTest.pass("Teardown action should still execute if previous test's teardown action threw an exception.");
+            resolve();
+        },
     });
-
-    result = result.then(() => {
-        return teardownExceptionTestSuite.runTestCases();
-    }).then(function resolved() {
-        ProtocolTest.fail("Promise from teardownExceptionTestSuite.runTestCases() should reject.");
-        return Promise.resolve(); // Continue this test.
-    }, function rejected(e) {
-        ProtocolTest.pass("Promise from teardownExceptionTestSuite.runTestCases() should reject.");
-        return Promise.resolve(); // Continue this test.
+    checkResult(promiseTeardownExceptionSuite, {
+        runCount: 2,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 0,
     });
 
-    let teardownFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.TeardownFailure");
-    teardownFailureTestSuite.addTestCase({
+    let promiseTeardownFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseTeardownFailure");
+    promiseTeardownFailureSuite.addTestCase({
         name: "TestWithExceptionDuringTeardown",
-        description: "Check execution order for teardown action that has failed.",
-        test(resolve, reject) { resolve(); },
-        teardown: (resolve, reject) => { reject(); },
+        test(resolve, reject) {
+            resolve();
+        },
+        teardown(resolve, reject) {
+            ProtocolTest.log("Rejecting...");
+            reject("PromiseTeardownFailure reject");
+        },
     });
-    teardownFailureTestSuite.addTestCase({
+    promiseTeardownFailureSuite.addTestCase({
         name: "TestAfterTeardownException",
-        description: "Check execution order for test after previous test's teardown throws an exception",
-        setup: (resolve, reject) => {
-            ProtocolTest.assert(false, "Setup action should not execute if previous test's teardown action failed.");
-            reject();
+        setup(resolve, reject) {
+            ProtocolTest.pass("Setup action should still execute if previous test's teardown action failed.");
+            resolve();
         },
         test(resolve, reject) {
-            ProtocolTest.assert(false, "Test should not execute if previous test's teardown action failed.");
-            reject();
-        }
+            ProtocolTest.pass("Test should still execute if previous test's teardown action failed.");
+            resolve();
+        },
+        teardown(resolve, reject) {
+            ProtocolTest.pass("Teardown action should still execute if previous test's teardown action failed.");
+            resolve();
+        },
+    });
+    checkResult(promiseTeardownFailureSuite, {
+        runCount: 2,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 0,
     });
 
-    result = result.then(() => {
-        return teardownFailureTestSuite.runTestCases();
-    }).then(function resolved() {
-        ProtocolTest.fail("Promise from teardownFailureTestSuite.runTestCases() should reject.");
-        return Promise.resolve(); // Continue this test.
-    }, function rejected(e) {
-        ProtocolTest.pass("Promise from teardownFailureTestSuite.runTestCases() should reject.");
-        return Promise.resolve(); // Continue this test.
+    let promiseTimeoutSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseTimeout");
+    promiseTimeoutSuite.addTestCase({
+        name: "PromiseTestWithTimeout",
+        test(resolve, reject) {
+            ProtocolTest.log("Timeout...");
+            setTimeout(() => {
+                resolve();
+            }, 50);
+        },
+        timeout: 5,
+    });
+    promiseTimeoutSuite.addTestCase({
+        name: "PromiseTestAfterTimeout",
+        setup(resolve, reject) {
+            ProtocolTest.pass("Setup action should still execute if previous test timed out.");
+            resolve();
+        },
+        test(resolve, reject) {
+            ProtocolTest.pass("Test should still execute if previous test timed out.");
+            resolve();
+        },
+        teardown(resolve, reject) {
+            ProtocolTest.pass("Teardown action should still execute if previous test timed out.");
+            resolve();
+        },
+    });
+    checkResult(promiseTimeoutSuite, {
+        runCount: 2,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 0,
     });
 
     // Async test functions.
-    let asyncFunctionSuccessTestSuiteDidEvaluate = false;
-    let asyncFunctionSuccessTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionSuccess");
-    asyncFunctionSuccessTestSuite.addTestCase({
+
+    let asyncFunctionSuccessSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionSuccess");
+    asyncFunctionSuccessSuite.addTestCase({
         name: "AsyncFunctionSuccess",
-        description: "Check that an async suite with async test functions can succeed",
         async test() {
-            asyncFunctionSuccessTestSuiteDidEvaluate = true;
-            return 42;
-        }
+        },
+    });
+    checkResult(asyncFunctionSuccessSuite, {
+        runCount: 1,
+        passCount: 1,
+        failCount: 0,
+        skipCount: 0,
     });
 
-    result = result.then(() => {
-        return asyncFunctionSuccessTestSuite.runTestCases();
-    }).then(function resolve(x) {
-        ProtocolTest.pass("Promise from asyncFunctionSuccessTestSuite.runTestCases() should succeed.");
-        ProtocolTest.expectThat(asyncFunctionSuccessTestSuiteDidEvaluate, "Promise did evaluate the async test function.");
-        ProtocolTest.expectEqual(x, 42, "Resolved value should be 42.");
-        return Promise.resolve(); // Continue this test.
-    }, function rejected(e) {
-        ProtocolTest.fail("Promise from asyncFunctionSuccessTestSuite.runTestCases() should succeed.");
-        return Promise.resolve(); // Continue this test.
-    });
-
-    let asyncFunctionExplicitFailureTestSuiteDidEvaluate = false;
-    let asyncFunctionExplicitFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionExplicitFailure");
-    asyncFunctionExplicitFailureTestSuite.addTestCase({
-        name: "AsyncFunctionFailure",
-        description: "Check that an async suite with async test functions that throws will reject",
+    let asyncFunctionExplicitExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionExplicitException");
+    asyncFunctionExplicitExceptionSuite.addTestCase({
+        name: "AsyncFunctionExplicitException",
         async test() {
-            asyncFunctionExplicitFailureTestSuiteDidEvaluate = true;
-            throw "AsyncFunctionFailure Exception Message";
-        }
+            ProtocolTest.log("Throwing...");
+            throw "AsyncFunctionExplicitException throw";
+        },
+    });
+    checkResult(asyncFunctionExplicitExceptionSuite, {
+        runCount: 1,
+        passCount: 0,
+        failCount: 1,
+        skipCount: 0,
     });
 
-    result = result.then(() => {
-        return asyncFunctionExplicitFailureTestSuite.runTestCases();
-    }).then(function resolve() {
-        ProtocolTest.fail("Promise from asyncFunctionExplicitFailureTestSuite.runTestCases() should reject.");
-        return Promise.resolve(); // Continue this test.
-    }, function rejected(e) {
-        ProtocolTest.pass("Promise from asyncFunctionExplicitFailureTestSuite.runTestCases() should reject.");
-        ProtocolTest.expectThat(asyncFunctionExplicitFailureTestSuiteDidEvaluate, "Promise did evaluate the async test function.");
-        ProtocolTest.expectEqual(e, "AsyncFunctionFailure Exception Message", "Rejected value should be thrown exception.");
-        return Promise.resolve(); // Continue this test.
-    });
-
-    let asyncFunctionRuntimeFailureTestSuiteDidEvaluate = false;
-    let asyncFunctionRuntimeFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionRuntimeFailure");
-    asyncFunctionRuntimeFailureTestSuite.addTestCase({
-        name: "AsyncFunctionFailure",
-        description: "Check that an async suite with async test functions that throws with a runtime error will reject",
+    let asyncFunctionRuntimeExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionRuntimeException");
+    asyncFunctionRuntimeExceptionSuite.addTestCase({
+        name: "AsyncFunctionRuntimeException",
         async test() {
-            asyncFunctionRuntimeFailureTestSuiteDidEvaluate = true;
+            ProtocolTest.log("Throwing...");
             ({}).x.x.x;
-        }
+        },
+    });
+    checkResult(asyncFunctionRuntimeExceptionSuite, {
+        runCount: 1,
+        passCount: 0,
+        failCount: 1,
+        skipCount: 0,
+    });
+
+    let asyncFunctionFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionFailure");
+    asyncFunctionFailureSuite.addTestCase({
+        name: "AsyncFunctionException",
+        async test() {
+            ProtocolTest.log("Rejecting...");
+            return Promise.reject("AsyncFunctionFailure reject");
+        },
+    });
+    checkResult(asyncFunctionFailureSuite, {
+        runCount: 1,
+        passCount: 0,
+        failCount: 1,
+        skipCount: 0,
     });
 
+    let asyncSequentialExecutionPhase = 0;
+    let asyncSequentialExecutionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSequentialExecution");
+    asyncSequentialExecutionSuite.addTestCase({
+        name: "1 (Pass)",
+        async test() {
+            ProtocolTest.assert(asyncSequentialExecutionPhase === 0);
+            asyncSequentialExecutionPhase = 1;
+        },
+    });
+    asyncSequentialExecutionSuite.addTestCase({
+        name: "2 (Pass)",
+        async test() {
+            ProtocolTest.assert(asyncSequentialExecutionPhase === 1);
+            asyncSequentialExecutionPhase = 2;
+        },
+    });
+    asyncSequentialExecutionSuite.addTestCase({
+        name: "3 (Pass)",
+        async test() {
+            ProtocolTest.assert(asyncSequentialExecutionPhase === 2);
+            asyncSequentialExecutionPhase = 3;
+        },
+    });
+    asyncSequentialExecutionSuite.addTestCase({
+        name: "4 (Pass)",
+        async test() {
+            ProtocolTest.assert(asyncSequentialExecutionPhase === 3);
+            asyncSequentialExecutionPhase = 4;
+        },
+    });
+    checkResult(asyncSequentialExecutionSuite, {
+        runCount: 4,
+        passCount: 4,
+        failCount: 0,
+        skipCount: 0,
+    });
     result = result.then(() => {
-        return asyncFunctionRuntimeFailureTestSuite.runTestCases();
-    }).then(function resolve() {
-        ProtocolTest.fail("Promise from asyncFunctionRuntimeFailureTestSuite.runTestCases() should reject.");
-        return Promise.resolve(); // Continue this test.
-    }, function rejected(e) {
-        ProtocolTest.pass("Promise from asyncFunctionRuntimeFailureTestSuite.runTestCases() should reject.");
-        ProtocolTest.expectThat(asyncFunctionRuntimeFailureTestSuiteDidEvaluate, "Promise did evaluate the async test function.");
-        ProtocolTest.expectThat(e instanceof TypeError, "Rejected value should be a runtime exception.");
-        return Promise.resolve(); // Continue this test.
-    });
-
-    // Async setup() and teardown() success test cases.
-    const asyncSetupAndTeardownSymbol = Symbol("async-suite-async-setup-and-teardown-token");
-    window[asyncSetupAndTeardownSymbol] = 0;
-
-    let asyncSetupAndAsyncTeardownTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupAndAsyncTeardown");
-    asyncSetupAndAsyncTeardownTestSuite.addTestCase({
-        name: "TestWithSetupAndTeardown",
-        description: "Check execution order for setup and teardown actions.",
+        ProtocolTest.assert(asyncSequentialExecutionPhase === 4);
+    });
+
+    let asyncContinueOnFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncContinueOnFailure");
+    asyncContinueOnFailureSuite.addTestCase({
+        name: "1 (Pass)",
+        async test() {
+        },
+    });
+    asyncContinueOnFailureSuite.addTestCase({
+        name: "2 (Fail)",
+        async test() {
+            ProtocolTest.log("Throwing...");
+            throw {x: "AsyncContinueOnFailure throw"};
+        },
+    });
+    asyncContinueOnFailureSuite.addTestCase({
+        name: "3 (Pass)",
+        async test() {
+        },
+    });
+    asyncContinueOnFailureSuite.addTestCase({
+        name: "4 (Fail)",
+        async test() {
+            ProtocolTest.log("Throwing...");
+            ({}).x.x.x;
+        },
+    });
+    asyncContinueOnFailureSuite.addTestCase({
+        name: "5 (Pass)",
+        async test() {
+        },
+    });
+    asyncContinueOnFailureSuite.addTestCase({
+        name: "6 (Fail)",
+        async test() {
+            ProtocolTest.log("Rejecting...");
+            return Promise.reject("AsyncContinueOnFailure reject");
+        },
+    });
+    checkResult(asyncContinueOnFailureSuite, {
+        runCount: 6,
+        passCount: 3,
+        failCount: 3,
+        skipCount: 0,
+    });
+
+    let asyncSetupAndTeardownPhase = 0;
+    let asyncSetupAndTeardownSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupAndTeardown");
+    asyncSetupAndTeardownSuite.addTestCase({
+        name: "AsyncTestWithSetupAndTeardown",
         async setup() {
-            window[asyncSetupAndTeardownSymbol] = 1;
+            ProtocolTest.assert(asyncSetupAndTeardownPhase === 0);
+            asyncSetupAndTeardownPhase = 1;
         },
         async test() {
-            ProtocolTest.expectThat(window[asyncSetupAndTeardownSymbol] === 1, "Test should see side effects of running setup() action.");
-            window[asyncSetupAndTeardownSymbol] = 2;
+            ProtocolTest.assert(asyncSetupAndTeardownPhase === 1);
+            asyncSetupAndTeardownPhase = 2;
         },
         async teardown() {
-            ProtocolTest.expectThat(window[asyncSetupAndTeardownSymbol] === 2, "Teardown should see side effects of running setup() action.");
-            window[asyncSetupAndTeardownSymbol] = 3;
-        }
+            ProtocolTest.assert(asyncSetupAndTeardownPhase === 2);
+            asyncSetupAndTeardownPhase = 3;
+        },
     });
-    asyncSetupAndAsyncTeardownTestSuite.addTestCase({
-        name: "TestRunningAfterTeardown",
-        description: "Check execution order for test after a teardown action.",
-        test(resolve, reject) {
-            ProtocolTest.expectThat(window[asyncSetupAndTeardownSymbol] === 3, "Test should see side effects of previous test's teardown() action.");
-            resolve();
+    asyncSetupAndTeardownSuite.addTestCase({
+        name: "AsyncTestRunningAfterTeardown",
+        async test() {
+            ProtocolTest.assert(asyncSetupAndTeardownPhase === 3);
         },
     });
+    checkResult(asyncSetupAndTeardownSuite, {
+        runCount: 2,
+        passCount: 2,
+        failCount: 0,
+        skipCount: 0,
+    });
 
-    result = result.then(() => {
-        return asyncSetupAndAsyncTeardownTestSuite.runTestCases();
-    }).then(function resolved() {
-        ProtocolTest.pass("Promise from asyncSetupAndAsyncTeardownTestSuite.runTestCases() should resolve.");
-        return Promise.resolve(); // Continue this test.
-    }, function rejected(e) {
-        ProtocolTest.fail("Promise from asyncSetupAndAsyncTeardownTestSuite.runTestCases() should resolve.");
-        return Promise.resolve(); // Continue this test.
-    });
-
-    // Async setup() failure test cases.
-    let asyncSetupExplicitFailureTestSuiteDidEvaluate = false;
-    let asyncSetupExplicitFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupExplicitFailure");
-    asyncSetupExplicitFailureTestSuite.addTestCase({
-        name: "AsyncFunctionFailure",
-        description: "Check that an async suite with async test functions that throws will reject",
+    let asyncSetupExplicitExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupExplicitException");
+    asyncSetupExplicitExceptionSuite.addTestCase({
+        name: "AsyncTestWithExplicitExceptionDuringSetup",
+        async setup() {
+            ProtocolTest.log("Throwing...");
+            throw "AsyncSetupExplicitException throw";
+        },
+        async test() {
+            ProtocolTest.fail("Test should not execute if its setup action threw an exception.");
+        },
+        async teardown() {
+            ProtocolTest.fail("Teardown action should not execute if its setup action threw an exception.");
+        },
+    });
+    asyncSetupExplicitExceptionSuite.addTestCase({
+        name: "AsyncTestAfterSetupExplicitException",
+        async setup() {
+            ProtocolTest.pass("Setup action should still execute if previous test's setup action threw an exception.");
+        },
         async test() {
-            asyncSetupExplicitFailureTestSuiteDidEvaluate = true;
-            throw "AsyncFunctionFailure Exception Message";
-        }
+            ProtocolTest.pass("Test should still execute if previous test's setup action threw an exception.");
+        },
+        async teardown() {
+            ProtocolTest.pass("Teardown action should still execute if previous test's setup action threw an exception.");
+        },
+    });
+    checkResult(asyncSetupExplicitExceptionSuite, {
+        runCount: 1,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 1,
     });
 
-    result = result.then(() => {
-        return asyncSetupExplicitFailureTestSuite.runTestCases();
-    }).then(function resolve() {
-        ProtocolTest.fail("Promise from asyncSetupExplicitFailureTestSuite.runTestCases() should reject.");
-        return Promise.resolve(); // Continue this test.
-    }, function rejected(e) {
-        ProtocolTest.pass("Promise from asyncSetupExplicitFailureTestSuite.runTestCases() should reject.");
-        ProtocolTest.expectThat(asyncSetupExplicitFailureTestSuiteDidEvaluate, "Promise did evaluate the async setup function.");
-        ProtocolTest.expectEqual(e, "AsyncFunctionFailure Exception Message", "Rejected value should be thrown exception.");
-        return Promise.resolve(); // Continue this test.
-    });
-
-    let asyncSetupRuntimeFailureTestSuiteDidEvaluate = false;
-    let asyncSetupRuntimeFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupRuntimeFailure");
-    asyncSetupRuntimeFailureTestSuite.addTestCase({
-        name: "AsyncFunctionFailure",
-        description: "Check that an async suite with an async setup function that throws with a runtime error will reject",
+    let asyncSetupRuntimeExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupRuntimeException");
+    asyncSetupRuntimeExceptionSuite.addTestCase({
+        name: "AsyncTestWithRuntimeExceptionDuringSetup",
         async setup() {
-            asyncSetupRuntimeFailureTestSuiteDidEvaluate = true;
+            ProtocolTest.log("Throwing...");
             ({}).x.x.x;
         },
-        async test() { return true; },
+        async test() {
+            ProtocolTest.fail("Test should not execute if its setup action threw an exception.");
+        },
+        async teardown() {
+            ProtocolTest.fail("Teardown action should not execute if its setup action threw an exception.");
+        },
+    });
+    asyncSetupRuntimeExceptionSuite.addTestCase({
+        name: "AsyncTestAfterSetupRuntimeException",
+        async setup() {
+            ProtocolTest.pass("Setup action should still execute if previous test's setup action threw an exception.");
+        },
+        async test() {
+            ProtocolTest.pass("Test should still execute if previous test's setup action threw an exception.");
+        },
+        async teardown() {
+            ProtocolTest.pass("Teardown action should still execute if previous test's setup action threw an exception.");
+        },
+    });
+    checkResult(asyncSetupRuntimeExceptionSuite, {
+        runCount: 1,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 1,
     });
 
-    result = result.then(() => {
-        return asyncSetupRuntimeFailureTestSuite.runTestCases();
-    }).then(function resolve() {
-        ProtocolTest.fail("Promise from asyncSetupRuntimeFailureTestSuite.runTestCases() should reject.");
-        return Promise.resolve(); // Continue this test.
-    }, function rejected(e) {
-        ProtocolTest.pass("Promise from asyncSetupRuntimeFailureTestSuite.runTestCases() should reject.");
-        ProtocolTest.expectThat(asyncSetupRuntimeFailureTestSuiteDidEvaluate, "Promise did evaluate the async setup function.");
-        ProtocolTest.expectThat(e instanceof TypeError, "Rejected value should be a runtime exception.");
-        return Promise.resolve(); // Continue this test.
-    });
-
-    // Async teardown() failure test cases.
-    let asyncTeardownExplicitFailureTestSuiteDidEvaluate = false;
-    let asyncTeardownExplicitFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTeardownExplicitFailure");
-    asyncTeardownExplicitFailureTestSuite.addTestCase({
-        name: "AsyncFunctionFailure",
-        description: "Check that an async suite with async test functions that throws will reject",
-        async test() { return true; },
+    let asyncSetupFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupFailure");
+    asyncSetupFailureSuite.addTestCase({
+        name: "AsyncTestWithFailureDuringSetup",
+        async setup() {
+            ProtocolTest.log("Rejecting...");
+            return Promise.reject("AsyncSetupFailure reject");
+        },
+        async test() {
+        },
+    });
+    asyncSetupFailureSuite.addTestCase({
+        name: "AsyncTestAfterSetupFailure",
+        async setup() {
+            ProtocolTest.pass("Setup action should still execute if previous test's setup action failed.");
+        },
+        async test() {
+            ProtocolTest.pass("Test should still execute if previous test's setup action failed.");
+        },
         async teardown() {
-            asyncTeardownExplicitFailureTestSuiteDidEvaluate = true;
-            throw "AsyncFunctionFailure Exception Message";
+            ProtocolTest.pass("Setup action should still execute if previous test's setup action failed.");
         },
     });
+    checkResult(asyncSetupFailureSuite, {
+        runCount: 1,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 1,
+    });
 
-    result = result.then(() => {
-        return asyncTeardownExplicitFailureTestSuite.runTestCases();
-    }).then(function resolve() {
-        ProtocolTest.fail("Promise from asyncTeardownExplicitFailureTestSuite.runTestCases() should reject.");
-        return Promise.resolve(); // Continue this test.
-    }, function rejected(e) {
-        ProtocolTest.pass("Promise from asyncTeardownExplicitFailureTestSuite.runTestCases() should reject.");
-        ProtocolTest.expectThat(asyncTeardownExplicitFailureTestSuiteDidEvaluate, "Promise did evaluate the async teardown function.");
-        ProtocolTest.expectEqual(e, "AsyncFunctionFailure Exception Message", "Rejected value should be thrown exception.");
-        return Promise.resolve(); // Continue this test.
-    });
-
-    let asyncTeardownRuntimeFailureTestSuiteDidEvaluate = false;
-    let asyncTeardownRuntimeFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTeardownRuntimeFailure");
-    asyncTeardownRuntimeFailureTestSuite.addTestCase({
-        name: "AsyncFunctionFailure",
-        description: "Check that an async suite with an async teardown function that throws with a runtime error will reject",
-        async test() { return true; },
+    let asyncTeardownExplicitExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTeardownExplicitException");
+    asyncTeardownExplicitExceptionSuite.addTestCase({
+        name: "AsyncTestWithExplicitExceptionDuringTeardown",
+        async test() {
+        },
+        async teardown() {
+            ProtocolTest.log("Throwing...");
+            throw "AsyncTeardownExplicitException throw";
+        },
+    });
+    asyncTeardownExplicitExceptionSuite.addTestCase({
+        name: "AsyncTestAfterTeardownExplicitException",
+        async setup() {
+            ProtocolTest.pass("Setup action should still execute if previous test's teardown action threw an exception.");
+        },
+        async test() {
+            ProtocolTest.pass("Test should still execute if previous test's teardown action threw an exception.");
+        },
         async teardown() {
-            asyncTeardownRuntimeFailureTestSuiteDidEvaluate = true;
+            ProtocolTest.pass("Teardown action should still execute if previous test's teardown action threw an exception.");
+        },
+    });
+    checkResult(asyncTeardownExplicitExceptionSuite, {
+        runCount: 2,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 0,
+    });
+
+    let asyncTeardownRuntimeExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTeardownRuntimeException");
+    asyncTeardownRuntimeExceptionSuite.addTestCase({
+        name: "AsyncTestWithRuntimeExceptionDuringTeardown",
+        async test() {
+        },
+        async teardown() {
+            ProtocolTest.log("Throwing...");
             ({}).x.x.x;
         },
     });
+    asyncTeardownRuntimeExceptionSuite.addTestCase({
+        name: "AsyncTestAfterTeardownRuntimeException",
+        async setup() {
+            ProtocolTest.pass("Setup action should still execute if previous test's teardown action threw an exception.");
+        },
+        async test() {
+            ProtocolTest.pass("Test should still execute if previous test's teardown action threw an exception.");
+        },
+        async teardown() {
+            ProtocolTest.pass("Teardown action should still execute if previous test's teardown action threw an exception.");
+        },
+    });
+    checkResult(asyncTeardownRuntimeExceptionSuite, {
+        runCount: 2,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 0,
+    });
 
-    result = result.then(() => {
-        return asyncTeardownRuntimeFailureTestSuite.runTestCases();
-    }).then(function resolve() {
-        ProtocolTest.fail("Promise from asyncTeardownRuntimeFailureTestSuite.runTestCases() should reject.");
-        return Promise.resolve(); // Continue this test.
-    }, function rejected(e) {
-        ProtocolTest.pass("Promise from asyncTeardownRuntimeFailureTestSuite.runTestCases() should reject.");
-        ProtocolTest.expectThat(asyncTeardownRuntimeFailureTestSuiteDidEvaluate, "Promise did evaluate the async teardown function.");
-        ProtocolTest.expectThat(e instanceof TypeError, "Rejected value should be a runtime exception.");
-        return Promise.resolve(); // Continue this test.
+    let asyncTeardownFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTeardownFailure");
+    asyncTeardownFailureSuite.addTestCase({
+        name: "AsyncTestWithFailureDuringTeardown",
+        async test() {
+        },
+        async teardown() {
+            ProtocolTest.log("Rejecting...");
+            return Promise.reject("AsyncTeardownFailure reject");
+        },
+    });
+    asyncTeardownFailureSuite.addTestCase({
+        name: "AsyncTestAfterTeardownFailure",
+        async setup() {
+            ProtocolTest.pass("Setup action should still execute if previous test's teardown action failed.");
+        },
+        async test() {
+            ProtocolTest.pass("Test should still execute if previous test's teardown action failed.");
+        },
+        async teardown() {
+            ProtocolTest.pass("Teardown action should still execute if previous test's teardown action failed.");
+        },
+    });
+    checkResult(asyncTeardownFailureSuite, {
+        runCount: 2,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 0,
+    });
+
+    let asyncTimeoutSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTimeout");
+    asyncTimeoutSuite.addTestCase({
+        name: "AsyncTestWithTimeout",
+        async test() {
+            ProtocolTest.log("Timeout...");
+            await Promise.delay(50);
+        },
+        timeout: 5,
+    });
+    asyncTimeoutSuite.addTestCase({
+        name: "AsyncTestAfterTimeout",
+        async setup() {
+            ProtocolTest.pass("Setup action should still execute if previous test timed out.");
+        },
+        async test() {
+            ProtocolTest.pass("Test should still execute if previous test timed out.");
+        },
+        async teardown() {
+            ProtocolTest.pass("Teardown action should still execute if previous test timed out.");
+        },
+    });
+    checkResult(asyncTimeoutSuite, {
+        runCount: 2,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 0,
     });
 
     // This will finish the test whether the chain was resolved or rejected.
-    result = result.then(() => { ProtocolTest.completeTest(); });
+    result = result.then(() => {
+        ProtocolTest.completeTest();
+    });
 }
 </script>
 </head>
index c27198b..d837b99 100644 (file)
-PASS: instantiating SyncTestSuite requires name argument.
-PASS: instantiating SyncTestSuite requires string name argument.
-PASS: instantiating SyncTestSuite requires non-whitespace name argument.
-PASS: instantiating SyncTestSuite requires test harness argument.
-PASS: should not be able to add empty test case.
-PASS: should not be able to add non-object test case.
-PASS: test case should require string name.
-PASS: test case should require non-whitespace name.
-PASS: test case should require test function.
-PASS: should not be able to specify non-Function `setup` parameter.
-PASS: should not be able to specify non-Function `setup` parameter.
-PASS: should not be able to specify non-Function `setup` parameter.
-PASS: should not be able to specify non-Function `teardown` parameter.
-PASS: should not be able to specify non-Function `teardown` parameter.
-PASS: should not be able to specify non-Function `teardown` parameter.
-PASS: should not be able to specify async `test` parameter.
-PASS: should not be able to specify async `setup` parameter.
-PASS: should not be able to specify async `teardown` parameter.
-PASS: should not be able to run empty test suite.
+PASS: Should produce an exception.
+Error: Must pass the test's harness as the first argument.
+PASS: Should produce an exception.
+Error: Must pass the test's harness as the first argument.
+PASS: Should produce an exception.
+Error: Must pass the test's harness as the first argument.
+PASS: Should produce an exception.
+Error: Must pass the test's harness as the first argument.
+PASS: Should produce an exception.
+TypeError: undefined is not an object (evaluating 'testcase.setup')
+PASS: Should produce an exception.
+Error: Tried to add non-object test case.
+PASS: Should produce an exception.
+Error: Tried to add test case without a name.
+PASS: Should produce an exception.
+Error: Tried to add test case without a name.
+PASS: Should produce an exception.
+Error: Tried to add test case without `test` function.
+PASS: Should produce an exception.
+Error: Tried to add test case with invalid `setup` parameter (must be a function).
+PASS: Should produce an exception.
+Error: Tried to add test case with invalid `setup` parameter (must be a function).
+PASS: Should produce an exception.
+Error: Tried to add test case with invalid `setup` parameter (must be a function).
+PASS: Should produce an exception.
+Error: Tried to add test case with invalid `teardown` parameter (must be a function).
+PASS: Should produce an exception.
+Error: Tried to add test case with invalid `teardown` parameter (must be a function).
+PASS: Should produce an exception.
+Error: Tried to add test case with invalid `teardown` parameter (must be a function).
+PASS: Should produce an exception.
+Error: Tried to pass a test case with an async `setup`, `test`, or `teardown` function, but this is a synchronous test suite.
+PASS: Should produce an exception.
+Error: Tried to pass a test case with an async `setup`, `test`, or `teardown` function, but this is a synchronous test suite.
+PASS: Should produce an exception.
+Error: Tried to pass a test case with an async `setup`, `test`, or `teardown` function, but this is a synchronous test suite.
+PASS: Should produce an exception.
+Error: Tried to call runTestCases() for suite with no test cases
 
 == Running test suite: SyncTestSuite.RunTwiceSuite
 -- Running test case: DummyTest0
 PASS: Return value of runTwiceSuite.runTestCases() should be true when all tests pass.
-PASS: should not be able to run a test suite twice.
+PASS: Should produce an exception.
+Error: Tried to call runTestCases() more than once.
 
 == Running test suite: SyncTestSuite.SequentialExecution
--- Running test case: DummyTest1
--- Running test case: DummyTest2
--- Running test case: DummyTest3
--- Running test case: FailingTest4
-!! EXCEPTION: [object Object]
+-- Running test case: 1 (Pass)
+-- Running test case: 2 (Pass)
+-- Running test case: 3 (Pass)
+-- Running test case: 4 (Pass)
+PASS: Return value of SyncTestSuite.SequentialExecution.runTestCases() should be true even if a test case fails.
+PASS: SyncTestSuite.SequentialExecution should have executed 4 tests.
+PASS: SyncTestSuite.SequentialExecution should have passed 4 tests.
+PASS: SyncTestSuite.SequentialExecution should have failed 0 tests.
+PASS: SyncTestSuite.SequentialExecution should have skipped 0 tests.
+
+== Running test suite: SyncTestSuite.ContinueOnFailure
+-- Running test case: 1 (Pass)
+-- Running test case: 2 (Fail)
+Throwing...
+!! EXCEPTION: {"x":"ContinueOnFailure throw"}
 Stack Trace: (suppressed)
-PASS: Return value of sequentialExecutionSuite.runTestCases() should be false when a test case fails.
-PASS: sequentialExecutionSuite should have executed four tests.
-PASS: sequentialExecutionSuite should have passed three tests.
-PASS: sequentialExecutionSuite should have failed 1 test.
-PASS: sequentialExecutionSuite should have skipped zero tests.
 
-== Running test suite: SyncTestSuite.AbortOnFailure
--- Running test case: PassingTest5
--- Running test case: FailingTest6
-PASS: Return value of abortOnFailureSuite.runTestCases() should be false when a test case fails.
-PASS: abortOnFailureSuite should have executed two tests.
-PASS: abortOnFailureSuite should have passed one test.
-PASS: abortOnFailureSuite should have failed one test.
-PASS: abortOnFailureSuite should have skipped one test.
+-- Running test case: 3 (Pass)
+-- Running test case: 4 (Fail)
+Failing...
+PASS: Return value of SyncTestSuite.ContinueOnFailure.runTestCases() should be true even if a test case fails.
+PASS: SyncTestSuite.ContinueOnFailure should have executed 4 tests.
+PASS: SyncTestSuite.ContinueOnFailure should have passed 2 tests.
+PASS: SyncTestSuite.ContinueOnFailure should have failed 2 tests.
+PASS: SyncTestSuite.ContinueOnFailure should have skipped 0 tests.
 
 == Running test suite: SyncTestSuite.SetupAndTeardown
 -- Running test setup.
 -- Running test case: TestWithSetupAndTeardown
-PASS: Test should see side effects of running setup() action.
 -- Running test teardown.
-PASS: Teardown should see side effects of running setup() action.
-
 -- Running test case: TestRunningAfterTeardown
-PASS: Test should see side effects of previous test's teardown() action.
+PASS: Return value of SyncTestSuite.SetupAndTeardown.runTestCases() should be true even if a test case fails.
+PASS: SyncTestSuite.SetupAndTeardown should have executed 2 tests.
+PASS: SyncTestSuite.SetupAndTeardown should have passed 2 tests.
+PASS: SyncTestSuite.SetupAndTeardown should have failed 0 tests.
+PASS: SyncTestSuite.SetupAndTeardown should have skipped 0 tests.
 
 == Running test suite: SyncTestSuite.SetupException
 -- Running test setup.
-!! EXCEPTION: 
+Throwing...
+!! EXCEPTION: SetupException throw
 Stack Trace: (suppressed)
 
+-- Running test setup.
+PASS: Setup action should still execute if previous test's setup action threw an exception.
+-- Running test case: TestAfterSetupException
+PASS: Test should still execute if previous test's setup action threw an exception.
+-- Running test teardown.
+PASS: Teardown action should still execute if previous test's setup action threw an exception.
+PASS: Return value of SyncTestSuite.SetupException.runTestCases() should be true even if a test case fails.
+PASS: SyncTestSuite.SetupException should have executed 1 tests.
+PASS: SyncTestSuite.SetupException should have passed 1 tests.
+PASS: SyncTestSuite.SetupException should have failed 1 tests.
+PASS: SyncTestSuite.SetupException should have skipped 1 tests.
+
 == Running test suite: SyncTestSuite.SetupFailure
 -- Running test setup.
+Failing...
 !! SETUP FAILED
 
+-- Running test setup.
+PASS: Setup action should still execute if previous test's setup action failed.
+-- Running test case: TestAfterSetupException
+PASS: Test should still execute if previous test's setup action failed.
+-- Running test teardown.
+PASS: Teardown action should still execute if previous test's setup action failed.
+PASS: Return value of SyncTestSuite.SetupFailure.runTestCases() should be true even if a test case fails.
+PASS: SyncTestSuite.SetupFailure should have executed 1 tests.
+PASS: SyncTestSuite.SetupFailure should have passed 1 tests.
+PASS: SyncTestSuite.SetupFailure should have failed 1 tests.
+PASS: SyncTestSuite.SetupFailure should have skipped 1 tests.
+
 == Running test suite: SyncTestSuite.TeardownException
 -- Running test case: TestWithExceptionDuringTeardown
 -- Running test teardown.
-!! EXCEPTION: 
+Throwing...
+!! EXCEPTION: TeardownException throw
 Stack Trace: (suppressed)
 
+-- Running test setup.
+PASS: Setup action should still execute if previous test's teardown action threw an exception.
+-- Running test case: TestAfterTeardownException
+PASS: Test should still execute if previous test's teardown action threw an exception.
+-- Running test teardown.
+PASS: Teardown action should still execute if previous test's teardown action threw an exception.
+PASS: Return value of SyncTestSuite.TeardownException.runTestCases() should be true even if a test case fails.
+PASS: SyncTestSuite.TeardownException should have executed 2 tests.
+PASS: SyncTestSuite.TeardownException should have passed 1 tests.
+PASS: SyncTestSuite.TeardownException should have failed 1 tests.
+PASS: SyncTestSuite.TeardownException should have skipped 0 tests.
+
 == Running test suite: SyncTestSuite.TeardownFailure
 -- Running test case: TestWithExceptionDuringTeardown
 -- Running test teardown.
+Failing...
 !! TEARDOWN FAILED
 
+-- Running test setup.
+PASS: Setup action should still execute if previous test's teardown action failed.
+-- Running test case: TestAfterTeardownException
+PASS: Test should still execute if previous test's teardown action failed.
+-- Running test teardown.
+PASS: Teardown action should still execute if previous test's teardown action failed.
+PASS: Return value of SyncTestSuite.TeardownFailure.runTestCases() should be true even if a test case fails.
+PASS: SyncTestSuite.TeardownFailure should have executed 2 tests.
+PASS: SyncTestSuite.TeardownFailure should have passed 1 tests.
+PASS: SyncTestSuite.TeardownFailure should have failed 1 tests.
+PASS: SyncTestSuite.TeardownFailure should have skipped 0 tests.
+
index 9390e8d..d65c095 100644 (file)
@@ -1,4 +1,4 @@
-<!doctype html>
+<!DOCTYPE html>
 <html>
 <head>
 <script src="../../http/tests/inspector/resources/protocol-test.js"></script>
@@ -7,350 +7,404 @@ function test()
 {
     ProtocolTest.suppressStackTraces = true;
 
-    try {
-        let result = new SyncTestSuite(this);
-        ProtocolTest.log("FAIL: instantiating SyncTestSuite requires name argument.");
-    } catch (e) {
-        ProtocolTest.log("PASS: instantiating SyncTestSuite requires name argument.");
-    }
+    ProtocolTest.expectException(() => {
+        new SyncTestSuite(this);
+    });
 
-    try {
-        let result = new SyncTestSuite(this, {});
-        ProtocolTest.log("FAIL: instantiating SyncTestSuite requires string name argument.");
-    } catch (e) {
-        ProtocolTest.log("PASS: instantiating SyncTestSuite requires string name argument.");
-    }
+    ProtocolTest.expectException(() => {
+        new SyncTestSuite(this, {});
+    });
 
-    try {
-        let result = new SyncTestSuite(this, "      ");
-        ProtocolTest.log("FAIL: instantiating SyncTestSuite requires non-whitespace name argument.");
-    } catch (e) {
-        ProtocolTest.log("PASS: instantiating SyncTestSuite requires non-whitespace name argument.");
-    }
+    ProtocolTest.expectException(() => {
+        new SyncTestSuite(this, "      ");
+    });
 
-    try {
-        let result = new SyncTestSuite("something", {});
-        ProtocolTest.log("FAIL: instantiating SyncTestSuite requires test harness argument.");
-    } catch (e) {
-        ProtocolTest.log("PASS: instantiating SyncTestSuite requires test harness argument.");
-    }
+    ProtocolTest.expectException(() => {
+        new SyncTestSuite("something", {});
+    });
 
     let badArgsSuite = ProtocolTest.createSyncSuite("dummy");
-    try {
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase();
-        ProtocolTest.log("FAIL: should not be able to add empty test case.");
-    } catch (e) {
-        ProtocolTest.log("PASS: should not be able to add empty test case.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase("string");
-        ProtocolTest.log("FAIL: should not be able to add non-object test case.");
-    } catch (e) {
-        ProtocolTest.log("PASS: should not be able to add non-object test case.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: {},
-            test: () => true,
+            test() {
+                return true;
+            },
         });
-        ProtocolTest.log("FAIL: test case should require string name.");
-    } catch (e) {
-        ProtocolTest.log("PASS: test case should require string name.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "        ",
-            test() {},
+            test() {
+            },
         });
-        ProtocolTest.log("FAIL: test case should require non-whitespace name.");
-    } catch (e) {
-        ProtocolTest.log("PASS: test case should require non-whitespace name.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
             test: null,
         });
-        ProtocolTest.log("FAIL: test case should require test function.");
-    } catch (e) {
-        ProtocolTest.log("PASS: test case should require test function.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
-            test() {},
+            test() {
+            },
             setup: "astd"
         });
-        ProtocolTest.log("FAIL: should not be able to specify non-Function `setup` parameter.");
-    } catch (e) {
-        ProtocolTest.log("PASS: should not be able to specify non-Function `setup` parameter.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
-            test() {},
+            test() {
+            },
             setup: 123
         });
-        ProtocolTest.log("FAIL: should not be able to specify non-Function `setup` parameter.");
-    } catch (e) {
-        ProtocolTest.log("PASS: should not be able to specify non-Function `setup` parameter.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
-            test() {},
+            test() {
+            },
             setup: Promise.resolve()
         });
-        ProtocolTest.log("FAIL: should not be able to specify non-Function `setup` parameter.");
-    } catch (e) {
-        ProtocolTest.log("PASS: should not be able to specify non-Function `setup` parameter.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
-            test() {},
+            test() {
+            },
             teardown: "astd"
         });
-        ProtocolTest.log("FAIL: should not be able to specify non-Function `teardown` parameter.");
-    } catch (e) {
-        ProtocolTest.log("PASS: should not be able to specify non-Function `teardown` parameter.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
-            test() {},
+            test() {
+            },
             teardown: 123
         });
-        ProtocolTest.log("FAIL: should not be able to specify non-Function `teardown` parameter.");
-    } catch (e) {
-        ProtocolTest.log("PASS: should not be able to specify non-Function `teardown` parameter.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
-            test() {},
+            test() {
+            },
             teardown: Promise.resolve()
         });
-        ProtocolTest.log("FAIL: should not be able to specify non-Function `teardown` parameter.");
-    } catch (e) {
-        ProtocolTest.log("PASS: should not be able to specify non-Function `teardown` parameter.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
-            async test() {},
+            async test() {
+            },
         });
-        ProtocolTest.log("FAIL: should not be able to specify async `test` parameter.");
-    } catch (e) {
-        ProtocolTest.log("PASS: should not be able to specify async `test` parameter.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
-            async setup() {},
-            test() {},
+            async setup() {
+            },
+            test() {
+            },
         });
-        ProtocolTest.log("FAIL: should not be able to specify async `setup` parameter.");
-    } catch (e) {
-        ProtocolTest.log("PASS: should not be able to specify async `setup` parameter.");
-    }
-    try {
+    });
+    ProtocolTest.expectException(() => {
         badArgsSuite.addTestCase({
             name: "foo",
-            test() {},
-            async teardown() {},
+            test() {
+            },
+            async teardown() {
+            },
         });
-        ProtocolTest.log("FAIL: should not be able to specify async `teardown` parameter.");
-    } catch (e) {
-        ProtocolTest.log("PASS: should not be able to specify async `teardown` parameter.");
-    }
+    });
 
-    let runEmptySuite = ProtocolTest.createSyncSuite("SyncTestSuite.RunEmptySuite");
-    try {
+    ProtocolTest.expectException(() => {
+        let runEmptySuite = ProtocolTest.createSyncSuite("SyncTestSuite.RunEmptySuite");
         runEmptySuite.runTestCases();
-        ProtocolTest.log("FAIL: should not be able to run empty test suite.");
-    } catch (e) {
-        ProtocolTest.log("PASS: should not be able to run empty test suite.");
-    }
+    });
 
     let runTwiceSuite = ProtocolTest.createSyncSuite("SyncTestSuite.RunTwiceSuite");
     runTwiceSuite.addTestCase({
         name: "DummyTest0",
-        description: "Check that a suite can't run more than once.",
-        test: () => true,
+        test() {
+            return true;
+        },
     });
 
-    try {
-        let result = runTwiceSuite.runTestCases();
-        ProtocolTest.expectThat(result === true, "Return value of runTwiceSuite.runTestCases() should be true when all tests pass.");
+    ProtocolTest.expectThat(runTwiceSuite.runTestCases() === true, "Return value of runTwiceSuite.runTestCases() should be true when all tests pass.");
+
+    ProtocolTest.expectException(() => {
+        runTwiceSuite.runTestCases();
+    });
 
-        runTwiceSuite.runTestCases(); // Try to trigger an error.
-        ProtocolTest.log("FAIL: should not be able to run a test suite twice.");
-    } catch (e) {
-        ProtocolTest.log("PASS: should not be able to run a test suite twice.");
+    function checkResult(suite, expectedCounts) {
+        ProtocolTest.expectThat(suite.runTestCases(), `Return value of ${suite.name}.runTestCases() should be true even if a test case fails.`);
+        ProtocolTest.expectEqual(expectedCounts.runCount, suite.runCount, `${suite.name} should have executed ${expectedCounts.runCount} tests.`);
+        ProtocolTest.expectEqual(expectedCounts.passCount, suite.passCount, `${suite.name} should have passed ${expectedCounts.passCount} tests.`);
+        ProtocolTest.expectEqual(expectedCounts.failCount, suite.failCount, `${suite.name} should have failed ${expectedCounts.failCount} tests.`);
+        ProtocolTest.expectEqual(expectedCounts.skipCount, suite.skipCount, `${suite.name} should have skipped ${expectedCounts.skipCount} tests.`);
     }
 
     let thrownError = new Error({"token": 666});
 
+    let sequentialExecutionPhase = 0;
     let sequentialExecutionSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SequentialExecution");
     sequentialExecutionSuite.addTestCase({
-        name: "DummyTest1",
-        description: "Check test case execution order.",
-        test: () => true
+        name: "1 (Pass)",
+        test() {
+            ProtocolTest.assert(sequentialExecutionPhase === 0);
+            sequentialExecutionPhase = 1;
+            return true;
+        },
     });
     sequentialExecutionSuite.addTestCase({
-        name: "DummyTest2",
-        description: "Check test case execution order.",
-        test: () => true
+        name: "2 (Pass)",
+        test() {
+            ProtocolTest.assert(sequentialExecutionPhase === 1);
+            sequentialExecutionPhase = 2;
+            return true;
+        },
     });
     sequentialExecutionSuite.addTestCase({
-        name: "DummyTest3",
-        description: "Check test case execution order.",
-        test: () => true
+        name: "3 (Pass)",
+        test() {
+            ProtocolTest.assert(sequentialExecutionPhase === 2);
+            sequentialExecutionPhase = 3;
+            return true;
+        },
     });
     sequentialExecutionSuite.addTestCase({
-        name: "FailingTest4",
-        description: "Check that test fails by throwing an Error instance.",
-        test() { throw thrownError; }
+        name: "4 (Pass)",
+        test() {
+            ProtocolTest.assert(sequentialExecutionPhase === 3);
+            sequentialExecutionPhase = 4;
+            return true;
+        },
     });
-
-    let result = sequentialExecutionSuite.runTestCases();
-    ProtocolTest.expectThat(result === false, "Return value of sequentialExecutionSuite.runTestCases() should be false when a test case fails.");
-    ProtocolTest.expectThat(sequentialExecutionSuite.runCount === 4, "sequentialExecutionSuite should have executed four tests.");
-    ProtocolTest.expectThat(sequentialExecutionSuite.passCount === 3, "sequentialExecutionSuite should have passed three tests.");
-    ProtocolTest.expectThat(sequentialExecutionSuite.failCount === 1, "sequentialExecutionSuite should have failed 1 test.");
-    ProtocolTest.expectThat(sequentialExecutionSuite.skipCount === 0, "sequentialExecutionSuite should have skipped zero tests.");
-
-    let abortOnFailureSuite = ProtocolTest.createSyncSuite("SyncTestSuite.AbortOnFailure");
-    abortOnFailureSuite.addTestCase({
-        name: "PassingTest5",
-        description: "This test is a dummy.",
-        test: () => true
-    });
-    abortOnFailureSuite.addTestCase({
-        name: "FailingTest6",
-        description: "This test should fail by explicitly returning `false`.",
-        test: () => false
-    });
-    abortOnFailureSuite.addTestCase({
-        name: "PassingTest7",
-        description: "This test should not executed when the preceding test fails.",
-        test: () => true
+    checkResult(sequentialExecutionSuite, {
+        runCount: 4,
+        passCount: 4,
+        failCount: 0,
+        skipCount: 0,
     });
 
-    abortOnFailureSuite.runTestCases();
-    ProtocolTest.expectThat(result === false, "Return value of abortOnFailureSuite.runTestCases() should be false when a test case fails.");
-    ProtocolTest.expectThat(abortOnFailureSuite.runCount === 2, "abortOnFailureSuite should have executed two tests.");
-    ProtocolTest.expectThat(abortOnFailureSuite.passCount === 1, "abortOnFailureSuite should have passed one test.");
-    ProtocolTest.expectThat(abortOnFailureSuite.failCount === 1, "abortOnFailureSuite should have failed one test.");
-    ProtocolTest.expectThat(abortOnFailureSuite.skipCount === 1, "abortOnFailureSuite should have skipped one test.");
-
-    let setupAndTeardownSymbol = Symbol("sync-suite-setup-and-teardown-token");
-    window[setupAndTeardownSymbol] = 0;
+    let continueOnFailureSuite = ProtocolTest.createSyncSuite("SyncTestSuite.ContinueOnFailure");
+    continueOnFailureSuite.addTestCase({
+        name: "1 (Pass)",
+        test() {
+            return true;
+        },
+    });
+    continueOnFailureSuite.addTestCase({
+        name: "2 (Fail)",
+        test() {
+            ProtocolTest.log("Throwing...");
+            throw {x: "ContinueOnFailure throw"};
+        },
+    });
+    continueOnFailureSuite.addTestCase({
+        name: "3 (Pass)",
+        test() {
+            return true;
+        },
+    });
+    continueOnFailureSuite.addTestCase({
+        name: "4 (Fail)",
+        test() {
+            ProtocolTest.log("Failing...");
+            return false;
+        },
+    });
+    checkResult(continueOnFailureSuite, {
+        runCount: 4,
+        passCount: 2,
+        failCount: 2,
+        skipCount: 0,
+    });
 
-    let setupAndTeardownTestSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupAndTeardown");
-    setupAndTeardownTestSuite.addTestCase({
+    let setupAndTeardownPhase = 0;
+    let setupAndTeardownSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupAndTeardown");
+    setupAndTeardownSuite.addTestCase({
         name: "TestWithSetupAndTeardown",
-        description: "Check execution order for setup and teardown actions.",
-        setup: () => {
-            window[setupAndTeardownSymbol] = 1;
+        setup() {
+            ProtocolTest.assert(setupAndTeardownPhase === 0);
+            setupAndTeardownPhase = 1;
             return true;
         },
         test() {
-            ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 1, "Test should see side effects of running setup() action.");
-            window[setupAndTeardownSymbol] = 2;
+            ProtocolTest.assert(setupAndTeardownPhase === 1);
+            setupAndTeardownPhase = 2;
             return true;
         },
-        teardown: () => {
-            ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 2, "Teardown should see side effects of running setup() action.");
-            window[setupAndTeardownSymbol] = 3;
+        teardown() {
+            ProtocolTest.assert(setupAndTeardownPhase === 2);
+            setupAndTeardownPhase = 3;
             return true;
-        }
+        },
     });
-    setupAndTeardownTestSuite.addTestCase({
+    setupAndTeardownSuite.addTestCase({
         name: "TestRunningAfterTeardown",
-        description: "Check execution order for test after a teardown action.",
         test() {
-            ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 3, "Test should see side effects of previous test's teardown() action.");
+            ProtocolTest.assert(setupAndTeardownPhase === 3);
             return true;
         },
     });
+    checkResult(setupAndTeardownSuite, {
+        runCount: 2,
+        passCount: 2,
+        failCount: 0,
+        skipCount: 0,
+    });
 
-    setupAndTeardownTestSuite.runTestCases();
-
-    let setupExceptionTestSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupException");
-    setupExceptionTestSuite.addTestCase({
+    
+    let setupExceptionSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupException");
+    setupExceptionSuite.addTestCase({
         name: "TestWithExceptionDuringSetup",
-        description: "Check execution order for setup action that throws an exception.",
-        setup: () => { throw new Error() },
+        setup() {
+            ProtocolTest.log("Throwing...");
+            throw "SetupException throw";
+        },
         test() {
-            ProtocolTest.assert(false, "Test should not execute if its setup action threw an exception.");
+            ProtocolTest.fail("Test should not execute if its setup action threw an exception.");
+            return false;
+        },
+        teardown() {
+            ProtocolTest.fail("Teardown action should not execute if its setup action threw an exception.");
             return false;
         },
-        teardown: () => {
-            ProtocolTest.assert(false, "Teardown action should not execute if its setup action threw an exception.");
-            return false;           
-        }
     });
-    setupExceptionTestSuite.runTestCases();
+    setupExceptionSuite.addTestCase({
+        name: "TestAfterSetupException",
+        setup() {
+            ProtocolTest.pass("Setup action should still execute if previous test's setup action threw an exception.");
+            return true;
+        },
+        test() {
+            ProtocolTest.pass("Test should still execute if previous test's setup action threw an exception.");
+            return true;
+        },
+        teardown() {
+            ProtocolTest.pass("Teardown action should still execute if previous test's setup action threw an exception.");
+            return true;
+        },
+    });
+    checkResult(setupExceptionSuite, {
+        runCount: 1,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 1,
+    });
 
-    let setupFailureTestSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupFailure");
-    setupFailureTestSuite.addTestCase({
+    let setupFailureSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupFailure");
+    setupFailureSuite.addTestCase({
         name: "TestWithFailureDuringSetup",
-        description: "Check execution order for setup action that has failed.",
-        setup: () => false,
+        setup() {
+            ProtocolTest.log("Failing...");
+            return false;
+        },
         test() {
-            ProtocolTest.assert(false, "Test should not execute if its setup action returned false.")
+            ProtocolTest.fail("Test should not execute if its setup action failed.");
+            return false;
+        },
+        teardown() {
+            ProtocolTest.fail("Teardown action should not execute if its setup action failed.");
             return false;
         },
-        teardown: () => {
-            ProtocolTest.assert(false, "Teardown action should not execute if its setup action returned false.")
-            return false;           
-        }
     });
-    setupFailureTestSuite.runTestCases();
+    setupFailureSuite.addTestCase({
+        name: "TestAfterSetupException",
+        setup() {
+            ProtocolTest.pass("Setup action should still execute if previous test's setup action failed.");
+            return true;
+        },
+        test() {
+            ProtocolTest.pass("Test should still execute if previous test's setup action failed.");
+            return true;
+        },
+        teardown() {
+            ProtocolTest.pass("Teardown action should still execute if previous test's setup action failed.");
+            return true;
+        },
+    });
+    checkResult(setupFailureSuite, {
+        runCount: 1,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 1,
+    });
 
-    let teardownExceptionTestSuite = ProtocolTest.createSyncSuite("SyncTestSuite.TeardownException");
-    teardownExceptionTestSuite.addTestCase({
+    let teardownExceptionSuite = ProtocolTest.createSyncSuite("SyncTestSuite.TeardownException");
+    teardownExceptionSuite.addTestCase({
         name: "TestWithExceptionDuringTeardown",
-        description: "Check execution order for teardown action that throws an exception.",
-        test: () => true,
-        teardown: () => { throw new Error() }
+        test() {
+            return true;
+        },
+        teardown() {
+            ProtocolTest.log("Throwing...");
+            throw "TeardownException throw";
+        },
     });
-    teardownExceptionTestSuite.addTestCase({
+    teardownExceptionSuite.addTestCase({
         name: "TestAfterTeardownException",
-        descrption: "Check execution order for test after previous test's teardown throws an exception",
-        setup: () => {
-            ProtocolTest.assert(false, "Setup action should not execute if previous test's teardown action threw an exception.");
-            return false;
+        setup() {
+            ProtocolTest.pass("Setup action should still execute if previous test's teardown action threw an exception.");
+            return true;
         },
         test() {
-            ProtocolTest.assert(false, "Test should not execute if previous test's teardown action threw an exception.");
-            return false;
-        }
+            ProtocolTest.pass("Test should still execute if previous test's teardown action threw an exception.");
+            return true;
+        },
+        teardown() {
+            ProtocolTest.pass("Teardown action should still execute if previous test's teardown action threw an exception.");
+            return true;
+        },
+    });
+    checkResult(teardownExceptionSuite, {
+        runCount: 2,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 0,
     });
-    teardownExceptionTestSuite.runTestCases();
 
-    let teardownFailureTestSuite = ProtocolTest.createSyncSuite("SyncTestSuite.TeardownFailure");
-    teardownFailureTestSuite.addTestCase({
+    let teardownFailureSuite = ProtocolTest.createSyncSuite("SyncTestSuite.TeardownFailure");
+    teardownFailureSuite.addTestCase({
         name: "TestWithExceptionDuringTeardown",
-        description: "Check execution order for teardown action that has failed.",
-        test: () => true,
-        teardown: () => false,
+        test() {
+            return true;
+        },
+        teardown() {
+            ProtocolTest.log("Failing...");
+            return false;
+        },
     });
-    teardownFailureTestSuite.addTestCase({
+    teardownFailureSuite.addTestCase({
         name: "TestAfterTeardownException",
-        descrption: "Check execution order for test after previous test's teardown throws an exception",
-        setup: () => {
-            ProtocolTest.assert(false, "Setup action should not execute if previous test's teardown action failed.");
-            return false;
+        setup() {
+            ProtocolTest.pass("Setup action should still execute if previous test's teardown action failed.");
+            return true;
         },
         test() {
-            ProtocolTest.assert(false, "Test should not execute if previous test's teardown action failed.");
-            return false;
-        }
+            ProtocolTest.pass("Test should still execute if previous test's teardown action failed.");
+            return true;
+        },
+        teardown() {
+            ProtocolTest.pass("Teardown action should still execute if previous test's teardown action failed.");
+            return true;
+        },
+    });
+    checkResult(teardownFailureSuite, {
+        runCount: 2,
+        passCount: 1,
+        failCount: 1,
+        skipCount: 0,
     });
-    teardownFailureTestSuite.runTestCases();
 
     ProtocolTest.completeTest();
 }
index 6809f8d..edf9f7c 100644 (file)
@@ -1,3 +1,30 @@
+2019-04-05  Devin Rousso  <drousso@apple.com>
+
+        Web Inspector: TestSuite test cases should have their own timeout to ensure tests fail with output instead of timeout by test runner
+        https://bugs.webkit.org/show_bug.cgi?id=162814
+        <rdar://problem/28574102>
+
+        Reviewed by Brian Burg.
+
+        A 10s timer is started for every test case added to an async suite. The timer is cleared
+        when the test finishes, but if the timer fires, the test is forcibly ended with an error.
+
+        This timer can be configured by setting a `timeout` value when adding the test case. Values
+        are expected to be in milliseconds. The value `-1` will prevent a timer from being set.
+
+        This change also relaxes the expectation that any individual test case failure will stop the
+        rest of the suite from running. Since timers are set per test case, it is possible to
+        recover from a "bad" test case to still run the remaining test cases.
+
+        NOTE: there may be unexpected behaviour if a test times out, as the timer doesn't actually
+        stop the execution of the test, so it may still run and log information, which may appear
+        "out of nowhere" in the middle of other tests.
+
+        * UserInterface/Test/TestSuite.js:
+        (TestSuite.prototype.get passCount):
+        (AsyncTestSuite.prototype.runTestCases):
+        (SyncTestSuite.prototype.runTestCases):
+
 2019-04-03  Devin Rousso  <drousso@apple.com>
 
         Web Inspector: Single click on links in non-read-only TextEditors should not follow links
index 3e17fdf..5e8192e 100644 (file)
@@ -54,7 +54,7 @@ TestSuite = class TestSuite
 
     get passCount()
     {
-        return this.runCount - this.failCount;
+        return this.runCount - (this.failCount - this.skipCount);
     }
 
     get skipCount()
@@ -135,45 +135,73 @@ AsyncTestSuite = class AsyncTestSuite extends TestSuite
 
         // Avoid adding newlines if nothing was logged.
         let priorLogCount = this._harness.logCount;
-        let result = this.testcases.reduce((chain, testcase, i) => {
-            if (testcase.setup) {
-                chain = chain.then(() => {
+
+        return Promise.resolve().then(() => Promise.chain(this.testcases.map((testcase, i) => () => new Promise(async (resolve, reject) => {
+            if (i > 0 && priorLogCount < this._harness.logCount)
+                this._harness.log("");
+            priorLogCount = this._harness.logCount;
+
+            let hasTimeout = testcase.timeout !== -1;
+            let timeoutId = undefined;
+            if (hasTimeout) {
+                let delay = testcase.timeout || 10000;
+                timeoutId = setTimeout(() => {
+                    if (!timeoutId)
+                        return;
+
+                    timeoutId = undefined;
+
+                    this.failCount++;
+                    this._harness.log(`!! TIMEOUT: took longer than ${delay}ms`);
+
+                    resolve();
+                }, delay);
+            }
+
+            try {
+                if (testcase.setup) {
                     this._harness.log("-- Running test setup.");
+                    priorLogCount++;
+
                     if (testcase.setup[Symbol.toStringTag] === "AsyncFunction")
-                        return testcase.setup();
-                    return new Promise(testcase.setup);
-                });
-            }
+                        await testcase.setup();
+                    else
+                        await new Promise(testcase.setup);
+                }
 
-            chain = chain.then(() => {
-                if (i > 0 && priorLogCount + 1 < this._harness.logCount)
-                    this._harness.log("");
+                this.runCount++;
 
-                priorLogCount = this._harness.logCount;
                 this._harness.log(`-- Running test case: ${testcase.name}`);
-                this.runCount++;
+                priorLogCount++;
+
                 if (testcase.test[Symbol.toStringTag] === "AsyncFunction")
-                    return testcase.test();
-                return new Promise(testcase.test);
-            });
+                    await testcase.test();
+                else
+                    await new Promise(testcase.test);
 
-            if (testcase.teardown) {
-                chain = chain.then(() => {
+                if (testcase.teardown) {
                     this._harness.log("-- Running test teardown.");
+                    priorLogCount++;
+
                     if (testcase.teardown[Symbol.toStringTag] === "AsyncFunction")
-                        return testcase.teardown();
-                    return new Promise(testcase.teardown);
-                });
+                        await testcase.teardown();
+                    else
+                        await new Promise(testcase.teardown);
+                }
+            } catch (e) {
+                this.failCount++;
+                this.logThrownObject(e);
             }
-            return chain;
-        }, Promise.resolve());
 
-        return result.catch((e) => {
-            this.failCount++;
-            this.logThrownObject(e);
+            if (!hasTimeout || timeoutId) {
+                clearTimeout(timeoutId);
+                timeoutId = undefined;
 
-            throw e; // Reject this promise by re-throwing the error.
-        });
+                resolve();
+            }
+        })))
+        // Clear result value.
+        .then(() => {}));
     }
 };
 
@@ -208,53 +236,51 @@ SyncTestSuite = class SyncTestSuite extends TestSuite
         let priorLogCount = this._harness.logCount;
         for (let i = 0; i < this.testcases.length; i++) {
             let testcase = this.testcases[i];
-            if (i > 0 && priorLogCount + 1 < this._harness.logCount)
-                this._harness.log("");
 
+            if (i > 0 && priorLogCount < this._harness.logCount)
+                this._harness.log("");
             priorLogCount = this._harness.logCount;
 
-            // Run the setup action, if one was provided.
-            if (testcase.setup) {
-                this._harness.log("-- Running test setup.");
-                try {
-                    let result = testcase.setup.call(null);
-                    if (result === false) {
+            try {
+                // Run the setup action, if one was provided.
+                if (testcase.setup) {
+                    this._harness.log("-- Running test setup.");
+                    priorLogCount++;
+
+                    let setupResult = testcase.setup();
+                    if (setupResult === false) {
                         this._harness.log("!! SETUP FAILED");
-                        return false;
+                        this.failCount++;
+                        continue;
                     }
-                } catch (e) {
-                    this.logThrownObject(e);
-                    return false;
                 }
-            }
 
-            this._harness.log("-- Running test case: " + testcase.name);
-            this.runCount++;
-            try {
-                let result = testcase.test.call(null);
-                if (result === false) {
+                this.runCount++;
+
+                this._harness.log(`-- Running test case: ${testcase.name}`);
+                priorLogCount++;
+
+                let testResult = testcase.test();
+                if (testResult === false) {
                     this.failCount++;
-                    return false;
+                    continue;
                 }
-            } catch (e) {
-                this.failCount++;
-                this.logThrownObject(e);
-                return false;
-            }
 
-            // Run the teardown action, if one was provided.
-            if (testcase.teardown) {
-                this._harness.log("-- Running test teardown.");
-                try {
-                    let result = testcase.teardown.call(null);
-                    if (result === false) {
+                // Run the teardown action, if one was provided.
+                if (testcase.teardown) {
+                    this._harness.log("-- Running test teardown.");
+                    priorLogCount++;
+
+                    let teardownResult = testcase.teardown();
+                    if (teardownResult === false) {
                         this._harness.log("!! TEARDOWN FAILED");
-                        return false;
+                        this.failCount++;
+                        continue;
                     }
-                } catch (e) {
-                    this.logThrownObject(e);
-                    return false;
                 }
+            } catch (e) {
+                this.failCount++;
+                this.logThrownObject(e);
             }
         }