Web Inspector: SyncTestSuite should complain if passed an async setup/test/teardown...
[WebKit-https.git] / Source / WebInspectorUI / UserInterface / Test / TestSuite.js
1 /*
2  * Copyright (C) 2015 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  *
8  * 1.  Redistributions of source code must retain the above copyright
9  *     notice, this list of conditions and the following disclaimer.
10  * 2.  Redistributions in binary form must reproduce the above copyright
11  *     notice, this list of conditions and the following disclaimer in the
12  *     documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
15  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
16  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
17  * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
18  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
19  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
20  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
21  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 TestSuite = class TestSuite
27 {
28     constructor(harness, name) {
29         if (!(harness instanceof TestHarness))
30             throw new Error("Must pass the test's harness as the first argument.");
31
32         if (typeof name !== "string" || !name.trim().length)
33             throw new Error("Tried to create TestSuite without string suite name.");
34
35         this.name = name;
36         this._harness = harness;
37
38         this.testcases = [];
39         this.runCount = 0;
40         this.failCount = 0;
41     }
42
43     // Use this if the test file only has one suite, and no handling
44     // of the value returned by runTestCases() is needed.
45     runTestCasesAndFinish()
46     {
47         throw new Error("Must be implemented by subclasses.");
48     }
49
50     runTestCases()
51     {
52         throw new Error("Must be implemented by subclasses.");
53     }
54
55     get passCount()
56     {
57         return this.runCount - this.failCount;
58     }
59
60     get skipCount()
61     {
62         if (this.failCount)
63             return this.testcases.length - this.runCount;
64         else
65             return 0;
66     }
67
68     addTestCase(testcase)
69     {
70         if (!testcase || !(testcase instanceof Object))
71             throw new Error("Tried to add non-object test case.");
72
73         if (typeof testcase.name !== "string" || !testcase.name.trim().length)
74             throw new Error("Tried to add test case without a name.");
75
76         if (typeof testcase.test !== "function")
77             throw new Error("Tried to add test case without `test` function.");
78
79         if (testcase.setup && typeof testcase.setup !== "function")
80             throw new Error("Tried to add test case with invalid `setup` parameter (must be a function).");
81
82         if (testcase.teardown && typeof testcase.teardown !== "function")
83             throw new Error("Tried to add test case with invalid `teardown` parameter (must be a function).");
84
85         this.testcases.push(testcase);
86     }
87
88     // Protected
89
90     logThrownObject(e)
91     {
92         let message = e;
93         let stack = "(unknown)";
94         if (e instanceof Error) {
95             message = e.message;
96             if (e.stack)
97                 stack = e.stack;
98         }
99
100         if (typeof message !== "string")
101             message = JSON.stringify(message);
102
103         let sanitizedStack = this._harness.sanitizeStack(stack);
104
105         let result = `!! EXCEPTION: ${message}`;
106         if (stack)
107             result += `\nStack Trace: ${sanitizedStack}`;
108
109         this._harness.log(result);
110     }
111 };
112
113 AsyncTestSuite = class AsyncTestSuite extends TestSuite
114 {
115     runTestCasesAndFinish()
116     {
117         let finish = () => { this._harness.completeTest(); };
118
119         this.runTestCases()
120             .then(finish)
121             .catch(finish);
122     }
123
124     runTestCases()
125     {
126         if (!this.testcases.length)
127             throw new Error("Tried to call runTestCases() for suite with no test cases");
128         if (this._startedRunning)
129             throw new Error("Tried to call runTestCases() more than once.");
130
131         this._startedRunning = true;
132
133         this._harness.log("");
134         this._harness.log(`== Running test suite: ${this.name}`);
135
136         // Avoid adding newlines if nothing was logged.
137         let priorLogCount = this._harness.logCount;
138         let result = this.testcases.reduce((chain, testcase, i) => {
139             if (testcase.setup) {
140                 chain = chain.then(() => {
141                     this._harness.log("-- Running test setup.");
142                     return new Promise(testcase.setup);
143                 });
144             }
145
146             chain = chain.then(() => {
147                 if (i > 0 && priorLogCount + 1 < this._harness.logCount)
148                     this._harness.log("");
149
150                 priorLogCount = this._harness.logCount;
151                 this._harness.log(`-- Running test case: ${testcase.name}`);
152                 this.runCount++;
153                 if (testcase.test[Symbol.toStringTag] === "AsyncFunction")
154                     return testcase.test();
155                 return new Promise(testcase.test);
156             });
157
158             if (testcase.teardown) {
159                 chain = chain.then(() => {
160                     this._harness.log("-- Running test teardown.");
161                     return new Promise(testcase.teardown);
162                 });
163             }
164             return chain;
165         }, Promise.resolve());
166
167         return result.catch((e) => {
168             this.failCount++;
169             this.logThrownObject(e);
170
171             throw e; // Reject this promise by re-throwing the error.
172         });
173     }
174 };
175
176 SyncTestSuite = class SyncTestSuite extends TestSuite
177 {
178     addTestCase(testcase)
179     {
180         if ([testcase.setup, testcase.teardown, testcase.test].some((fn) => fn && fn[Symbol.toStringTag] === "AsyncFunction"))
181             throw new Error("Tried to pass a test case with an async `setup`, `test`, or `teardown` function, but this is a synchronous test suite.")
182
183         super.addTestCase(testcase);
184     }
185
186     runTestCasesAndFinish()
187     {
188         this.runTestCases();
189         this._harness.completeTest();
190     }
191
192     runTestCases()
193     {
194         if (!this.testcases.length)
195             throw new Error("Tried to call runTestCases() for suite with no test cases");
196         if (this._startedRunning)
197             throw new Error("Tried to call runTestCases() more than once.");
198
199         this._startedRunning = true;
200
201         this._harness.log("");
202         this._harness.log(`== Running test suite: ${this.name}`);
203
204         let priorLogCount = this._harness.logCount;
205         for (let i = 0; i < this.testcases.length; i++) {
206             let testcase = this.testcases[i];
207             if (i > 0 && priorLogCount + 1 < this._harness.logCount)
208                 this._harness.log("");
209
210             priorLogCount = this._harness.logCount;
211
212             // Run the setup action, if one was provided.
213             if (testcase.setup) {
214                 this._harness.log("-- Running test setup.");
215                 try {
216                     let result = testcase.setup.call(null);
217                     if (result === false) {
218                         this._harness.log("!! SETUP FAILED");
219                         return false;
220                     }
221                 } catch (e) {
222                     this.logThrownObject(e);
223                     return false;
224                 }
225             }
226
227             this._harness.log("-- Running test case: " + testcase.name);
228             this.runCount++;
229             try {
230                 let result = testcase.test.call(null);
231                 if (result === false) {
232                     this.failCount++;
233                     return false;
234                 }
235             } catch (e) {
236                 this.failCount++;
237                 this.logThrownObject(e);
238                 return false;
239             }
240
241             // Run the teardown action, if one was provided.
242             if (testcase.teardown) {
243                 this._harness.log("-- Running test teardown.");
244                 try {
245                     let result = testcase.teardown.call(null);
246                     if (result === false) {
247                         this._harness.log("!! TEARDOWN FAILED");
248                         return false;
249                     }
250                 } catch (e) {
251                     this.logThrownObject(e);
252                     return false;
253                 }
254             }
255         }
256
257         return true;
258     }
259 };