d62602f670ba1bdb0e279d71108b245fbb230ddb
[WebKit-https.git] / Source / WebInspectorUI / UserInterface / Test / TestSuite.js
1 /*
2  * Copyright (C) 2015 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  *
8  * 1.  Redistributions of source code must retain the above copyright
9  *     notice, this list of conditions and the following disclaimer.
10  * 2.  Redistributions in binary form must reproduce the above copyright
11  *     notice, this list of conditions and the following disclaimer in the
12  *     documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
15  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
16  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
17  * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
18  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
19  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
20  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
21  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 TestSuite = class TestSuite
27 {
28     constructor(harness, name) {
29         if (!(harness instanceof TestHarness))
30             throw new Error("Must pass the test's harness as the first argument.");
31
32         if (typeof name !== "string" || !name.trim().length)
33             throw new Error("Tried to create TestSuite without string suite name.");
34
35         this.name = name;
36         this._harness = harness;
37
38         this.testcases = [];
39         this.runCount = 0;
40         this.failCount = 0;
41     }
42
43     // Use this if the test file only has one suite, and no handling
44     // of the value returned by runTestCases() is needed.
45     runTestCasesAndFinish()
46     {
47         throw new Error("Must be implemented by subclasses.");
48     }
49
50     runTestCases()
51     {
52         throw new Error("Must be implemented by subclasses.");
53     }
54
55     get passCount()
56     {
57         return this.runCount - this.failCount;
58     }
59
60     get skipCount()
61     {
62         if (this.failCount)
63             return this.testcases.length - this.runCount;
64         else
65             return 0;
66     }
67
68     addTestCase(testcase)
69     {
70         if (!testcase || !(testcase instanceof Object))
71             throw new Error("Tried to add non-object test case.");
72
73         if (typeof testcase.name !== "string" || !testcase.name.trim().length)
74             throw new Error("Tried to add test case without a name.");
75
76         if (typeof testcase.test !== "function")
77             throw new Error("Tried to add test case without `test` function.");
78
79         if (testcase.setup && typeof testcase.setup !== "function")
80             throw new Error("Tried to add test case with invalid `setup` parameter (must be a function).");
81
82         if (testcase.teardown && typeof testcase.teardown !== "function")
83             throw new Error("Tried to add test case with invalid `teardown` parameter (must be a function).");
84
85         this.testcases.push(testcase);
86     }
87
88     // Protected
89
90     logThrownObject(e)
91     {
92         let message = e;
93         let stack = "(unknown)";
94         if (e instanceof Error) {
95             message = e.message;
96             if (e.stack)
97                 stack = e.stack;
98         }
99
100         if (typeof message !== "string")
101             message = JSON.stringify(message);
102
103         let sanitizedStack = this._harness.sanitizeStack(stack);
104
105         let result = `!! EXCEPTION: ${message}`;
106         if (stack)
107             result += `\nStack Trace: ${sanitizedStack}`;
108
109         this._harness.log(result);
110     }
111 };
112
113 AsyncTestSuite = class AsyncTestSuite extends TestSuite
114 {
115     runTestCasesAndFinish()
116     {
117         let finish = () => { this._harness.completeTest(); };
118
119         this.runTestCases()
120             .then(finish)
121             .catch(finish);
122     }
123
124     runTestCases()
125     {
126         if (!this.testcases.length)
127             throw new Error("Tried to call runTestCases() for suite with no test cases");
128         if (this._startedRunning)
129             throw new Error("Tried to call runTestCases() more than once.");
130
131         this._startedRunning = true;
132
133         this._harness.log("");
134         this._harness.log(`== Running test suite: ${this.name}`);
135
136         // Avoid adding newlines if nothing was logged.
137         let priorLogCount = this._harness.logCount;
138         let result = this.testcases.reduce((chain, testcase, i) => {
139             if (testcase.setup) {
140                 chain = chain.then(() => {
141                     this._harness.log("-- Running test setup.");
142                     return new Promise(testcase.setup);
143                 });
144             }
145
146             chain = chain.then(() => {
147                 if (i > 0 && priorLogCount + 1 < this._harness.logCount)
148                     this._harness.log("");
149
150                 priorLogCount = this._harness.logCount;
151                 this._harness.log(`-- Running test case: ${testcase.name}`);
152                 this.runCount++;
153                 if (testcase.test[Symbol.toStringTag] === "AsyncFunction")
154                     return testcase.test();
155                 return new Promise(testcase.test);
156             });
157
158             if (testcase.teardown) {
159                 chain = chain.then(() => {
160                     this._harness.log("-- Running test teardown.");
161                     return new Promise(testcase.teardown);
162                 });
163             }
164             return chain;
165         }, Promise.resolve());
166
167         return result.catch((e) => {
168             this.failCount++;
169             this.logThrownObject(e);
170
171             throw e; // Reject this promise by re-throwing the error.
172         });
173     }
174 };
175
176 SyncTestSuite = class SyncTestSuite extends TestSuite
177 {
178     runTestCasesAndFinish()
179     {
180         this.runTestCases();
181         this._harness.completeTest();
182     }
183
184     runTestCases()
185     {
186         if (!this.testcases.length)
187             throw new Error("Tried to call runTestCases() for suite with no test cases");
188         if (this._startedRunning)
189             throw new Error("Tried to call runTestCases() more than once.");
190
191         this._startedRunning = true;
192
193         this._harness.log("");
194         this._harness.log(`== Running test suite: ${this.name}`);
195
196         let priorLogCount = this._harness.logCount;
197         for (let i = 0; i < this.testcases.length; i++) {
198             let testcase = this.testcases[i];
199             if (i > 0 && priorLogCount + 1 < this._harness.logCount)
200                 this._harness.log("");
201
202             priorLogCount = this._harness.logCount;
203
204             // Run the setup action, if one was provided.
205             if (testcase.setup) {
206                 this._harness.log("-- Running test setup.");
207                 try {
208                     let result = testcase.setup.call(null);
209                     if (result === false) {
210                         this._harness.log("!! SETUP FAILED");
211                         return false;
212                     }
213                 } catch (e) {
214                     this.logThrownObject(e);
215                     return false;
216                 }
217             }
218
219             this._harness.log("-- Running test case: " + testcase.name);
220             this.runCount++;
221             try {
222                 let result = testcase.test.call(null);
223                 if (result === false) {
224                     this.failCount++;
225                     return false;
226                 }
227             } catch (e) {
228                 this.failCount++;
229                 this.logThrownObject(e);
230                 return false;
231             }
232
233             // Run the teardown action, if one was provided.
234             if (testcase.teardown) {
235                 this._harness.log("-- Running test teardown.");
236                 try {
237                     let result = testcase.teardown.call(null);
238                     if (result === false) {
239                         this._harness.log("!! TEARDOWN FAILED");
240                         return false;
241                     }
242                 } catch (e) {
243                     this.logThrownObject(e);
244                     return false;
245                 }
246             }
247         }
248
249         return true;
250     }
251 };