7ad586cb30980dc1cf924b3699809fc324326338
[WebKit-https.git] / Source / WebInspectorUI / UserInterface / Test / TestHarness.js
1 /*
2  * Copyright (C) 2015, 2016 Apple Inc. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  *
8  * 1.  Redistributions of source code must retain the above copyright
9  *     notice, this list of conditions and the following disclaimer.
10  * 2.  Redistributions in binary form must reproduce the above copyright
11  *     notice, this list of conditions and the following disclaimer in the
12  *     documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
15  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
16  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
17  * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
18  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
19  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
20  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
21  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25
26 TestHarness = class TestHarness extends WI.Object
27 {
28     constructor()
29     {
30         super();
31
32         this._logCount = 0;
33         this._failureObjects = new Map;
34         this._failureObjectIdentifier = 1;
35
36         // Options that are set per-test for debugging purposes.
37         this.forceDebugLogging = false;
38
39         // Options that are set per-test to ensure deterministic output.
40         this.suppressStackTraces = false;
41     }
42
43     completeTest()
44     {
45         throw new Error("Must be implemented by subclasses.");
46     }
47
48     addResult()
49     {
50         throw new Error("Must be implemented by subclasses.");
51     }
52
53     debugLog()
54     {
55         throw new Error("Must be implemented by subclasses.");
56     }
57
58     evaluateInPage(string, callback)
59     {
60         throw new Error("Must be implemented by subclasses.");
61     }
62
63     debug()
64     {
65         throw new Error("Must be implemented by subclasses.");
66     }
67
68     createAsyncSuite(name)
69     {
70         return new AsyncTestSuite(this, name);
71     }
72
73     createSyncSuite(name)
74     {
75         return new SyncTestSuite(this, name);
76     }
77
78     get logCount()
79     {
80         return this._logCount;
81     }
82
83     log(message)
84     {
85         ++this._logCount;
86
87         if (this.forceDebugLogging)
88             this.debugLog(message);
89         else
90             this.addResult(message);
91     }
92
93     json(object, filter)
94     {
95         this.log(JSON.stringify(object, filter || null, 2));
96     }
97
98     assert(condition, message)
99     {
100         if (condition)
101             return;
102
103         let stringifiedMessage = TestHarness.messageAsString(message);
104         this.log("ASSERT: " + stringifiedMessage);
105     }
106
107     expectThat(actual, message)
108     {
109         this._expect(TestHarness.ExpectationType.True, !!actual, message, actual);
110     }
111
112     expectFalse(actual, message)
113     {
114         this._expect(TestHarness.ExpectationType.False, !actual, message, actual);
115     }
116
117     expectNull(actual, message)
118     {
119         this._expect(TestHarness.ExpectationType.Null, actual === null, message, actual, null);
120     }
121
122     expectNotNull(actual, message)
123     {
124         this._expect(TestHarness.ExpectationType.NotNull, actual !== null, message, actual);
125     }
126
127     expectEqual(actual, expected, message)
128     {
129         this._expect(TestHarness.ExpectationType.Equal, expected === actual, message, actual, expected);
130     }
131
132     expectNotEqual(actual, expected, message)
133     {
134         this._expect(TestHarness.ExpectationType.NotEqual, expected !== actual, message, actual, expected);
135     }
136
137     expectShallowEqual(actual, expected, message)
138     {
139         this._expect(TestHarness.ExpectationType.ShallowEqual, Object.shallowEqual(actual, expected), message, actual, expected);
140     }
141
142     expectNotShallowEqual(actual, expected, message)
143     {
144         this._expect(TestHarness.ExpectationType.NotShallowEqual, !Object.shallowEqual(actual, expected), message, actual, expected);
145     }
146
147     expectEqualWithAccuracy(actual, expected, accuracy, message)
148     {
149         console.assert(typeof expected === "number");
150         console.assert(typeof actual === "number");
151
152         this._expect(TestHarness.ExpectationType.EqualWithAccuracy, Math.abs(expected - actual) <= accuracy, message, actual, expected, accuracy);
153     }
154
155     expectLessThan(actual, expected, message)
156     {
157         this._expect(TestHarness.ExpectationType.LessThan, actual < expected, message, actual, expected);
158     }
159
160     expectLessThanOrEqual(actual, expected, message)
161     {
162         this._expect(TestHarness.ExpectationType.LessThanOrEqual, actual <= expected, message, actual, expected);
163     }
164
165     expectGreaterThan(actual, expected, message)
166     {
167         this._expect(TestHarness.ExpectationType.GreaterThan, actual > expected, message, actual, expected);
168     }
169
170     expectGreaterThanOrEqual(actual, expected, message)
171     {
172         this._expect(TestHarness.ExpectationType.GreaterThanOrEqual, actual >= expected, message, actual, expected);
173     }
174
175     pass(message)
176     {
177         let stringifiedMessage = TestHarness.messageAsString(message);
178         this.log("PASS: " + stringifiedMessage);
179     }
180
181     fail(message)
182     {
183         let stringifiedMessage = TestHarness.messageAsString(message);
184         this.log("FAIL: " + stringifiedMessage);
185     }
186
187     // Protected
188
189     static messageAsString(message)
190     {
191         if (message instanceof Element)
192             return message.textContent;
193
194         return typeof message !== "string" ? JSON.stringify(message) : message;
195     }
196
197     static sanitizeURL(url)
198     {
199         if (!url)
200             return "(unknown)";
201
202         let lastPathSeparator = Math.max(url.lastIndexOf("/"), url.lastIndexOf("\\"));
203         let location = lastPathSeparator > 0 ? url.substr(lastPathSeparator + 1) : url;
204         if (!location.length)
205             location = "(unknown)";
206
207         // Clean up the location so it is bracketed or in parenthesis.
208         if (url.indexOf("[native code]") !== -1)
209             location = "[native code]";
210
211         return location;
212     }
213
214     static sanitizeStackFrame(frame, i)
215     {
216         // Most frames are of the form "functionName@file:///foo/bar/File.js:345".
217         // But, some frames do not have a functionName. Get rid of the file path.
218         let nameAndURLSeparator = frame.indexOf("@");
219         let frameName = nameAndURLSeparator > 0 ? frame.substr(0, nameAndURLSeparator) : "(anonymous)";
220
221         let lastPathSeparator = Math.max(frame.lastIndexOf("/"), frame.lastIndexOf("\\"));
222         let frameLocation = lastPathSeparator > 0 ? frame.substr(lastPathSeparator + 1) : frame;
223         if (!frameLocation.length)
224             frameLocation = "unknown";
225
226         // Clean up the location so it is bracketed or in parenthesis.
227         if (frame.indexOf("[native code]") !== -1)
228             frameLocation = "[native code]";
229         else
230             frameLocation = "(" + frameLocation + ")";
231
232         return `#${i}: ${frameName} ${frameLocation}`;
233     }
234
235     sanitizeStack(stack)
236     {
237         if (this.suppressStackTraces)
238             return "(suppressed)";
239
240         if (!stack || typeof stack !== "string")
241             return "(unknown)";
242
243         return stack.split("\n").map(TestHarness.sanitizeStackFrame).join("\n");
244     }
245
246     // Private
247
248     _expect(type, condition, message, ...values)
249     {
250         console.assert(values.length > 0, "Should have an 'actual' value.");
251
252         if (!message || !condition) {
253             values = values.map(this._expectationValueAsString.bind(this));
254             message = message || this._expectationMessageFormat(type).format(...values);
255         }
256
257         if (condition) {
258             this.pass(message);
259             return;
260         }
261
262         message += "\n    Expected: " + this._expectedValueFormat(type).format(...values.slice(1));
263         message += "\n    Actual: " + values[0];
264
265         this.fail(message);
266     }
267
268     _expectationValueAsString(value)
269     {
270         let instanceIdentifier = (object) => {
271             let id = this._failureObjects.get(object);
272             if (!id) {
273                 id = this._failureObjectIdentifier++;
274                 this._failureObjects.set(object, id);
275             }
276             return "#" + id;
277         };
278
279         const maximumValueStringLength = 200;
280         const defaultValueString = String(new Object); // [object Object]
281
282         // Special case for numbers, since JSON.stringify converts Infinity and NaN to null.
283         if (typeof value === "number")
284             return value;
285
286         try {
287             let valueString = JSON.stringify(value);
288             if (valueString.length <= maximumValueStringLength)
289                 return valueString;
290         } catch { }
291
292         try {
293             let valueString = String(value);
294             if (valueString === defaultValueString && value.constructor && value.constructor.name !== "Object")
295                 return value.constructor.name + " instance " + instanceIdentifier(value);
296             return valueString;
297         } catch {
298             return defaultValueString;
299         }
300     }
301
302     _expectationMessageFormat(type)
303     {
304         switch (type) {
305         case TestHarness.ExpectationType.True:
306             return "expectThat(%s)";
307         case TestHarness.ExpectationType.False:
308             return "expectFalse(%s)";
309         case TestHarness.ExpectationType.Null:
310             return "expectNull(%s)";
311         case TestHarness.ExpectationType.NotNull:
312             return "expectNotNull(%s)";
313         case TestHarness.ExpectationType.Equal:
314             return "expectEqual(%s, %s)";
315         case TestHarness.ExpectationType.NotEqual:
316             return "expectNotEqual(%s, %s)";
317         case TestHarness.ExpectationType.ShallowEqual:
318             return "expectShallowEqual(%s, %s)";
319         case TestHarness.ExpectationType.NotShallowEqual:
320             return "expectNotShallowEqual(%s, %s)";
321         case TestHarness.ExpectationType.EqualWithAccuracy:
322             return "expectEqualWithAccuracy(%s, %s, %s)";
323         case TestHarness.ExpectationType.LessThan:
324             return "expectLessThan(%s, %s)";
325         case TestHarness.ExpectationType.LessThanOrEqual:
326             return "expectLessThanOrEqual(%s, %s)";
327         case TestHarness.ExpectationType.GreaterThan:
328             return "expectGreaterThan(%s, %s)";
329         case TestHarness.ExpectationType.GreaterThanOrEqual:
330             return "expectGreaterThanOrEqual(%s, %s)";
331         default:
332             console.error("Unknown TestHarness.ExpectationType type: " + type);
333             return null;
334         }
335     }
336
337     _expectedValueFormat(type)
338     {
339         switch (type) {
340         case TestHarness.ExpectationType.True:
341             return "truthy";
342         case TestHarness.ExpectationType.False:
343             return "falsey";
344         case TestHarness.ExpectationType.NotNull:
345             return "not null";
346         case TestHarness.ExpectationType.NotEqual:
347         case TestHarness.ExpectationType.NotShallowEqual:
348             return "not %s";
349         case TestHarness.ExpectationType.EqualWithAccuracy:
350             return "%s +/- %s";
351         case TestHarness.ExpectationType.LessThan:
352             return "less than %s";
353         case TestHarness.ExpectationType.LessThanOrEqual:
354             return "less than or equal to %s";
355         case TestHarness.ExpectationType.GreaterThan:
356             return "greater than %s";
357         case TestHarness.ExpectationType.GreaterThanOrEqual:
358             return "greater than or equal to %s";
359         default:
360             return "%s";
361         }
362     }
363 };
364
365 TestHarness.ExpectationType = {
366     True: Symbol("expect-true"),
367     False: Symbol("expect-false"),
368     Null: Symbol("expect-null"),
369     NotNull: Symbol("expect-not-null"),
370     Equal: Symbol("expect-equal"),
371     NotEqual: Symbol("expect-not-equal"),
372     ShallowEqual: Symbol("expect-shallow-equal"),
373     NotShallowEqual: Symbol("expect-not-shallow-equal"),
374     EqualWithAccuracy: Symbol("expect-equal-with-accuracy"),
375     LessThan: Symbol("expect-less-than"),
376     LessThanOrEqual: Symbol("expect-less-than-or-equal"),
377     GreaterThan: Symbol("expect-greater-than"),
378     GreaterThanOrEqual: Symbol("expect-greater-than-or-equal"),
379 };