stress/check-string-ident.js is improperly skipped
[WebKit-https.git] / PerformanceTests / resources / runner.js
1 // There are tests for computeStatistics() located in LayoutTests/fast/harness/perftests
2
3 if (window.testRunner) {
4     testRunner.waitUntilDone();
5     testRunner.dumpAsText();
6 }
7
8 (function () {
9     var logLines = window.testRunner ? [] : null;
10     var verboseLogging = false;
11     var completedIterations;
12     var callsPerIteration = 1;
13     var currentTest = null;
14     var results;
15     var jsHeapResults;
16     var mallocHeapResults;
17     var iterationCount = undefined;
18     var lastResponsivenessTimestamp = 0;
19     var _longestResponsivenessDelay = 0;
20     var continueCheckingResponsiveness = false;
21
22     var PerfTestRunner = {};
23
24     // To make the benchmark results predictable, we replace Math.random with a
25     // 100% deterministic alternative.
26     PerfTestRunner.randomSeed = PerfTestRunner.initialRandomSeed = 49734321;
27
28     PerfTestRunner.resetRandomSeed = function() {
29         PerfTestRunner.randomSeed = PerfTestRunner.initialRandomSeed
30     }
31
32     PerfTestRunner.random = Math.random = function() {
33         // Robert Jenkins' 32 bit integer hash function.
34         var randomSeed = PerfTestRunner.randomSeed;
35         randomSeed = ((randomSeed + 0x7ed55d16) + (randomSeed << 12))  & 0xffffffff;
36         randomSeed = ((randomSeed ^ 0xc761c23c) ^ (randomSeed >>> 19)) & 0xffffffff;
37         randomSeed = ((randomSeed + 0x165667b1) + (randomSeed << 5))   & 0xffffffff;
38         randomSeed = ((randomSeed + 0xd3a2646c) ^ (randomSeed << 9))   & 0xffffffff;
39         randomSeed = ((randomSeed + 0xfd7046c5) + (randomSeed << 3))   & 0xffffffff;
40         randomSeed = ((randomSeed ^ 0xb55a4f09) ^ (randomSeed >>> 16)) & 0xffffffff;
41         PerfTestRunner.randomSeed = randomSeed;
42         return (randomSeed & 0xfffffff) / 0x10000000;
43     };
44
45     PerfTestRunner.now = window.performance && window.performance.now ? function () { return window.performance.now(); } : Date.now;
46
47     PerfTestRunner.logInfo = function (text) {
48         if (verboseLogging)
49             this.log(text);
50     }
51
52     PerfTestRunner.logDetail = function (label, value) {
53         if (verboseLogging)
54             this.log('    ' + label + ': ' + value);
55     }
56
57     PerfTestRunner.loadFile = function (path) {
58         var xhr = new XMLHttpRequest();
59         xhr.open("GET", path, false);
60         xhr.send(null);
61         return xhr.responseText;
62     }
63
64     PerfTestRunner.computeStatistics = function (times, unit) {
65         var data = times.slice();
66
67         // Add values from the smallest to the largest to avoid the loss of significance
68         data.sort(function(a,b){return a-b;});
69
70         var middle = Math.floor(data.length / 2);
71         var result = {
72             min: data[0],
73             max: data[data.length - 1],
74             median: data.length % 2 ? data[middle] : (data[middle - 1] + data[middle]) / 2,
75         };
76
77         // Compute the mean and variance using Knuth's online algorithm (has good numerical stability).
78         var squareSum = 0;
79         result.values = times;
80         result.mean = 0;
81         for (var i = 0; i < data.length; ++i) {
82             var x = data[i];
83             var delta = x - result.mean;
84             var sweep = i + 1.0;
85             result.mean += delta / sweep;
86             squareSum += delta * (x - result.mean);
87         }
88         result.variance = data.length <= 1 ? 0 : squareSum / (data.length - 1);
89         result.stdev = Math.sqrt(result.variance);
90         result.unit = unit || "ms";
91
92         return result;
93     }
94
95     PerfTestRunner.logStatistics = function (values, unit, title) {
96         var statistics = this.computeStatistics(values, unit);
97         this.log("");
98         this.log(title + " -> [" + statistics.values.join(", ") + "] " + statistics.unit);
99         ["mean", "median", "stdev", "min", "max"].forEach(function (name) {
100             PerfTestRunner.logDetail(name, statistics[name] + ' ' + statistics.unit);
101         });
102     }
103
104     function getUsedMallocHeap() {
105         var stats = window.internals.mallocStatistics();
106         return stats.committedVMBytes - stats.freeListBytes;
107     }
108
109     function getUsedJSHeap() {
110         return window.internals.memoryInfo().usedJSHeapSize;
111     }
112
113     PerfTestRunner.gc = function () {
114         if (window.GCController)
115             window.GCController.collect();
116         else {
117             function gcRec(n) {
118                 if (n < 1)
119                     return {};
120                 var temp = {i: "ab" + i + (i / 100000)};
121                 temp += "foo";
122                 gcRec(n-1);
123             }
124             for (var i = 0; i < 1000; i++)
125                 gcRec(10);
126         }
127     };
128
129     function logInDocument(text) {
130         if (!document.getElementById("log")) {
131             var pre = document.createElement("pre");
132             pre.id = "log";
133             document.body.appendChild(pre);
134         }
135         document.getElementById("log").innerHTML += text + "\n";
136         window.scrollTo(0, document.body.height);
137     }
138
139     PerfTestRunner.log = function (text) {
140         if (logLines)
141             logLines.push(text);
142         else
143             logInDocument(text);
144     }
145
146     function logFatalError(text) {
147         PerfTestRunner.log(text);
148         finish();
149     }
150
151     function start(test, runner, doNotLogStart) {
152         if (!test) {
153             logFatalError("Got a bad test object.");
154             return;
155         }
156         currentTest = test;
157         // FIXME: We should be using multiple instances of test runner on Dromaeo as well but it's too slow now.
158         // FIXME: Don't hard code the number of in-process iterations to use inside a test runner.
159         iterationCount = test.customIterationCount || (window.testRunner ? 5 : 20);
160         completedIterations = -1;
161         results = [];
162         jsHeapResults = [];
163         mallocHeapResults = [];
164         verboseLogging = !window.testRunner;
165         if (!doNotLogStart) {
166             PerfTestRunner.logInfo('');
167             PerfTestRunner.logInfo("Running " + iterationCount + " times");
168         }
169         if (test.doNotIgnoreInitialRun)
170             completedIterations++;
171         if (runner)
172             scheduleNextRun(runner);
173     }
174
175     function scheduleNextRun(runner) {
176         PerfTestRunner.gc();
177         window.setTimeout(function () {
178             try {
179                 if (currentTest.setup)
180                     currentTest.setup();
181
182                 var measuredValue = runner();
183             } catch (exception) {
184                 logFatalError("Got an exception while running test.run with name=" + exception.name + ", message=" + exception.message);
185                 return;
186             }
187
188             completedIterations++;
189
190             try {
191                 ignoreWarmUpAndLog(measuredValue);
192             } catch (exception) {
193                 logFatalError("Got an exception while logging the result with name=" + exception.name + ", message=" + exception.message);
194                 return;
195             }
196
197             if (completedIterations < iterationCount)
198                 scheduleNextRun(runner);
199             else
200                 finish();
201         }, 0);
202     }
203
204     function ignoreWarmUpAndLog(measuredValue, doNotLogProgress) {
205         var labeledResult = measuredValue + " " + PerfTestRunner.unit;
206         if (completedIterations <= 0) {
207             if (!doNotLogProgress)
208                 PerfTestRunner.logDetail(completedIterations, labeledResult + " (Ignored warm-up run)");
209             return;
210         }
211
212         results.push(measuredValue);
213         if (window.internals && !currentTest.doNotMeasureMemoryUsage) {
214             jsHeapResults.push(getUsedJSHeap());
215             mallocHeapResults.push(getUsedMallocHeap());
216         }
217         if (!doNotLogProgress)
218             PerfTestRunner.logDetail(completedIterations, labeledResult);
219     }
220
221     function finish() {
222         try {
223             var prefix = currentTest.name || '';
224             if (currentTest.description)
225                 PerfTestRunner.log("Description: " + currentTest.description);
226             metric = {'fps': 'FrameRate', 'runs/s': 'Runs', 'ms': 'Time'}[PerfTestRunner.unit];
227             var suffix = currentTest.aggregator ? ':' + currentTest.aggregator : '';
228             PerfTestRunner.logStatistics(results, PerfTestRunner.unit, prefix + ":" + metric + suffix);
229             if (jsHeapResults.length) {
230                 PerfTestRunner.logStatistics(jsHeapResults, "bytes", prefix + ":JSHeap");
231                 PerfTestRunner.logStatistics(mallocHeapResults, "bytes", prefix + ":Malloc");
232             }
233             if (logLines && !currentTest.continueTesting)
234                 logLines.forEach(logInDocument);
235             if (currentTest.done)
236                 currentTest.done();
237         } catch (exception) {
238             logInDocument("Got an exception while finalizing the test with name=" + exception.name + ", message=" + exception.message);
239         }
240
241         if (!currentTest.continueTesting) {
242             if (window.testRunner)
243                 testRunner.notifyDone();
244             return;
245         }
246
247         currentTest = null;
248     }
249
250     PerfTestRunner.prepareToMeasureValuesAsync = function (test) {
251         PerfTestRunner.unit = test.unit;
252         start(test);
253     }
254
255     PerfTestRunner.measureValueAsync = function (measuredValue) {
256         completedIterations++;
257
258         try {
259             ignoreWarmUpAndLog(measuredValue);
260         } catch (exception) {
261             logFatalError("Got an exception while logging the result with name=" + exception.name + ", message=" + exception.message);
262             return false;
263         }
264
265         if (completedIterations >= iterationCount) {
266             finish();
267             return false;
268         }
269
270         return true;
271     }
272
273     PerfTestRunner.reportValues = function (test, values) {
274         PerfTestRunner.unit = test.unit;
275         start(test, null, true);
276         for (var i = 0; i < values.length; i++) {
277             completedIterations++;
278             ignoreWarmUpAndLog(values[i], true);
279         }
280         finish();
281     }
282
283     PerfTestRunner.measureTime = function (test) {
284         PerfTestRunner.unit = "ms";
285         start(test, measureTimeOnce);
286     }
287
288     function measureTimeOnce() {
289         var start = PerfTestRunner.now();
290         var returnValue = currentTest.run();
291         var end = PerfTestRunner.now();
292
293         if (returnValue - 0 === returnValue) {
294             if (returnValue < 0)
295                 PerfTestRunner.log("runFunction returned a negative value: " + returnValue);
296             return returnValue;
297         }
298
299         return end - start;
300     }
301
302     PerfTestRunner.measureRunsPerSecond = function (test) {
303         PerfTestRunner.unit = "runs/s";
304         start(test, measureRunsPerSecondOnce);
305     }
306
307     function measureRunsPerSecondOnce() {
308         var timeToRun = 750;
309         var totalTime = 0;
310         var numberOfRuns = 0;
311
312         while (totalTime < timeToRun) {
313             totalTime += callRunAndMeasureTime(callsPerIteration);
314             numberOfRuns += callsPerIteration;
315             if (completedIterations < 0 && totalTime < 100)
316                 callsPerIteration = Math.max(10, 2 * callsPerIteration);
317         }
318
319         return numberOfRuns * 1000 / totalTime;
320     }
321
322     function callRunAndMeasureTime(callsPerIteration) {
323         var startTime = PerfTestRunner.now();
324         for (var i = 0; i < callsPerIteration; i++)
325             currentTest.run();
326         return PerfTestRunner.now() - startTime;
327     }
328
329     PerfTestRunner.startCheckingResponsiveness = function() {
330         lastResponsivenessTimestamp = PerfTestRunner.now();
331         _longestResponsivenessDelay = 0;
332         continueCheckingResponsiveness = true;
333
334         var timeoutFunction = function() {
335             var now = PerfTestRunner.now();
336             var delta = now - lastResponsivenessTimestamp;
337             if (delta > _longestResponsivenessDelay)
338                 _longestResponsivenessDelay = delta;    
339
340             lastResponsivenessTimestamp = now;
341             if (continueCheckingResponsiveness)
342                 setTimeout(timeoutFunction, 0);
343         }
344         
345         timeoutFunction();
346     }
347
348     PerfTestRunner.stopCheckingResponsiveness = function() {
349         continueCheckingResponsiveness = false;
350     }
351
352     PerfTestRunner.longestResponsivenessDelay = function() {
353         return _longestResponsivenessDelay;
354     }
355
356     PerfTestRunner.measurePageLoadTime = function(test) {
357         test.run = function() {
358             var file = PerfTestRunner.loadFile(test.path);
359             if (!test.chunkSize)
360                 this.chunkSize = 50000;
361
362             var chunks = [];
363             // The smaller the chunks the more style resolves we do.
364             // Smaller chunk sizes will show more samples in style resolution.
365             // Larger chunk sizes will show more samples in line layout.
366             // Smaller chunk sizes run slower overall, as the per-chunk overhead is high.
367             var chunkCount = Math.ceil(file.length / this.chunkSize);
368             for (var chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) {
369                 var chunk = file.substr(chunkIndex * this.chunkSize, this.chunkSize);
370                 chunks.push(chunk);
371             }
372
373             PerfTestRunner.logInfo("Testing " + file.length + " byte document in " + chunkCount + " " + this.chunkSize + " byte chunks.");
374
375             var iframe = document.createElement("iframe");
376             document.body.appendChild(iframe);
377
378             iframe.sandbox = '';  // Prevent external loads which could cause write() to return before completing the parse.
379             iframe.style.width = "600px"; // Have a reasonable size so we're not line-breaking on every character.
380             iframe.style.height = "800px";
381             iframe.contentDocument.open();
382
383             for (var chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
384                 iframe.contentDocument.write(chunks[chunkIndex]);
385                 // Note that we won't cause a style resolve until we've encountered the <body> element.
386                 // Thus the number of chunks counted above is not exactly equal to the number of style resolves.
387                 if (iframe.contentDocument.body)
388                     iframe.contentDocument.body.clientHeight; // Force a full layout/style-resolve.
389                 else if (iframe.documentElement.localName == 'html')
390                     iframe.contentDocument.documentElement.offsetWidth; // Force the painting.
391             }
392
393             iframe.contentDocument.close();
394             document.body.removeChild(iframe);
395         };
396
397         PerfTestRunner.measureTime(test);
398     }
399
400     window.PerfTestRunner = PerfTestRunner;
401 })();