278c176ad373d483dffb2dd38002c31ba159557b
[WebKit-https.git] / PerformanceTests / resources / runner.js
1 // There are tests for computeStatistics() located in LayoutTests/fast/harness/perftests
2
3 // We need access to console.memory for the memory measurements
4 if (window.internals)
5     internals.settings.setMemoryInfoEnabled(true);
6
7 if (window.testRunner) {
8     testRunner.waitUntilDone();
9     testRunner.dumpAsText();
10 }
11
12 (function () {
13     var logLines = null;
14     var completedRuns = -1;
15     var callsPerIteration = 1;
16     var currentTest = null;
17     var results = [];
18     var jsHeapResults = [];
19     var mallocHeapResults = [];
20     var runCount = undefined;
21
22     var PerfTestRunner = {};
23
24     // To make the benchmark results predictable, we replace Math.random with a
25     // 100% deterministic alternative.
26     PerfTestRunner.randomSeed = PerfTestRunner.initialRandomSeed = 49734321;
27
28     PerfTestRunner.resetRandomSeed = function() {
29         PerfTestRunner.randomSeed = PerfTestRunner.initialRandomSeed
30     }
31
32     PerfTestRunner.random = Math.random = function() {
33         // Robert Jenkins' 32 bit integer hash function.
34         var randomSeed = PerfTestRunner.randomSeed;
35         randomSeed = ((randomSeed + 0x7ed55d16) + (randomSeed << 12))  & 0xffffffff;
36         randomSeed = ((randomSeed ^ 0xc761c23c) ^ (randomSeed >>> 19)) & 0xffffffff;
37         randomSeed = ((randomSeed + 0x165667b1) + (randomSeed << 5))   & 0xffffffff;
38         randomSeed = ((randomSeed + 0xd3a2646c) ^ (randomSeed << 9))   & 0xffffffff;
39         randomSeed = ((randomSeed + 0xfd7046c5) + (randomSeed << 3))   & 0xffffffff;
40         randomSeed = ((randomSeed ^ 0xb55a4f09) ^ (randomSeed >>> 16)) & 0xffffffff;
41         PerfTestRunner.randomSeed = randomSeed;
42         return (randomSeed & 0xfffffff) / 0x10000000;
43     };
44
45     PerfTestRunner.now = window.performance && window.performance.now ? function () { return window.performance.now(); } : Date.now;
46
47     PerfTestRunner.logInfo = function (text) {
48         if (!window.testRunner)
49             this.log(text);
50     }
51
52     PerfTestRunner.loadFile = function (path) {
53         var xhr = new XMLHttpRequest();
54         xhr.open("GET", path, false);
55         xhr.send(null);
56         return xhr.responseText;
57     }
58
59     PerfTestRunner.computeStatistics = function (times, unit) {
60         var data = times.slice();
61
62         // Add values from the smallest to the largest to avoid the loss of significance
63         data.sort(function(a,b){return a-b;});
64
65         var middle = Math.floor(data.length / 2);
66         var result = {
67             min: data[0],
68             max: data[data.length - 1],
69             median: data.length % 2 ? data[middle] : (data[middle - 1] + data[middle]) / 2,
70         };
71
72         // Compute the mean and variance using Knuth's online algorithm (has good numerical stability).
73         var squareSum = 0;
74         result.values = times;
75         result.mean = 0;
76         for (var i = 0; i < data.length; ++i) {
77             var x = data[i];
78             var delta = x - result.mean;
79             var sweep = i + 1.0;
80             result.mean += delta / sweep;
81             squareSum += delta * (x - result.mean);
82         }
83         result.variance = data.length <= 1 ? 0 : squareSum / (data.length - 1);
84         result.stdev = Math.sqrt(result.variance);
85         result.unit = unit || "ms";
86
87         return result;
88     }
89
90     PerfTestRunner.logStatistics = function (values, unit, title) {
91         var statistics = this.computeStatistics(values, unit);
92         this.printStatistics(statistics, title);
93     }
94
95     PerfTestRunner.printStatistics = function (statistics, title) {
96         this.log("");
97         this.log(title);
98         if (statistics.values)
99             this.log("values " + statistics.values.join(", ") + " " + statistics.unit);
100         this.log("avg " + statistics.mean + " " + statistics.unit);
101         this.log("median " + statistics.median + " " + statistics.unit);
102         this.log("stdev " + statistics.stdev + " " + statistics.unit);
103         this.log("min " + statistics.min + " " + statistics.unit);
104         this.log("max " + statistics.max + " " + statistics.unit);
105     }
106
107     PerfTestRunner.getUsedMallocHeap = function() {
108         var stats = window.internals.mallocStatistics();
109         return stats.committedVMBytes - stats.freeListBytes;
110     }
111
112     PerfTestRunner.getUsedJSHeap = function() {
113         return console.memory.usedJSHeapSize;
114     }
115
116     PerfTestRunner.getAndPrintMemoryStatistics = function() {
117         if (!window.internals)
118             return;
119         var jsMemoryStats = PerfTestRunner.computeStatistics([PerfTestRunner.getUsedJSHeap()], "bytes");
120         PerfTestRunner.printStatistics(jsMemoryStats, "JS Heap:");
121
122         var mallocMemoryStats = PerfTestRunner.computeStatistics([PerfTestRunner.getUsedMallocHeap()], "bytes");
123         PerfTestRunner.printStatistics(mallocMemoryStats, "Malloc:");
124     }
125
126     PerfTestRunner.gc = function () {
127         if (window.GCController)
128             window.GCController.collect();
129         else {
130             function gcRec(n) {
131                 if (n < 1)
132                     return {};
133                 var temp = {i: "ab" + i + (i / 100000)};
134                 temp += "foo";
135                 gcRec(n-1);
136             }
137             for (var i = 0; i < 1000; i++)
138                 gcRec(10);
139         }
140     };
141
142     function logInDocument(text) {
143         if (!document.getElementById("log")) {
144             var pre = document.createElement("pre");
145             pre.id = "log";
146             document.body.appendChild(pre);
147         }
148         document.getElementById("log").innerHTML += text + "\n";
149         window.scrollTo(0, document.body.height);
150     }
151
152     PerfTestRunner.log = function (text) {
153         if (logLines)
154             logLines.push(text);
155         else
156             logInDocument(text);
157     }
158
159     function logFatalError(text) {
160         PerfTestRunner.log(text);
161         finish();
162     }
163
164     function start(test, runner) {
165         if (!test) {
166             logFatalError("Got a bad test object.");
167             return;
168         }
169         currentTest = test;
170         runCount = test.runCount || 20;
171         logLines = window.testRunner ? [] : null;
172         PerfTestRunner.log("Running " + runCount + " times");
173         if (runner)
174             scheduleNextRun(runner);
175     }
176
177     function scheduleNextRun(runner) {
178         PerfTestRunner.gc();
179         window.setTimeout(function () {
180             try {
181                 var measuredValue = runner();
182             } catch (exception) {
183                 logFatalError("Got an exception while running test.run with name=" + exception.name + ", message=" + exception.message);
184                 return;
185             }
186
187             completedRuns++;
188
189             try {
190                 ignoreWarmUpAndLog(measuredValue);
191             } catch (exception) {
192                 logFatalError("Got an exception while logging the result with name=" + exception.name + ", message=" + exception.message);
193                 return;
194             }
195
196             if (completedRuns < runCount)
197                 scheduleNextRun(runner);
198             else
199                 finish();
200         }, 0);
201     }
202
203     function ignoreWarmUpAndLog(measuredValue) {
204         var labeledResult = measuredValue + " " + PerfTestRunner.unit;
205         if (completedRuns <= 0)
206             PerfTestRunner.log("Ignoring warm-up run (" + labeledResult + ")");
207         else {
208             results.push(measuredValue);
209             if (window.internals) {
210                 jsHeapResults.push(PerfTestRunner.getUsedJSHeap());
211                 mallocHeapResults.push(PerfTestRunner.getUsedMallocHeap());
212             }
213             PerfTestRunner.log(labeledResult);
214         }
215     }
216
217     function finish() {
218         try {
219             if (currentTest.description)
220                 PerfTestRunner.log("Description: " + currentTest.description);
221             PerfTestRunner.logStatistics(results, PerfTestRunner.unit, "Time:");
222             if (jsHeapResults.length) {
223                 PerfTestRunner.logStatistics(jsHeapResults, "bytes", "JS Heap:");
224                 PerfTestRunner.logStatistics(mallocHeapResults, "bytes", "Malloc:");
225             }
226             if (logLines)
227                 logLines.forEach(logInDocument);
228             if (currentTest.done)
229                 currentTest.done();
230         } catch (exception) {
231             logInDocument("Got an exception while finalizing the test with name=" + exception.name + ", message=" + exception.message);
232         }
233
234         if (window.testRunner)
235             testRunner.notifyDone();
236     }
237
238     PerfTestRunner.prepareToMeasureValuesAsync = function (test) {
239         PerfTestRunner.unit = test.unit;
240         start(test);
241     }
242
243     PerfTestRunner.measureValueAync = function (measuredValue) {
244         completedRuns++;
245
246         try {
247             ignoreWarmUpAndLog(measuredValue);
248         } catch (exception) {
249             logFatalError("Got an exception while logging the result with name=" + exception.name + ", message=" + exception.message);
250             return;
251         }
252
253         if (completedRuns >= runCount)
254             finish();
255     }
256
257     PerfTestRunner.measureTime = function (test) {
258         PerfTestRunner.unit = "ms";
259         start(test, measureTimeOnce);
260     }
261
262     function measureTimeOnce() {
263         var start = PerfTestRunner.now();
264         var returnValue = currentTest.run();
265         var end = PerfTestRunner.now();
266
267         if (returnValue - 0 === returnValue) {
268             if (returnValue <= 0)
269                 PerfTestRunner.log("runFunction returned a non-positive value: " + returnValue);
270             return returnValue;
271         }
272
273         return end - start;
274     }
275
276     PerfTestRunner.measureRunsPerSecond = function (test) {
277         PerfTestRunner.unit = "runs/s";
278         start(test, measureRunsPerSecondOnce);
279     }
280
281     function measureRunsPerSecondOnce() {
282         var timeToRun = 750;
283         var totalTime = 0;
284         var numberOfRuns = 0;
285
286         if (currentTest.setup)
287             currentTest.setup();
288
289         while (totalTime < timeToRun) {
290             totalTime += callRunAndMeasureTime(callsPerIteration);
291             numberOfRuns += callsPerIteration;
292             if (completedRuns < 0 && totalTime < 100)
293                 callsPerIteration = Math.max(10, 2 * callsPerIteration);
294         }
295
296         return numberOfRuns * 1000 / totalTime;
297     }
298
299     function callRunAndMeasureTime(callsPerIteration) {
300         var startTime = PerfTestRunner.now();
301         for (var i = 0; i < callsPerIteration; i++)
302             currentTest.run();
303         return PerfTestRunner.now() - startTime;
304     }
305
306
307     PerfTestRunner.measurePageLoadTime = function(test) {
308         test.run = function() {
309             var file = PerfTestRunner.loadFile(test.path);
310             if (!test.chunkSize)
311                 this.chunkSize = 50000;
312
313             var chunks = [];
314             // The smaller the chunks the more style resolves we do.
315             // Smaller chunk sizes will show more samples in style resolution.
316             // Larger chunk sizes will show more samples in line layout.
317             // Smaller chunk sizes run slower overall, as the per-chunk overhead is high.
318             var chunkCount = Math.ceil(file.length / this.chunkSize);
319             for (var chunkIndex = 0; chunkIndex < chunkCount; chunkIndex++) {
320                 var chunk = file.substr(chunkIndex * this.chunkSize, this.chunkSize);
321                 chunks.push(chunk);
322             }
323
324             PerfTestRunner.logInfo("Testing " + file.length + " byte document in " + chunkCount + " " + this.chunkSize + " byte chunks.");
325
326             var iframe = document.createElement("iframe");
327             document.body.appendChild(iframe);
328
329             iframe.sandbox = '';  // Prevent external loads which could cause write() to return before completing the parse.
330             iframe.style.width = "600px"; // Have a reasonable size so we're not line-breaking on every character.
331             iframe.style.height = "800px";
332             iframe.contentDocument.open();
333
334             for (var chunkIndex = 0; chunkIndex < chunks.length; chunkIndex++) {
335                 iframe.contentDocument.write(chunks[chunkIndex]);
336                 // Note that we won't cause a style resolve until we've encountered the <body> element.
337                 // Thus the number of chunks counted above is not exactly equal to the number of style resolves.
338                 if (iframe.contentDocument.body)
339                     iframe.contentDocument.body.clientHeight; // Force a full layout/style-resolve.
340                 else if (iframe.documentElement.localName == 'html')
341                     iframe.contentDocument.documentElement.offsetWidth; // Force the painting.
342             }
343
344             iframe.contentDocument.close();
345             document.body.removeChild(iframe);
346         };
347
348         PerfTestRunner.measureTime(test);
349     }
350
351     window.PerfTestRunner = PerfTestRunner;
352 })();