Add some new controllers, and refine tests
[WebKit-https.git] / PerformanceTests / Animometer / resources / runner / animometer.js
1 ResultsDashboard = Utilities.createClass(
2     function(options, testData)
3     {
4         this._iterationsSamplers = [];
5         this._options = options;
6         this._results = null;
7         if (testData) {
8             this._iterationsSamplers = testData;
9             this._processData();
10         }
11     }, {
12
13     push: function(suitesSamplers)
14     {
15         this._iterationsSamplers.push(suitesSamplers);
16     },
17
18     _processData: function()
19     {
20         this._results = {};
21         this._results[Strings.json.results.iterations] = [];
22
23         var iterationsScores = [];
24         this._iterationsSamplers.forEach(function(iteration, index) {
25             var testsScores = [];
26
27             var result = {};
28             this._results[Strings.json.results.iterations][index] = result;
29
30             var suitesResult = {};
31             result[Strings.json.results.tests] = suitesResult;
32
33             for (var suiteName in iteration) {
34                 var suiteData = iteration[suiteName];
35
36                 var suiteResult = {};
37                 suitesResult[suiteName] = suiteResult;
38
39                 for (var testName in suiteData) {
40                     if (!suiteData[testName][Strings.json.result])
41                         this.calculateScore(suiteData[testName]);
42
43                     suiteResult[testName] = suiteData[testName][Strings.json.result];
44                     delete suiteData[testName][Strings.json.result];
45
46                     testsScores.push(suiteResult[testName][Strings.json.score]);
47                 }
48             }
49
50             result[Strings.json.score] = Statistics.geometricMean(testsScores);
51             iterationsScores.push(result[Strings.json.score]);
52         }, this);
53
54         this._results[Strings.json.score] = Statistics.sampleMean(iterationsScores.length, iterationsScores.reduce(function(a, b) { return a + b; }));
55     },
56
57     calculateScore: function(data)
58     {
59         var result = {};
60         data[Strings.json.result] = result;
61         var samples = data[Strings.json.samples];
62
63         var desiredFrameLength = 1000/60;
64         if (this._options["controller"] == "ramp30")
65             desiredFrameLength = 1000/30;
66
67         function findRegression(series) {
68             var minIndex = Math.round(.025 * series.length);
69             var maxIndex = Math.round(.975 * (series.length - 1));
70             var minComplexity = series[minIndex].complexity;
71             var maxComplexity = series[maxIndex].complexity;
72             if (Math.abs(maxComplexity - minComplexity) < 20 && maxIndex - minIndex < 20) {
73                 minIndex = 0;
74                 maxIndex = series.length - 1;
75                 minComplexity = series[minIndex].complexity;
76                 maxComplexity = series[maxIndex].complexity;
77             }
78
79             return {
80                 minComplexity: minComplexity,
81                 maxComplexity: maxComplexity,
82                 samples: series.slice(minIndex, maxIndex + 1),
83                 regression: new Regression(
84                     series,
85                     function (datum, i) { return datum[i].complexity; },
86                     function (datum, i) { return datum[i].frameLength; },
87                     minIndex, maxIndex, desiredFrameLength)
88             };
89         }
90
91         var complexitySamples;
92         [Strings.json.complexity, Strings.json.complexityAverage].forEach(function(seriesName) {
93             if (!(seriesName in samples))
94                 return;
95
96             var regression = {};
97             result[seriesName] = regression;
98             var regressionResult = findRegression(samples[seriesName]);
99             if (seriesName == Strings.json.complexity)
100                 complexitySamples = regressionResult.samples;
101             var calculation = regressionResult.regression;
102             regression[Strings.json.regressions.segment1] = [
103                 [regressionResult.minComplexity, calculation.s1 + calculation.t1 * regressionResult.minComplexity],
104                 [calculation.complexity, calculation.s1 + calculation.t1 * calculation.complexity]
105             ];
106             regression[Strings.json.regressions.segment2] = [
107                 [calculation.complexity, calculation.s2 + calculation.t2 * calculation.complexity],
108                 [regressionResult.maxComplexity, calculation.s2 + calculation.t2 * regressionResult.maxComplexity]
109             ];
110             regression[Strings.json.complexity] = calculation.complexity;
111             regression[Strings.json.measurements.stdev] = Math.sqrt(calculation.error / samples[seriesName].length);
112         });
113
114         if (["ramp", "ramp30"].indexOf(this._options["controller"]) != -1) {
115             var timeComplexity = new Experiment;
116             data[Strings.json.controller].forEach(function(regression) {
117                 timeComplexity.sample(regression[Strings.json.complexity]);
118             });
119
120             var experimentResult = {};
121             result[Strings.json.controller] = experimentResult;
122             experimentResult[Strings.json.score] = timeComplexity.mean();
123             experimentResult[Strings.json.measurements.average] = timeComplexity.mean();
124             experimentResult[Strings.json.measurements.stdev] = timeComplexity.standardDeviation();
125             experimentResult[Strings.json.measurements.percent] = timeComplexity.percentage();
126
127             result[Strings.json.complexity][Strings.json.bootstrap] = Regression.bootstrap(complexitySamples, 2500, function(resample) {
128                     resample.sort(function(a, b) {
129                         return a.complexity - b.complexity;
130                     });
131
132                     var regressionResult = findRegression(resample);
133                     return regressionResult.regression.complexity;
134                 }, .95);
135             result[Strings.json.score] = result[Strings.json.complexity][Strings.json.bootstrap].median;
136
137         } else {
138             var marks = data[Strings.json.marks];
139             var samplingStartIndex = 0, samplingEndIndex = -1;
140             if (Strings.json.samplingStartTimeOffset in marks)
141                 samplingStartIndex = marks[Strings.json.samplingStartTimeOffset].index;
142             if (Strings.json.samplingEndTimeOffset in marks)
143                 samplingEndIndex = marks[Strings.json.samplingEndTimeOffset].index;
144
145             var averageComplexity = new Experiment;
146             var averageFrameLength = new Experiment;
147             samples[Strings.json.controller].forEach(function (sample, i) {
148                 if (i >= samplingStartIndex && (samplingEndIndex == -1 || i < samplingEndIndex)) {
149                     averageComplexity.sample(sample.complexity);
150                     if (sample.smoothedFrameLength && sample.smoothedFrameLength != -1)
151                         averageFrameLength.sample(sample.smoothedFrameLength);
152                 }
153             });
154
155             var experimentResult = {};
156             result[Strings.json.controller] = experimentResult;
157             experimentResult[Strings.json.measurements.average] = averageComplexity.mean();
158             experimentResult[Strings.json.measurements.concern] = averageComplexity.concern(Experiment.defaults.CONCERN);
159             experimentResult[Strings.json.measurements.stdev] = averageComplexity.standardDeviation();
160             experimentResult[Strings.json.measurements.percent] = averageComplexity.percentage();
161
162             experimentResult = {};
163             result[Strings.json.frameLength] = experimentResult;
164             experimentResult[Strings.json.measurements.average] = 1000 / averageFrameLength.mean();
165             experimentResult[Strings.json.measurements.concern] = averageFrameLength.concern(Experiment.defaults.CONCERN);
166             experimentResult[Strings.json.measurements.stdev] = averageFrameLength.standardDeviation();
167             experimentResult[Strings.json.measurements.percent] = averageFrameLength.percentage();
168
169             result[Strings.json.score] = averageComplexity.score(Experiment.defaults.CONCERN);
170         }
171     },
172
173     get data()
174     {
175         return this._iterationsSamplers;
176     },
177
178     get results()
179     {
180         if (this._results)
181             return this._results[Strings.json.results.iterations];
182         this._processData();
183         return this._results[Strings.json.results.iterations];
184     },
185
186     get options()
187     {
188         return this._options;
189     },
190
191     get score()
192     {
193         if (this._results)
194             return this._results[Strings.json.score];
195         this._processData();
196         return this._results[Strings.json.score];
197     }
198 });
199
200 ResultsTable = Utilities.createClass(
201     function(element, headers)
202     {
203         this.element = element;
204         this._headers = headers;
205
206         this._flattenedHeaders = [];
207         this._headers.forEach(function(header) {
208             if (header.disabled)
209                 return;
210
211             if (header.children)
212                 this._flattenedHeaders = this._flattenedHeaders.concat(header.children);
213             else
214                 this._flattenedHeaders.push(header);
215         }, this);
216
217         this._flattenedHeaders = this._flattenedHeaders.filter(function (header) {
218             return !header.disabled;
219         });
220
221         this.clear();
222     }, {
223
224     clear: function()
225     {
226         this.element.innerHTML = "";
227     },
228
229     _addHeader: function()
230     {
231         var thead = Utilities.createElement("thead", {}, this.element);
232         var row = Utilities.createElement("tr", {}, thead);
233
234         this._headers.forEach(function (header) {
235             if (header.disabled)
236                 return;
237
238             var th = Utilities.createElement("th", {}, row);
239             if (header.title != Strings.text.graph)
240                 th.textContent = header.title;
241             if (header.children)
242                 th.colSpan = header.children.length;
243         });
244     },
245
246     _addEmptyRow: function()
247     {
248         var row = Utilities.createElement("tr", {}, this.element);
249         this._flattenedHeaders.forEach(function (header) {
250             return Utilities.createElement("td", { class: "suites-separator" }, row);
251         });
252     },
253
254     _addTest: function(testName, testResults, options)
255     {
256         var row = Utilities.createElement("tr", {}, this.element);
257
258         this._flattenedHeaders.forEach(function (header) {
259             var td = Utilities.createElement("td", {}, row);
260             if (header.title == Strings.text.testName) {
261                 td.textContent = testName;
262             } else if (header.text) {
263                 var data = testResults[header.text];
264                 if (typeof data == "number")
265                     data = data.toFixed(2);
266                 td.textContent = data;
267             }
268         }, this);
269     },
270
271     _addIteration: function(iterationResult, iterationData, options)
272     {
273         var testsResults = iterationResult[Strings.json.results.tests];
274         for (var suiteName in testsResults) {
275             this._addEmptyRow();
276             var suiteResult = testsResults[suiteName];
277             var suiteData = iterationData[suiteName];
278             for (var testName in suiteResult)
279                 this._addTest(testName, suiteResult[testName], options, suiteData[testName]);
280         }
281     },
282
283     showIterations: function(dashboard)
284     {
285         this.clear();
286         this._addHeader();
287
288         var iterationsResults = dashboard.results;
289         iterationsResults.forEach(function(iterationResult, index) {
290             this._addIteration(iterationResult, dashboard.data[index], dashboard.options);
291         }, this);
292     }
293 });
294
295 window.benchmarkRunnerClient = {
296     iterationCount: 1,
297     options: null,
298     results: null,
299
300     initialize: function(suites, options)
301     {
302         this.options = options;
303     },
304
305     willStartFirstIteration: function()
306     {
307         this.results = new ResultsDashboard(this.options);
308     },
309
310     didRunSuites: function(suitesSamplers)
311     {
312         this.results.push(suitesSamplers);
313     },
314
315     didRunTest: function(testData)
316     {
317         this.results.calculateScore(testData);
318     },
319
320     didFinishLastIteration: function()
321     {
322         benchmarkController.showResults();
323     }
324 };
325
326 window.sectionsManager =
327 {
328     showSection: function(sectionIdentifier, pushState)
329     {
330         var currentSectionElement = document.querySelector("section.selected");
331         console.assert(currentSectionElement);
332
333         var newSectionElement = document.getElementById(sectionIdentifier);
334         console.assert(newSectionElement);
335
336         currentSectionElement.classList.remove("selected");
337         newSectionElement.classList.add("selected");
338
339         if (pushState)
340             history.pushState({section: sectionIdentifier}, document.title);
341     },
342
343     setSectionScore: function(sectionIdentifier, score, mean)
344     {
345         document.querySelector("#" + sectionIdentifier + " .score").textContent = score;
346         if (mean)
347             document.querySelector("#" + sectionIdentifier + " .mean").innerHTML = mean;
348     },
349
350     populateTable: function(tableIdentifier, headers, dashboard)
351     {
352         var table = new ResultsTable(document.getElementById(tableIdentifier), headers);
353         table.showIterations(dashboard);
354     }
355 };
356
357 window.benchmarkController = {
358     _startBenchmark: function(suites, options, frameContainerID)
359     {
360         benchmarkRunnerClient.initialize(suites, options);
361         var frameContainer = document.getElementById(frameContainerID);
362         var runner = new BenchmarkRunner(suites, frameContainer, benchmarkRunnerClient);
363         runner.runMultipleIterations();
364
365         sectionsManager.showSection("test-container");
366     },
367
368     startBenchmark: function()
369     {
370         var options = {
371             "test-interval": 20,
372             "display": "minimal",
373             "controller": "ramp",
374             "kalman-process-error": 1,
375             "kalman-measurement-error": 4,
376             "time-measurement": "performance"
377         };
378         this._startBenchmark(Suites, options, "test-container");
379     },
380
381     showResults: function()
382     {
383         if (!this.addedKeyEvent) {
384             document.addEventListener("keypress", this.selectResults, false);
385             this.addedKeyEvent = true;
386         }
387
388         var dashboard = benchmarkRunnerClient.results;
389
390         sectionsManager.setSectionScore("results", dashboard.score.toFixed(2));
391         sectionsManager.populateTable("results-header", Headers.testName, dashboard);
392         sectionsManager.populateTable("results-score", Headers.score, dashboard);
393         sectionsManager.showSection("results", true);
394     },
395
396     selectResults: function(event)
397     {
398         if (event.charCode != 115) // 's'
399             return;
400
401         event.target.selectRange = ((event.target.selectRange || 0) + 1) % 3;
402
403         var selection = window.getSelection();
404         selection.removeAllRanges();
405         var range = document.createRange();
406         switch (event.target.selectRange) {
407             case 0: {
408                 range.setStart(document.querySelector("#results .score"), 0);
409                 range.setEndAfter(document.querySelector("#results-score > tr:last-of-type"), 0);
410                 break;
411             }
412             case 1: {
413                 range.selectNodeContents(document.querySelector("#results .score"));
414                 break;
415             }
416             case 2: {
417                 range.selectNode(document.getElementById("results-score"));
418                 break;
419             }
420         }
421         selection.addRange(range);
422     }
423 };