Escape builder names in url* and pathFor* methods of BuildbotSyncer
[WebKit-https.git] / Websites / perf.webkit.org / server-tests / api-report-tests.js
1 'use strict';
2
3 const assert = require('assert');
4
5 const TestServer = require('./resources/test-server.js');
6 const addBuilderForReport = require('./resources/common-operations.js').addBuilderForReport;
7 const addSlaveForReport = require('./resources/common-operations.js').addSlaveForReport;
8 const connectToDatabaseInEveryTest = require('./resources/common-operations.js').connectToDatabaseInEveryTest;
9
10 describe("/api/report", function () {
11     this.timeout(1000);
12     TestServer.inject();
13     connectToDatabaseInEveryTest();
14
15     function emptyReport()
16     {
17         return {
18             "buildNumber": "123",
19             "buildTime": "2013-02-28T10:12:03.388304",
20             "builderName": "someBuilder",
21             "slaveName": "someSlave",
22             "builderPassword": "somePassword",
23             "platform": "Mountain Lion",
24             "tests": {},
25             "revisions": {
26                 "OS X": {
27                     "revision": "10.8.2 12C60"
28                 },
29                 "WebKit": {
30                     "revision": "141977",
31                     "timestamp": "2013-02-06T08:55:20.9Z"
32                 }
33             }
34         };
35     }
36
37     function emptySlaveReport()
38     {
39         return {
40             "buildNumber": "123",
41             "buildTime": "2013-02-28T10:12:03.388304",
42             "builderName": "someBuilder",
43             "slaveName": "someSlave",
44             "slavePassword": "otherPassword",
45             "platform": "Mountain Lion",
46             "tests": {},
47             "revisions": {
48                 "OS X": {
49                     "revision": "10.8.2 12C60"
50                 },
51                 "WebKit": {
52                     "revision": "141977",
53                     "timestamp": "2013-02-06T08:55:20.9Z"
54                 }
55             }
56         };
57     }
58
59     it("should reject error when builder name is missing", function (done) {
60         TestServer.remoteAPI().postJSON('/api/report/', [{"buildTime": "2013-02-28T10:12:03.388304"}]).then(function (response) {
61             assert.equal(response['status'], 'MissingBuilderName');
62             done();
63         }).catch(done);
64     });
65
66     it("should reject error when build time is missing", function (done) {
67         addBuilderForReport(emptyReport()).then(function () {
68             return TestServer.remoteAPI().postJSON('/api/report/', [{"builderName": "someBuilder", "builderPassword": "somePassword"}]);
69         }).then(function (response) {
70             assert.equal(response['status'], 'MissingBuildTime');
71             done();
72         });
73     });
74
75     it("should reject when there are no builders", function (done) {
76         TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]).then(function (response) {
77             assert.equal(response['status'], 'BuilderNotFound');
78             assert.equal(response['failureStored'], false);
79             assert.equal(response['processedRuns'], 0);
80             return TestServer.database().selectAll('reports');
81         }).then(function (reports) {
82             assert.equal(reports.length, 0);
83             done();
84         }).catch(done);
85     });
86
87     it("should reject a report without a builder password", function (done) {
88         addBuilderForReport(emptyReport()).then(function () {
89             var report = [{
90                 "buildNumber": "123",
91                 "buildTime": "2013-02-28T10:12:03.388304",
92                 "builderName": "someBuilder",
93                 "tests": {},
94                 "revisions": {}}];
95             return TestServer.remoteAPI().postJSON('/api/report/', report);
96         }).then(function (response) {
97             assert.equal(response['status'], 'BuilderNotFound');
98             assert.equal(response['failureStored'], false);
99             assert.equal(response['processedRuns'], 0);
100             return TestServer.database().selectAll('reports');
101         }).then(function (reports) {
102             assert.equal(reports.length, 0);
103             done();
104         }).catch(done);
105     });
106
107     it("should store a report from a valid builder", function (done) {
108         addBuilderForReport(emptyReport()).then(function () {
109             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
110         }).then(function (response) {
111             assert.equal(response['status'], 'OK');
112             assert.equal(response['failureStored'], false);
113             assert.equal(response['processedRuns'], 1);
114             return TestServer.database().selectAll('reports');
115         }).then(function (reports) {
116             assert.equal(reports.length, 1);
117             const submittedContent = emptyReport();
118             const storedContent = JSON.parse(reports[0]['content']);
119
120             delete submittedContent['builderPassword'];
121             delete submittedContent['tests'];
122             delete storedContent['tests'];
123             assert.deepEqual(storedContent, submittedContent);
124
125             done();
126         }).catch(done);
127     });
128
129     it("should treat the slave password as the builder password if there is no matching slave", function (done) {
130         let report = emptyReport();
131         report['slavePassword'] = report['builderPassword'];
132         delete report['builderPassword'];
133
134         addSlaveForReport(report).then(function () {
135             return TestServer.remoteAPI().postJSON('/api/report/', [report]);
136         }).then(function (response) {
137             assert.equal(response['status'], 'OK');
138             assert.equal(response['failureStored'], false);
139             assert.equal(response['processedRuns'], 1);
140             return TestServer.database().selectAll('reports');
141         }).then(function (reports) {
142             assert.equal(reports.length, 1);
143             const storedContent = JSON.parse(reports[0]['content']);
144
145             delete report['slavePassword'];
146             delete report['tests'];
147             delete storedContent['tests'];
148             assert.deepEqual(storedContent, report);
149
150             done();
151         }).catch(done);
152     });
153
154     it("should store a report from a valid slave", function (done) {
155         addSlaveForReport(emptySlaveReport()).then(function () {
156             return TestServer.remoteAPI().postJSON('/api/report/', [emptySlaveReport()]);
157         }).then(function (response) {
158             assert.equal(response['status'], 'OK');
159             assert.equal(response['failureStored'], false);
160             assert.equal(response['processedRuns'], 1);
161             return TestServer.database().selectAll('reports');
162         }).then(function (reports) {
163             assert.equal(reports.length, 1);
164             const submittedContent = emptySlaveReport();
165             const storedContent = JSON.parse(reports[0]['content']);
166
167             delete submittedContent['slavePassword'];
168             delete submittedContent['tests'];
169             delete storedContent['tests'];
170             assert.deepEqual(storedContent, submittedContent);
171
172             done();
173         }).catch(done);
174     });
175
176     it("should store the builder name but not the builder password", function (done) {
177         addBuilderForReport(emptyReport()).then(function () {
178             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
179         }).then(function (response) {
180             return TestServer.database().selectAll('reports');
181         }).then(function (reports) {
182             assert.equal(reports.length, 1);
183             const storedContent = JSON.parse(reports[0]['content']);
184             assert.equal(storedContent['builderName'], emptyReport()['builderName']);
185             assert(!('builderPassword' in storedContent));
186             done();
187         }).catch(done);
188     });
189
190     it("should add a slave if there isn't one and the report was authenticated by a builder", function (done) {
191         addBuilderForReport(emptyReport()).then(function () {
192             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
193         }).then(function (response) {
194             return TestServer.database().selectAll('build_slaves');
195         }).then(function (slaves) {
196             assert.equal(slaves.length, 1);
197             assert.equal(slaves[0]['name'], emptyReport()['slaveName']);
198             done();
199         }).catch(done);
200     });
201
202     it("should add a builder if there isn't one and the report was authenticated by a slave", function (done) {
203         addSlaveForReport(emptySlaveReport()).then(function () {
204             return TestServer.remoteAPI().postJSON('/api/report/', [emptySlaveReport()]);
205         }).then(function (response) {
206             return TestServer.database().selectAll('builders');
207         }).then(function (builders) {
208             assert.equal(builders.length, 1);
209             assert.equal(builders[0]['name'], emptyReport()['builderName']);
210             done();
211         }).catch(done);
212     });
213
214     it("should add a build", function (done) {
215         addBuilderForReport(emptyReport()).then(function () {
216             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
217         }).then(function () {
218             return TestServer.database().selectAll('builds');
219         }).then(function (builds) {
220             assert.strictEqual(builds[0]['number'], 123);
221             done();
222         }).catch(done);
223     });
224
225     it("should add the platform", function (done) {
226         addBuilderForReport(emptyReport()).then(function () {
227             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
228         }).then(function () {
229             return TestServer.database().selectAll('platforms');
230         }).then(function (platforms) {
231             assert.equal(platforms.length, 1);
232             assert.equal(platforms[0]['name'], 'Mountain Lion');
233             done();
234         }).catch(done);
235     });
236
237     it("should add repositories and build revisions", function (done) {
238         addBuilderForReport(emptyReport()).then(function () {
239             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
240         }).then(function (response) {
241             const db = TestServer.database();
242             return Promise.all([
243                 db.selectAll('repositories'),
244                 db.selectAll('commits'),
245                 db.selectAll('build_commits', 'build_commit'),
246             ]);
247         }).then(function (result) {
248             const repositories = result[0];
249             const commits = result[1];
250             const buildCommitsRelations = result[2];
251             assert.equal(repositories.length, 2);
252             assert.deepEqual(repositories.map(function (row) { return row['name']; }).sort(), ['OS X', 'WebKit']);
253
254             assert.equal(commits.length, 2);
255             assert.equal(buildCommitsRelations.length, 2);
256             assert.equal(buildCommitsRelations[0]['build_commit'], commits[0]['id']);
257             assert.equal(buildCommitsRelations[1]['build_commit'], commits[1]['id']);
258             assert.equal(buildCommitsRelations[0]['commit_build'], buildCommitsRelations[1]['commit_build']);
259
260             let repositoryIdToName = {};
261             for (let repository of repositories)
262                 repositoryIdToName[repository['id']] = repository['name'];
263
264             let repositoryNameToRevisionRow = {};
265             for (let commit of commits)
266                 repositoryNameToRevisionRow[repositoryIdToName[commit['repository']]] = commit;
267
268             assert.equal(repositoryNameToRevisionRow['OS X']['revision'], '10.8.2 12C60');
269             assert.equal(repositoryNameToRevisionRow['WebKit']['revision'], '141977');
270             assert.equal(repositoryNameToRevisionRow['WebKit']['time'].toString(),
271                 new Date('2013-02-06 08:55:20.9').toString());
272             done();
273         }).catch(done);
274     });
275
276     it("should not create a duplicate build for the same build number if build times are close", function (done) {
277         let firstReport = emptyReport();
278         firstReport['buildTime'] = '2013-02-28T10:12:04';
279         let secondReport = emptyReport();
280         secondReport['buildTime'] = '2013-02-28T10:22:03';
281
282         addBuilderForReport(emptyReport()).then(function () {
283             return TestServer.remoteAPI().postJSON('/api/report/', [firstReport]);
284         }).then(function (response) {
285             assert.equal(response['status'], 'OK');
286             return TestServer.database().selectAll('builds');
287         }).then(function (builds) {
288             assert.equal(builds.length, 1);
289             return TestServer.remoteAPI().postJSON('/api/report/', [secondReport]);
290         }).then(function (response) {
291             assert.equal(response['status'], 'OK');
292             return TestServer.database().selectAll('builds');
293         }).then(function (builds) {
294             assert.equal(builds.length, 1);
295             done();
296         }).catch(done);
297     });
298
299     it("should create distinct builds for the same build number if build times are far apart", function (done) {
300         let firstReport = emptyReport();
301         firstReport['buildTime'] = '2013-02-28T10:12:03';
302         let secondReport = emptyReport();
303         secondReport['buildTime'] = '2014-01-20T22:23:34';
304
305         addBuilderForReport(emptyReport()).then(function () {
306             return TestServer.remoteAPI().postJSON('/api/report/', [firstReport]);
307         }).then(function (response) {
308             assert.equal(response['status'], 'OK');
309             return TestServer.database().selectAll('builds');
310         }).then(function (builds) {
311             assert.equal(builds.length, 1);
312             return TestServer.remoteAPI().postJSON('/api/report/', [secondReport]);
313         }).then(function (response) {
314             assert.equal(response['status'], 'OK');
315             return TestServer.database().selectAll('builds');
316         }).then(function (builds) {
317             assert.equal(builds.length, 2);
318             done();
319         }).catch(done);
320     });
321
322     it("should reject a report with mismatching revision info", function (done) {
323         let firstReport = emptyReport();
324         firstReport['revisions'] = {
325             "WebKit": {
326                 "revision": "141977",
327                 "timestamp": "2013-02-06T08:55:20.96Z"
328             }
329         };
330
331         let secondReport = emptyReport();
332         secondReport['revisions'] = {
333             "WebKit": {
334                 "revision": "150000",
335                 "timestamp": "2013-05-13T10:50:29.6Z"
336             }
337         };
338
339         addBuilderForReport(firstReport).then(function () {
340             return TestServer.remoteAPI().postJSON('/api/report/', [firstReport]);
341         }).then(function (response) {
342             assert.equal(response['status'], 'OK');
343             return TestServer.database().selectAll('builds');
344         }).then(function (builds) {
345             assert.equal(builds.length, 1);
346             return TestServer.remoteAPI().postJSON('/api/report/', [secondReport]);
347         }).then(function (response) {
348             assert.equal(response['status'], 'MismatchingCommitRevision');
349             assert(JSON.stringify(response).indexOf('141977') >= 0);
350             assert(JSON.stringify(response).indexOf('150000') >= 0);
351             assert.equal(response['failureStored'], true);
352             assert.equal(response['processedRuns'], 0);
353             done();
354         }).catch(done);
355     });
356
357     const reportWithTwoLevelsOfAggregations = {
358         "buildNumber": "123",
359         "buildTime": "2013-02-28T10:12:03.388304",
360         "builderName": "someBuilder",
361         "builderPassword": "somePassword",
362         "platform": "Mountain Lion",
363         "tests": {
364             "DummyPageLoading": {
365                 "metrics": {"Time": { "aggregators" : ["Arithmetic"], "current": [300, 310, 320, 330] }},
366                 "tests": {
367                     "apple.com": {
368                         "metrics": {"Time": { "current": [500, 510, 520, 530] }},
369                         "url": "http://www.apple.com"
370                     },
371                     "webkit.org": {
372                         "metrics": {"Time": { "current": [100, 110, 120, 130] }},
373                         "url": "http://www.webkit.org"
374                     }
375                 }
376             },
377             "DummyBenchmark": {
378                 "metrics": {"Time": ["Arithmetic"]},
379                 "tests": {
380                     "DOM": {
381                         "metrics": {"Time": ["Geometric", "Arithmetic"]},
382                         "tests": {
383                             "ModifyNodes": {"metrics": {"Time": { "current": [[11, 12, 13, 14, 15], [16, 17, 18, 19, 20], [21, 22, 23, 24, 25]] }}},
384                             "TraverseNodes": {"metrics": {"Time": { "current": [[31, 32, 33, 34, 35], [36, 37, 38, 39, 40], [41, 42, 43, 44, 45]] }}}
385                         }
386                     },
387                     "CSS": {"metrics": {"Time": { "current": [[101, 102, 103, 104, 105], [106, 107, 108, 109, 110], [111, 112, 113, 114, 115]] }}}
388                 }
389             }
390         },
391         "revisions": {
392             "OS X": {
393                 "revision": "10.8.2 12C60"
394             },
395             "WebKit": {
396                 "revision": "141977",
397                 "timestamp": "2013-02-06T08:55:20.9Z"
398             }
399         }};
400
401     function reportAfterAddingBuilderAndAggregators(report)
402     {
403         return addBuilderForReport(report).then(function () {
404             const db = TestServer.database();
405             return Promise.all([
406                 db.insert('aggregators', {name: 'Arithmetic'}),
407                 db.insert('aggregators', {name: 'Geometric'}),
408             ]);
409         }).then(function () {
410             return TestServer.remoteAPI().postJSON('/api/report/', [report]);
411         }).then(function (response) {
412             assert.equal(response['status'], 'OK');
413             assert.equal(response['failureStored'], false);
414             return response;
415         });
416     }
417
418     function fetchRunForMetric(testName, metricName,callback) {
419         queryAndFetchAll('SELECT * FROM test_runs WHERE run_config IN'
420             + '(SELECT config_id FROM test_configurations, test_metrics, tests WHERE config_metric = metric_id AND metric_test = test_id AND'
421             + 'test_name = $1 AND metric_name = $2)',
422             ['Arithmetic', 'values.reduce(function (a, b) { return a + b; }) / values.length'], function () {
423             queryAndFetchAll('INSERT INTO aggregators (aggregator_name, aggregator_definition) values ($1, $2)',
424                 ['Geometric', 'Math.pow(values.reduce(function (a, b) { return a * b; }), 1 / values.length)'], callback);
425         });
426     }
427
428     it("should accept a report with aggregators", function (done) {
429         reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(function () {
430             done();
431         }).catch(done);
432     });
433
434     it("should add tests", function (done) {
435         reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(function () {
436             return TestServer.database().selectAll('tests');
437         }).then(function (tests) {
438             assert.deepEqual(tests.map(function (row) { return row['name']; }).sort(),
439                 ['CSS', 'DOM', 'DummyBenchmark', 'DummyPageLoading', 'ModifyNodes', 'TraverseNodes', 'apple.com', 'webkit.org']);
440             done();
441         }).catch(done);
442     });
443
444     it("should add metrics", function (done) {
445         reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(function () {
446             return TestServer.database().query('SELECT * FROM tests, test_metrics LEFT JOIN aggregators ON metric_aggregator = aggregator_id WHERE metric_test = test_id');
447         }).then(function (result) {
448             let testNameToMetrics = {};
449             result.rows.forEach(function (row) {
450                 if (!(row['test_name'] in testNameToMetrics))
451                     testNameToMetrics[row['test_name']] = new Array;
452                 testNameToMetrics[row['test_name']].push([row['metric_name'], row['aggregator_name']]);
453             });
454             assert.deepEqual(testNameToMetrics['CSS'], [['Time', null]]);
455             assert.deepEqual(testNameToMetrics['DOM'].sort(), [['Time', 'Arithmetic'], ['Time', 'Geometric']]);
456             assert.deepEqual(testNameToMetrics['DummyBenchmark'], [['Time', 'Arithmetic']]);
457             assert.deepEqual(testNameToMetrics['DummyPageLoading'], [['Time', 'Arithmetic']]);
458             assert.deepEqual(testNameToMetrics['ModifyNodes'], [['Time', null]]);
459             assert.deepEqual(testNameToMetrics['TraverseNodes'], [['Time', null]]);
460             assert.deepEqual(testNameToMetrics['apple.com'], [['Time', null]]);
461             assert.deepEqual(testNameToMetrics['webkit.org'], [['Time', null]]);
462             done();
463         }).catch(done);
464     });
465
466     function fetchTestConfig(testName, metricName)
467     {
468         return TestServer.database().query(`SELECT * FROM tests, test_metrics, test_configurations
469             WHERE test_id = metric_test AND metric_id = config_metric
470             AND test_name = $1 AND metric_name = $2`, [testName, metricName]).then(function (result) {
471                 assert.equal(result.rows.length, 1);
472                 return result.rows[0];
473             });
474     }
475
476     function fetchTestRunIterationsForMetric(testName, metricName)
477     {
478         const db = TestServer.database();
479         return fetchTestConfig(testName, metricName).then(function (config) {
480             return db.selectFirstRow('test_runs', {config: config['config_id']});
481         }).then(function (run) {
482             return db.selectRows('run_iterations', {run: run['id']}, {sortBy: 'order'}).then(function (iterations) {
483                 return {run: run, iterations: iterations};
484             });
485         });
486     }
487
488     it("should store run values", function (done) {
489         reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(function () {
490             return fetchTestRunIterationsForMetric('apple.com', 'Time');
491         }).then(function (result) {
492             const run = result.run;
493             const runId = run['id'];
494             assert.deepEqual(result.iterations, [
495                 {run: runId, order: 0, group: null, value: 500, relative_time: null},
496                 {run: runId, order: 1, group: null, value: 510, relative_time: null},
497                 {run: runId, order: 2, group: null, value: 520, relative_time: null},
498                 {run: runId, order: 3, group: null, value: 530, relative_time: null}]);
499             var sum = 500 + 510 + 520 + 530;
500             assert.equal(run['mean_cache'], sum / result.iterations.length);
501             assert.equal(run['sum_cache'], sum);
502             assert.equal(run['square_sum_cache'], 500 * 500 + 510 * 510 + 520 * 520 + 530 * 530);
503             return fetchTestRunIterationsForMetric('CSS', 'Time');
504         }).then(function (result) {
505             const run = result.run;
506             const runId = run['id'];
507             assert.deepEqual(result.iterations, [
508                 {run: runId, order: 0, group: 0, value: 101, relative_time: null},
509                 {run: runId, order: 1, group: 0, value: 102, relative_time: null},
510                 {run: runId, order: 2, group: 0, value: 103, relative_time: null},
511                 {run: runId, order: 3, group: 0, value: 104, relative_time: null},
512                 {run: runId, order: 4, group: 0, value: 105, relative_time: null},
513                 {run: runId, order: 5, group: 1, value: 106, relative_time: null},
514                 {run: runId, order: 6, group: 1, value: 107, relative_time: null},
515                 {run: runId, order: 7, group: 1, value: 108, relative_time: null},
516                 {run: runId, order: 8, group: 1, value: 109, relative_time: null},
517                 {run: runId, order: 9, group: 1, value: 110, relative_time: null},
518                 {run: runId, order: 10, group: 2, value: 111, relative_time: null},
519                 {run: runId, order: 11, group: 2, value: 112, relative_time: null},
520                 {run: runId, order: 12, group: 2, value: 113, relative_time: null},
521                 {run: runId, order: 13, group: 2, value: 114, relative_time: null},
522                 {run: runId, order: 14, group: 2, value: 115, relative_time: null}]);
523             let sum = 0;
524             let squareSum = 0;
525             for (let value = 101; value <= 115; ++value) {
526                 sum += value;
527                 squareSum += value * value;
528             }
529             assert.equal(run['mean_cache'], sum / result.iterations.length);
530             assert.equal(run['sum_cache'], sum);
531             assert.equal(run['square_sum_cache'], squareSum);
532             done();
533         }).catch(done);
534     });
535
536     it("should store aggregated run values", function (done) {
537         reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(function () {
538             return fetchTestRunIterationsForMetric('DummyPageLoading', 'Time');
539         }).then(function (result) {
540             const run = result.run;
541             const runId = result.run['id'];
542             const expectedValues = [(500 + 100) / 2, (510 + 110) / 2, (520 + 120) / 2, (530 + 130) / 2];
543             assert.deepEqual(result.iterations, [
544                 {run: runId, order: 0, group: null, value: expectedValues[0], relative_time: null},
545                 {run: runId, order: 1, group: null, value: expectedValues[1], relative_time: null},
546                 {run: runId, order: 2, group: null, value: expectedValues[2], relative_time: null},
547                 {run: runId, order: 3, group: null, value: expectedValues[3], relative_time: null}]);
548             const sum = expectedValues.reduce(function (sum, value) { return sum + value; }, 0);
549             assert.equal(run['mean_cache'], sum / result.iterations.length);
550             assert.equal(run['sum_cache'], sum);
551             assert.equal(run['square_sum_cache'], expectedValues.reduce(function (sum, value) { return sum + value * value; }, 0));
552             done();
553         }).catch(done);
554     });
555
556     it("should be able to compute the aggregation of aggregated values", function (done) {
557         reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(function () {
558             return fetchTestRunIterationsForMetric('DummyBenchmark', 'Time');
559         }).then(function (result) {
560             const run = result.run;
561             const runId = run['id'];
562             const expectedIterations = [];
563             let sum = 0;
564             let squareSum = 0;
565             for (let i = 0; i < 15; ++i) {
566                 const value = i + 1;
567                 const DOMMean = ((10 + value) + (30 + value)) / 2;
568                 const expectedValue = (DOMMean + 100 + value) / 2;
569                 sum += expectedValue;
570                 squareSum += expectedValue * expectedValue;
571                 expectedIterations.push({run: runId, order: i, group: Math.floor(i / 5), value: expectedValue, relative_time: null});
572             }
573             assert.deepEqual(result.iterations, expectedIterations);
574             assert.equal(run['mean_cache'], sum / result.iterations.length);
575             assert.equal(run['sum_cache'], sum);
576             assert.equal(run['square_sum_cache'], squareSum);
577             done();
578         }).catch(done);
579     });
580
581     function reportWithSameSubtestName()
582     {
583         return {
584             "buildNumber": "123",
585             "buildTime": "2013-02-28T10:12:03.388304",
586             "builderName": "someBuilder",
587             "builderPassword": "somePassword",
588             "platform": "Mountain Lion",
589             "tests": {
590                 "Suite1": {
591                     "tests": {
592                         "test1": {
593                             "metrics": {"Time": { "current": [1, 2, 3, 4, 5] }}
594                         },
595                         "test2": {
596                             "metrics": {"Time": { "current": [6, 7, 8, 9, 10] }}
597                         }
598                     }
599                 },
600                 "Suite2": {
601                     "tests": {
602                         "test1": {
603                             "metrics": {"Time": { "current": [11, 12, 13, 14, 15] }}
604                         },
605                         "test2": {
606                             "metrics": {"Time": { "current": [16, 17, 18, 19, 20] }}
607                         }
608                     }
609                 }
610             }
611         };
612     }
613
614     it("should be able to add a report with same subtest name", function (done) {
615         reportAfterAddingBuilderAndAggregators(reportWithSameSubtestName()).then(function () {
616             done();
617         }).catch(done);
618     });
619
620     it("should be able to reuse the same test rows", function (done) {
621         reportAfterAddingBuilderAndAggregators(reportWithSameSubtestName()).then(function () {
622             return TestServer.database().selectAll('tests');
623         }).then(function (tests) {
624             assert.equal(tests.length, 6);
625             let newReport = reportWithSameSubtestName();
626             newReport.buildNumber = "125";
627             newReport.buildTime = "2013-02-28T12:17:24.1";
628             return TestServer.remoteAPI().postJSON('/api/report/', [newReport]);
629         }).then(function (response) {
630             assert.equal(response['status'], 'OK');
631             return TestServer.database().selectAll('tests');
632         }).then(function (tests) {
633             assert.equal(tests.length, 6);
634             done();
635         }).catch(done);
636     });
637
638     const reportWithSameSingleValue = {
639         "buildNumber": "123",
640         "buildTime": "2013-02-28T10:12:03.388304",
641         "builderName": "someBuilder",
642         "builderPassword": "somePassword",
643         "platform": "Mountain Lion",
644         "tests": {
645             "suite": {
646                 "metrics": {"Combined": ["Arithmetic"]},
647                 "tests": {
648                     "test1": {
649                         "metrics": {"Combined": { "current": 3 }}
650                     },
651                     "test2": {
652                         "metrics": {"Combined": { "current": 7 }}
653                     }
654                 }
655             },
656         }};
657
658     it("should be able to add a report with single value results", function (done) {
659         reportAfterAddingBuilderAndAggregators(reportWithSameSingleValue).then(function () {
660             return fetchTestRunIterationsForMetric('test1', 'Combined');
661         }).then(function (result) {
662             const run = result.run;
663             assert.equal(run['iteration_count_cache'], 1);
664             assert.equal(run['mean_cache'], 3);
665             assert.equal(run['sum_cache'], 3);
666             assert.equal(run['square_sum_cache'], 9);
667             return fetchTestRunIterationsForMetric('suite', 'Combined');
668         }).then(function (result) {
669             const run = result.run;
670             assert.equal(run['iteration_count_cache'], 1);
671             assert.equal(run['mean_cache'], 5);
672             assert.equal(run['sum_cache'], 5);
673             assert.equal(run['square_sum_cache'], 25);
674             done();
675         }).catch(done);
676     });
677
678     const reportWithSameValuePairs = {
679         "buildNumber": "123",
680         "buildTime": "2013-02-28T10:12:03.388304",
681         "builderName": "someBuilder",
682         "builderPassword": "somePassword",
683         "platform": "Mountain Lion",
684         "tests": {
685                 "test": {
686                     "metrics": {"FrameRate": { "current": [[[0, 4], [100, 5], [205, 3]]] }}
687                 },
688             },
689         };
690
691     it("should be able to add a report with (relative time, value) pairs", function (done) {
692         reportAfterAddingBuilderAndAggregators(reportWithSameValuePairs).then(function () {
693             return fetchTestRunIterationsForMetric('test', 'FrameRate');
694         }).then(function (result) {
695             const run = result.run;
696             assert.equal(run['iteration_count_cache'], 3);
697             assert.equal(run['mean_cache'], 4);
698             assert.equal(run['sum_cache'], 12);
699             assert.equal(run['square_sum_cache'], 16 + 25 + 9);
700
701             const runId = run['id'];
702             assert.deepEqual(result.iterations, [
703                 {run: runId, order: 0, group: null, value: 4, relative_time: 0},
704                 {run: runId, order: 1, group: null, value: 5, relative_time: 100},
705                 {run: runId, order: 2, group: null, value: 3, relative_time: 205}]);
706             done();
707         }).catch(done);
708     });
709
710     const reportsUpdatingDifferentTests = [
711         {
712             "buildNumber": "123",
713             "buildTime": "2013-02-28T10:12:03",
714             "builderName": "someBuilder",
715             "builderPassword": "somePassword",
716             "platform": "Mountain Lion",
717             "tests": {"test1": {"metrics": {"Time": {"current": 3}}}}
718         },
719         {
720             "buildNumber": "124",
721             "buildTime": "2013-02-28T11:31:21",
722             "builderName": "someBuilder",
723             "builderPassword": "somePassword",
724             "platform": "Mountain Lion",
725             "tests": {"test2": {"metrics": {"Time": {"current": 3}}}}
726         },
727         {
728             "buildNumber": "125",
729             "buildTime": "2013-02-28T12:45:34",
730             "builderName": "someBuilder",
731             "builderPassword": "somePassword",
732             "platform": "Mountain Lion",
733             "tests": {"test1": {"metrics": {"Time": {"current": 3}}}}
734         },
735     ];
736
737     it("should update the last modified date of test configurations with new runs", function (done) {
738         addBuilderForReport(reportsUpdatingDifferentTests[0]).then(function () {
739             return TestServer.remoteAPI().postJSON('/api/report/', [reportsUpdatingDifferentTests[0]]);
740         }).then(function (response) {
741             assert.equal(response['status'], 'OK');
742             return fetchTestConfig('test1', 'Time');
743         }).then(function (originalConfig) {
744             return TestServer.remoteAPI().postJSON('/api/report/', [reportsUpdatingDifferentTests[2]]).then(function () {
745                 return fetchTestConfig('test1', 'Time');
746             }).then(function (config) {
747                 assert(originalConfig['config_runs_last_modified'] instanceof Date);
748                 assert(config['config_runs_last_modified'] instanceof Date);
749                 assert(+originalConfig['config_runs_last_modified'] < +config['config_runs_last_modified']);
750                 done();
751             });
752         }).catch(done);
753     });
754
755     it("should not update the last modified date of unrelated test configurations", function (done) {
756         addBuilderForReport(reportsUpdatingDifferentTests[0]).then(function () {
757             return TestServer.remoteAPI().postJSON('/api/report/', [reportsUpdatingDifferentTests[0]]);
758         }).then(function (response) {
759             assert.equal(response['status'], 'OK');
760             return fetchTestConfig('test1', 'Time');
761         }).then(function (originalConfig) {
762             return TestServer.remoteAPI().postJSON('/api/report/', [reportsUpdatingDifferentTests[1]]).then(function (response) {
763                 assert.equal(response['status'], 'OK');
764                 return fetchTestConfig('test1', 'Time');
765             }).then(function (config) {
766                 assert(originalConfig['config_runs_last_modified'] instanceof Date);
767                 assert(config['config_runs_last_modified'] instanceof Date);
768                 assert.equal(+originalConfig['config_runs_last_modified'], +config['config_runs_last_modified']);
769                 done();
770             });
771         }).catch(done);
772     });
773 });