"/api/report" does not check commit time correctly.
[WebKit-https.git] / Websites / perf.webkit.org / server-tests / api-report-tests.js
1 'use strict';
2
3 const assert = require('assert');
4
5 const TestServer = require('./resources/test-server.js');
6 const addBuilderForReport = require('./resources/common-operations.js').addBuilderForReport;
7 const addSlaveForReport = require('./resources/common-operations.js').addSlaveForReport;
8 const prepareServerTest = require('./resources/common-operations.js').prepareServerTest;
9 const MockData = require('./resources/mock-data.js');
10
11 describe("/api/report", function () {
12     prepareServerTest(this);
13
14     function emptyReport()
15     {
16         return {
17             "buildNumber": "123",
18             "buildTime": "2013-02-28T10:12:03.388304",
19             "builderName": "someBuilder",
20             "slaveName": "someSlave",
21             "builderPassword": "somePassword",
22             "platform": "Mountain Lion",
23             "tests": {},
24             "revisions": {
25                 "macOS": {
26                     "revision": "10.8.2 12C60"
27                 },
28                 "WebKit": {
29                     "revision": "141977",
30                     "timestamp": "2013-02-06T08:55:20.9Z"
31                 }
32             }
33         };
34     }
35
36     function reportWitMismatchingCommitTime()
37     {
38         return {
39             "buildNumber": "124",
40             "buildTime": "2013-02-28T10:12:03.388304",
41             "builderName": "someBuilder",
42             "slaveName": "someSlave",
43             "builderPassword": "somePassword",
44             "platform": "Mountain Lion",
45             "tests": {},
46             "revisions": {
47                 "macOS": {
48                     "revision": "10.8.2 12C60"
49                 },
50                 "WebKit": {
51                     "revision": "141977",
52                     "timestamp": "2013-02-06T08:55:10.9Z"
53                 }
54             }
55         };
56     }
57
58     function reportWithOneSecondCommitTimeDifference()
59     {
60         return {
61             "buildNumber": "125",
62             "buildTime": "2013-02-28T10:12:03.388304",
63             "builderName": "someBuilder",
64             "slaveName": "someSlave",
65             "builderPassword": "somePassword",
66             "platform": "Mountain Lion",
67             "tests": {},
68             "revisions": {
69                 "macOS": {
70                     "revision": "10.8.2 12C60"
71                 },
72                 "WebKit": {
73                     "revision": "141977",
74                     "timestamp": "2013-02-06T08:55:19.9Z"
75                 }
76             }
77         };
78     }
79
80     function emptySlaveReport()
81     {
82         return {
83             "buildNumber": "123",
84             "buildTime": "2013-02-28T10:12:03.388304",
85             "builderName": "someBuilder",
86             "slaveName": "someSlave",
87             "slavePassword": "otherPassword",
88             "platform": "Mountain Lion",
89             "tests": {},
90             "revisions": {
91                 "macOS": {
92                     "revision": "10.8.2 12C60"
93                 },
94                 "WebKit": {
95                     "revision": "141977",
96                     "timestamp": "2013-02-06T08:55:20.9Z"
97                 }
98             }
99         };
100     }
101
102     it("should reject error when builder name is missing", () => {
103         return TestServer.remoteAPI().postJSON('/api/report/', [{"buildTime": "2013-02-28T10:12:03.388304"}]).then((response) => {
104             assert.equal(response['status'], 'MissingBuilderName');
105         });
106     });
107
108     it("should reject error when build time is missing", () => {
109         return addBuilderForReport(emptyReport()).then(() => {
110             return TestServer.remoteAPI().postJSON('/api/report/', [{"builderName": "someBuilder", "builderPassword": "somePassword"}]);
111         }).then((response) => {
112             assert.equal(response['status'], 'MissingBuildTime');
113         });
114     });
115
116     it("should reject when there are no builders", () => {
117         return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]).then((response) => {
118             assert.equal(response['status'], 'BuilderNotFound');
119             assert.equal(response['failureStored'], false);
120             assert.equal(response['processedRuns'], 0);
121             return TestServer.database().selectAll('reports');
122         }).then((reports) => {
123             assert.equal(reports.length, 0);
124         });
125     });
126
127     it("should reject a report without a builder password", () => {
128         return addBuilderForReport(emptyReport()).then(() => {
129             var report = [{
130                 "buildNumber": "123",
131                 "buildTime": "2013-02-28T10:12:03.388304",
132                 "builderName": "someBuilder",
133                 "tests": {},
134                 "revisions": {}}];
135             return TestServer.remoteAPI().postJSON('/api/report/', report);
136         }).then((response) => {
137             assert.equal(response['status'], 'BuilderNotFound');
138             assert.equal(response['failureStored'], false);
139             assert.equal(response['processedRuns'], 0);
140             return TestServer.database().selectAll('reports');
141         }).then((reports) => {
142             assert.equal(reports.length, 0);
143         });
144     });
145
146     it('should reject report with "MismatchingCommitTime" if time difference is larger than 1 second', async () => {
147         await addBuilderForReport(emptyReport());
148         let response = await TestServer.remoteAPI().postJSON('/api/report/', [reportWitMismatchingCommitTime()]);
149         assert.equal(response['status'], 'OK');
150         assert.equal(response['failureStored'], false);
151         assert.equal(response['processedRuns'], 1);
152
153         response = await TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
154         assert.equal(response['status'], 'MismatchingCommitTime');
155         assert.equal(response['failureStored'], true);
156         assert.equal(response['processedRuns'], 0);
157     });
158
159     it('should not reject report if the commit time difference is within 1 second"', async () => {
160         await addBuilderForReport(emptyReport());
161         let response = await TestServer.remoteAPI().postJSON('/api/report/', [reportWithOneSecondCommitTimeDifference()]);
162         assert.equal(response['status'], 'OK');
163         assert.equal(response['failureStored'], false);
164         assert.equal(response['processedRuns'], 1);
165
166         response = await TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
167         assert.equal(response['status'], 'OK');
168         assert.equal(response['failureStored'], false);
169         assert.equal(response['processedRuns'], 1);
170     });
171
172     it("should store a report from a valid builder", () => {
173         return addBuilderForReport(emptyReport()).then(() => {
174             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
175         }).then((response) => {
176             assert.equal(response['status'], 'OK');
177             assert.equal(response['failureStored'], false);
178             assert.equal(response['processedRuns'], 1);
179             return TestServer.database().selectAll('reports');
180         }).then((reports) => {
181             assert.equal(reports.length, 1);
182             const submittedContent = emptyReport();
183             const storedContent = JSON.parse(reports[0]['content']);
184
185             delete submittedContent['builderPassword'];
186             delete submittedContent['tests'];
187             delete storedContent['tests'];
188             assert.deepEqual(storedContent, submittedContent);
189         });
190     });
191
192     it("should treat the slave password as the builder password if there is no matching slave", () => {
193         let report = emptyReport();
194         report['slavePassword'] = report['builderPassword'];
195         delete report['builderPassword'];
196
197         return addSlaveForReport(report).then(() => {
198             return TestServer.remoteAPI().postJSON('/api/report/', [report]);
199         }).then((response) => {
200             assert.equal(response['status'], 'OK');
201             assert.equal(response['failureStored'], false);
202             assert.equal(response['processedRuns'], 1);
203             return TestServer.database().selectAll('reports');
204         }).then((reports) => {
205             assert.equal(reports.length, 1);
206             const storedContent = JSON.parse(reports[0]['content']);
207
208             delete report['slavePassword'];
209             delete report['tests'];
210             delete storedContent['tests'];
211             assert.deepEqual(storedContent, report);
212         });
213     });
214
215     it("should store a report from a valid slave", () => {
216         return addSlaveForReport(emptySlaveReport()).then(() => {
217             return TestServer.remoteAPI().postJSON('/api/report/', [emptySlaveReport()]);
218         }).then((response) => {
219             assert.equal(response['status'], 'OK');
220             assert.equal(response['failureStored'], false);
221             assert.equal(response['processedRuns'], 1);
222             return TestServer.database().selectAll('reports');
223         }).then((reports) => {
224             assert.equal(reports.length, 1);
225             const submittedContent = emptySlaveReport();
226             const storedContent = JSON.parse(reports[0]['content']);
227
228             delete submittedContent['slavePassword'];
229             delete submittedContent['tests'];
230             delete storedContent['tests'];
231             assert.deepEqual(storedContent, submittedContent);
232         });
233     });
234
235     it("should store the builder name but not the builder password", () => {
236         return addBuilderForReport(emptyReport()).then(() => {
237             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
238         }).then((response) => {
239             return TestServer.database().selectAll('reports');
240         }).then((reports) => {
241             assert.equal(reports.length, 1);
242             const storedContent = JSON.parse(reports[0]['content']);
243             assert.equal(storedContent['builderName'], emptyReport()['builderName']);
244             assert(!('builderPassword' in storedContent));
245         });
246     });
247
248     it("should add a slave if there isn't one and the report was authenticated by a builder", () => {
249         return addBuilderForReport(emptyReport()).then(() => {
250             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
251         }).then((response) => {
252             return TestServer.database().selectAll('build_slaves');
253         }).then((slaves) => {
254             assert.equal(slaves.length, 1);
255             assert.equal(slaves[0]['name'], emptyReport()['slaveName']);
256         });
257     });
258
259     it("should add a builder if there isn't one and the report was authenticated by a slave", () => {
260         return addSlaveForReport(emptySlaveReport()).then(() => {
261             return TestServer.remoteAPI().postJSON('/api/report/', [emptySlaveReport()]);
262         }).then((response) => {
263             return TestServer.database().selectAll('builders');
264         }).then((builders) => {
265             assert.equal(builders.length, 1);
266             assert.equal(builders[0]['name'], emptyReport()['builderName']);
267         });
268     });
269
270     it("should add a build", () => {
271         return addBuilderForReport(emptyReport()).then(() => {
272             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
273         }).then(() => {
274             return TestServer.database().selectAll('builds');
275         }).then((builds) => {
276             assert.strictEqual(builds[0]['number'], 123);
277         });
278     });
279
280     it("should add the platform", () => {
281         return addBuilderForReport(emptyReport()).then(() => {
282             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
283         }).then(() => {
284             return TestServer.database().selectAll('platforms');
285         }).then((platforms) => {
286             assert.equal(platforms.length, 1);
287             assert.equal(platforms[0]['name'], 'Mountain Lion');
288         });
289     });
290
291     it("should add repositories and build revisions", () => {
292         return addBuilderForReport(emptyReport()).then(() => {
293             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
294         }).then((response) => {
295             const db = TestServer.database();
296             return Promise.all([
297                 db.selectAll('repositories'),
298                 db.selectAll('commits'),
299                 db.selectAll('build_commits', 'build_commit'),
300             ]);
301         }).then((result) => {
302             const repositories = result[0];
303             const commits = result[1];
304             const buildCommitsRelations = result[2];
305             assert.equal(repositories.length, 2);
306             assert.deepEqual(repositories.map((row) => row['name']).sort(), ['WebKit', 'macOS']);
307
308             assert.equal(commits.length, 2);
309             assert.equal(buildCommitsRelations.length, 2);
310             assert.equal(buildCommitsRelations[0]['build_commit'], commits[0]['id']);
311             assert.equal(buildCommitsRelations[1]['build_commit'], commits[1]['id']);
312             assert.equal(buildCommitsRelations[0]['commit_build'], buildCommitsRelations[1]['commit_build']);
313
314             let repositoryIdToName = {};
315             for (let repository of repositories)
316                 repositoryIdToName[repository['id']] = repository['name'];
317
318             let repositoryNameToRevisionRow = {};
319             for (let commit of commits)
320                 repositoryNameToRevisionRow[repositoryIdToName[commit['repository']]] = commit;
321
322             assert.equal(repositoryNameToRevisionRow['macOS']['revision'], '10.8.2 12C60');
323             assert.equal(repositoryNameToRevisionRow['WebKit']['revision'], '141977');
324             assert.equal(repositoryNameToRevisionRow['WebKit']['time'].toString(),
325                 new Date('2013-02-06 08:55:20.9').toString());
326         });
327     });
328
329     it("should not create a duplicate build for the same build number if build times are close", () => {
330         const firstReport = emptyReport();
331         firstReport['buildTime'] = '2013-02-28T10:12:04';
332         const secondReport = emptyReport();
333         secondReport['buildTime'] = '2013-02-28T10:22:03';
334
335         return addBuilderForReport(emptyReport()).then(() => {
336             return TestServer.remoteAPI().postJSON('/api/report/', [firstReport]);
337         }).then((response) => {
338             assert.equal(response['status'], 'OK');
339             return TestServer.database().selectAll('builds');
340         }).then((builds) => {
341             assert.equal(builds.length, 1);
342             return TestServer.remoteAPI().postJSON('/api/report/', [secondReport]);
343         }).then((response) => {
344             assert.equal(response['status'], 'OK');
345             return TestServer.database().selectAll('builds');
346         }).then((builds) => {
347             assert.equal(builds.length, 1);
348         });
349     });
350
351     it("should create distinct builds for the same build number if build times are far apart", () => {
352         const firstReport = emptyReport();
353         firstReport['buildTime'] = '2013-02-28T10:12:03';
354         const secondReport = emptyReport();
355         secondReport['buildTime'] = '2014-01-20T22:23:34';
356
357         return addBuilderForReport(emptyReport()).then(() => {
358             return TestServer.remoteAPI().postJSON('/api/report/', [firstReport]);
359         }).then((response) => {
360             assert.equal(response['status'], 'OK');
361             return TestServer.database().selectAll('builds');
362         }).then((builds) => {
363             assert.equal(builds.length, 1);
364             return TestServer.remoteAPI().postJSON('/api/report/', [secondReport]);
365         }).then((response) => {
366             assert.equal(response['status'], 'OK');
367             return TestServer.database().selectAll('builds');
368         }).then((builds) => {
369             assert.equal(builds.length, 2);
370         });
371     });
372
373     it("should reject a report with mismatching revision info", () => {
374         const firstReport = emptyReport();
375         firstReport['revisions'] = {
376             "WebKit": {
377                 "revision": "141977",
378                 "timestamp": "2013-02-06T08:55:20.96Z"
379             }
380         };
381
382         const secondReport = emptyReport();
383         secondReport['revisions'] = {
384             "WebKit": {
385                 "revision": "150000",
386                 "timestamp": "2013-05-13T10:50:29.6Z"
387             }
388         };
389
390         return addBuilderForReport(firstReport).then(() => {
391             return TestServer.remoteAPI().postJSON('/api/report/', [firstReport]);
392         }).then((response) => {
393             assert.equal(response['status'], 'OK');
394             return TestServer.database().selectAll('builds');
395         }).then((builds) => {
396             assert.equal(builds.length, 1);
397             return TestServer.remoteAPI().postJSON('/api/report/', [secondReport]);
398         }).then((response) => {
399             assert.equal(response['status'], 'MismatchingCommitRevision');
400             assert(JSON.stringify(response).indexOf('141977') >= 0);
401             assert(JSON.stringify(response).indexOf('150000') >= 0);
402             assert.equal(response['failureStored'], true);
403             assert.equal(response['processedRuns'], 0);
404         });
405     });
406
407     const reportWithTwoLevelsOfAggregations = {
408         "buildNumber": "123",
409         "buildTime": "2013-02-28T10:12:03.388304",
410         "builderName": "someBuilder",
411         "builderPassword": "somePassword",
412         "platform": "Mountain Lion",
413         "tests": {
414             "DummyPageLoading": {
415                 "metrics": {"Time": { "aggregators" : ["Arithmetic"], "current": [300, 310, 320, 330] }},
416                 "tests": {
417                     "apple.com": {
418                         "metrics": {"Time": { "current": [500, 510, 520, 530] }},
419                         "url": "http://www.apple.com"
420                     },
421                     "webkit.org": {
422                         "metrics": {"Time": { "current": [100, 110, 120, 130] }},
423                         "url": "http://www.webkit.org"
424                     }
425                 }
426             },
427             "DummyBenchmark": {
428                 "metrics": {"Time": ["Arithmetic"]},
429                 "tests": {
430                     "DOM": {
431                         "metrics": {"Time": ["Geometric", "Arithmetic"]},
432                         "tests": {
433                             "ModifyNodes": {"metrics": {"Time": { "current": [[11, 12, 13, 14, 15], [16, 17, 18, 19, 20], [21, 22, 23, 24, 25]] }}},
434                             "TraverseNodes": {"metrics": {"Time": { "current": [[31, 32, 33, 34, 35], [36, 37, 38, 39, 40], [41, 42, 43, 44, 45]] }}}
435                         }
436                     },
437                     "CSS": {"metrics": {"Time": { "current": [[101, 102, 103, 104, 105], [106, 107, 108, 109, 110], [111, 112, 113, 114, 115]] }}}
438                 }
439             }
440         },
441         "revisions": {
442             "macOS": {
443                 "revision": "10.8.2 12C60"
444             },
445             "WebKit": {
446                 "revision": "141977",
447                 "timestamp": "2013-02-06T08:55:20.9Z"
448             }
449         }};
450
451     function reportAfterAddingBuilderAndAggregatorsWithResponse(report)
452     {
453         return addBuilderForReport(report).then(() => {
454             const db = TestServer.database();
455             return Promise.all([
456                 db.insert('aggregators', {name: 'Arithmetic'}),
457                 db.insert('aggregators', {name: 'Geometric'}),
458             ]);
459         }).then(() => TestServer.remoteAPI().postJSON('/api/report/', [report]));
460     }
461
462     function reportAfterAddingBuilderAndAggregators(report)
463     {
464         return reportAfterAddingBuilderAndAggregatorsWithResponse(report).then((response) => {
465             assert.equal(response['status'], 'OK');
466             assert.equal(response['failureStored'], false);
467             return response;
468         });
469     }
470
471     function fetchRunForMetric(testName, metricName,callback) {
472         queryAndFetchAll('SELECT * FROM test_runs WHERE run_config IN'
473             + '(SELECT config_id FROM test_configurations, test_metrics, tests WHERE config_metric = metric_id AND metric_test = test_id AND'
474             + 'test_name = $1 AND metric_name = $2)',
475             ['Arithmetic', 'values.reduce(function (a, b) { return a + b; }) / values.length'], () => {
476             queryAndFetchAll('INSERT INTO aggregators (aggregator_name, aggregator_definition) values ($1, $2)',
477                 ['Geometric', 'Math.pow(values.reduce(function (a, b) { return a * b; }), 1 / values.length)'], callback);
478         });
479     }
480
481     it("should accept a report with aggregators", () => {
482         return reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations);
483     });
484
485     it("should add tests", () => {
486         return reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(() => {
487             return TestServer.database().selectAll('tests');
488         }).then((tests) => {
489             assert.deepEqual(tests.map((row) => { return row['name']; }).sort(),
490                 ['CSS', 'DOM', 'DummyBenchmark', 'DummyPageLoading', 'ModifyNodes', 'TraverseNodes', 'apple.com', 'webkit.org']);
491         });
492     });
493
494     it("should add metrics", () => {
495         return reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(() => {
496             return TestServer.database().query('SELECT * FROM tests, test_metrics LEFT JOIN aggregators ON metric_aggregator = aggregator_id WHERE metric_test = test_id');
497         }).then((result) => {
498             const testNameToMetrics = {};
499             result.rows.forEach((row) => {
500                 if (!(row['test_name'] in testNameToMetrics))
501                     testNameToMetrics[row['test_name']] = new Array;
502                 testNameToMetrics[row['test_name']].push([row['metric_name'], row['aggregator_name']]);
503             });
504             assert.deepEqual(testNameToMetrics['CSS'], [['Time', null]]);
505             assert.deepEqual(testNameToMetrics['DOM'].sort(), [['Time', 'Arithmetic'], ['Time', 'Geometric']]);
506             assert.deepEqual(testNameToMetrics['DummyBenchmark'], [['Time', 'Arithmetic']]);
507             assert.deepEqual(testNameToMetrics['DummyPageLoading'], [['Time', 'Arithmetic']]);
508             assert.deepEqual(testNameToMetrics['ModifyNodes'], [['Time', null]]);
509             assert.deepEqual(testNameToMetrics['TraverseNodes'], [['Time', null]]);
510             assert.deepEqual(testNameToMetrics['apple.com'], [['Time', null]]);
511             assert.deepEqual(testNameToMetrics['webkit.org'], [['Time', null]]);
512         });
513     });
514
515     function fetchTestConfig(testName, metricName)
516     {
517         return TestServer.database().query(`SELECT * FROM tests, test_metrics, test_configurations
518             WHERE test_id = metric_test AND metric_id = config_metric
519             AND test_name = $1 AND metric_name = $2`, [testName, metricName]).then((result) => {
520                 assert.equal(result.rows.length, 1);
521                 return result.rows[0];
522             });
523     }
524
525     function fetchTestRunIterationsForMetric(testName, metricName)
526     {
527         const db = TestServer.database();
528         return fetchTestConfig(testName, metricName).then((config) => {
529             return db.selectFirstRow('test_runs', {config: config['config_id']});
530         }).then((run) => {
531             return db.selectRows('run_iterations', {run: run['id']}, {sortBy: 'order'}).then((iterations) => {
532                 return {run: run, iterations: iterations};
533             });
534         });
535     }
536
537     it("should store run values", () => {
538         return reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(() => {
539             return fetchTestRunIterationsForMetric('apple.com', 'Time');
540         }).then((result) => {
541             const run = result.run;
542             const runId = run['id'];
543             assert.deepEqual(result.iterations, [
544                 {run: runId, order: 0, group: null, value: 500, relative_time: null},
545                 {run: runId, order: 1, group: null, value: 510, relative_time: null},
546                 {run: runId, order: 2, group: null, value: 520, relative_time: null},
547                 {run: runId, order: 3, group: null, value: 530, relative_time: null}]);
548             const sum = 500 + 510 + 520 + 530;
549             assert.equal(run['mean_cache'], sum / result.iterations.length);
550             assert.equal(run['sum_cache'], sum);
551             assert.equal(run['square_sum_cache'], 500 * 500 + 510 * 510 + 520 * 520 + 530 * 530);
552             return fetchTestRunIterationsForMetric('CSS', 'Time');
553         }).then((result) => {
554             const run = result.run;
555             const runId = run['id'];
556             assert.deepEqual(result.iterations, [
557                 {run: runId, order: 0, group: 0, value: 101, relative_time: null},
558                 {run: runId, order: 1, group: 0, value: 102, relative_time: null},
559                 {run: runId, order: 2, group: 0, value: 103, relative_time: null},
560                 {run: runId, order: 3, group: 0, value: 104, relative_time: null},
561                 {run: runId, order: 4, group: 0, value: 105, relative_time: null},
562                 {run: runId, order: 5, group: 1, value: 106, relative_time: null},
563                 {run: runId, order: 6, group: 1, value: 107, relative_time: null},
564                 {run: runId, order: 7, group: 1, value: 108, relative_time: null},
565                 {run: runId, order: 8, group: 1, value: 109, relative_time: null},
566                 {run: runId, order: 9, group: 1, value: 110, relative_time: null},
567                 {run: runId, order: 10, group: 2, value: 111, relative_time: null},
568                 {run: runId, order: 11, group: 2, value: 112, relative_time: null},
569                 {run: runId, order: 12, group: 2, value: 113, relative_time: null},
570                 {run: runId, order: 13, group: 2, value: 114, relative_time: null},
571                 {run: runId, order: 14, group: 2, value: 115, relative_time: null}]);
572             let sum = 0;
573             let squareSum = 0;
574             for (let value = 101; value <= 115; ++value) {
575                 sum += value;
576                 squareSum += value * value;
577             }
578             assert.equal(run['mean_cache'], sum / result.iterations.length);
579             assert.equal(run['sum_cache'], sum);
580             assert.equal(run['square_sum_cache'], squareSum);
581         });
582     });
583
584     it("should store aggregated run values", () => {
585         return reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(() => {
586             return fetchTestRunIterationsForMetric('DummyPageLoading', 'Time');
587         }).then((result) => {
588             const run = result.run;
589             const runId = result.run['id'];
590             const expectedValues = [(500 + 100) / 2, (510 + 110) / 2, (520 + 120) / 2, (530 + 130) / 2];
591             assert.deepEqual(result.iterations, [
592                 {run: runId, order: 0, group: null, value: expectedValues[0], relative_time: null},
593                 {run: runId, order: 1, group: null, value: expectedValues[1], relative_time: null},
594                 {run: runId, order: 2, group: null, value: expectedValues[2], relative_time: null},
595                 {run: runId, order: 3, group: null, value: expectedValues[3], relative_time: null}]);
596             const sum = expectedValues.reduce(function (sum, value) { return sum + value; }, 0);
597             assert.equal(run['mean_cache'], sum / result.iterations.length);
598             assert.equal(run['sum_cache'], sum);
599             assert.equal(run['square_sum_cache'], expectedValues.reduce(function (sum, value) { return sum + value * value; }, 0));
600         });
601     });
602
603     it("should be able to compute the aggregation of aggregated values", () => {
604         return reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(() => {
605             return fetchTestRunIterationsForMetric('DummyBenchmark', 'Time');
606         }).then((result) => {
607             const run = result.run;
608             const runId = run['id'];
609             const expectedIterations = [];
610             let sum = 0;
611             let squareSum = 0;
612             for (let i = 0; i < 15; ++i) {
613                 const value = i + 1;
614                 const DOMMean = ((10 + value) + (30 + value)) / 2;
615                 const expectedValue = (DOMMean + 100 + value) / 2;
616                 sum += expectedValue;
617                 squareSum += expectedValue * expectedValue;
618                 expectedIterations.push({run: runId, order: i, group: Math.floor(i / 5), value: expectedValue, relative_time: null});
619             }
620             assert.deepEqual(result.iterations, expectedIterations);
621             assert.equal(run['mean_cache'], sum / result.iterations.length);
622             assert.equal(run['sum_cache'], sum);
623             assert.equal(run['square_sum_cache'], squareSum);
624         });
625     });
626
627     function makeReport(tests)
628     {
629         return {
630             "buildNumber": "123",
631             "buildTime": "2013-02-28T10:12:03.388304",
632             "builderName": "someBuilder",
633             "builderPassword": "somePassword",
634             "platform": "Mountain Lion",
635             tests,
636             "revisions": {
637                 "macOS": {
638                     "revision": "10.8.2 12C60"
639                 },
640                 "WebKit": {
641                     "revision": "141977",
642                     "timestamp": "2013-02-06T08:55:20.9Z"
643                 }
644             }
645         };
646     }
647
648     it("should be able to compute the aggregation of differently aggregated values", async () => {
649         const reportWithDifferentAggregators = makeReport({
650             "DummyBenchmark": {
651                 "metrics": {"Time": ["Arithmetic"]},
652                 "tests": {
653                     "DOM": {
654                         "metrics": {"Time": ["Total"]},
655                         "tests": {
656                             "ModifyNodes": {"metrics": {"Time": { "current": [[1, 2], [3, 4]] }}},
657                             "TraverseNodes": {"metrics": {"Time": { "current": [[11, 12], [13, 14]] }}}
658                         }
659                     },
660                     "CSS": {"metrics": {"Time": { "current": [[21, 22], [23, 24]] }}}
661                 }
662             }
663         });
664
665         await reportAfterAddingBuilderAndAggregators(reportWithDifferentAggregators);
666         const result = await fetchTestRunIterationsForMetric('DummyBenchmark', 'Time');
667
668         const run = result.run;
669         const runId = run['id'];
670         const expectedIterations = [];
671         let sum = 0;
672         let squareSum = 0;
673         for (let i = 0; i < 4; ++i) {
674             const value = i + 1;
675             const DOMTotal = (value + 10 + value);
676             const expectedValue = (DOMTotal + (20 + value)) / 2;
677             sum += expectedValue;
678             squareSum += expectedValue * expectedValue;
679             expectedIterations.push({run: runId, order: i, group: Math.floor(i / 2), value: expectedValue, relative_time: null});
680         }
681         assert.deepEqual(result.iterations, expectedIterations);
682         assert.equal(run['mean_cache'], sum / result.iterations.length);
683         assert.equal(run['sum_cache'], sum);
684         assert.equal(run['square_sum_cache'], squareSum);
685     });
686
687     it("should skip subtests without any metric during aggregation", async () => {
688         const reportWithSubtestMissingMatchingMetric = makeReport({
689             "DummyBenchmark": {
690                 "metrics": {"Time": ["Arithmetic"]},
691                 "tests": {
692                     "DOM": {
693                         "metrics": {"Time": ["Total"]},
694                         "tests": {
695                             "ModifyNodes": {"metrics": {"Time": { "current": [[1, 2], [3, 4]] }}},
696                             "TraverseNodes": {"metrics": {"Time": { "current": [[11, 12], [13, 14]] }}}
697                         }
698                     },
699                     "CSS": {"metrics": {"Time": { "current": [[21, 22], [23, 24]] }}},
700                     "AuxiliaryResult": {
701                         "tests": {
702                             "ASubTest": {
703                                 "metrics": {"Time": {"current": [31, 32]}}
704                             }
705                         }
706                     }
707                 }
708             }
709         });
710
711         await reportAfterAddingBuilderAndAggregators(reportWithSubtestMissingMatchingMetric);
712         const benchmarkResult = await fetchTestRunIterationsForMetric('DummyBenchmark', 'Time');
713
714         let run = benchmarkResult.run.id;
715         assert.equal(benchmarkResult.iterations.length, 4);
716         assert.deepEqual(benchmarkResult.iterations[0], {run, order: 0, group: 0, value: ((1 + 11) + 21) / 2, relative_time: null});
717         assert.deepEqual(benchmarkResult.iterations[1], {run, order: 1, group: 0, value: ((2 + 12) + 22) / 2, relative_time: null});
718         assert.deepEqual(benchmarkResult.iterations[2], {run, order: 2, group: 1, value: ((3 + 13) + 23) / 2, relative_time: null});
719         assert.deepEqual(benchmarkResult.iterations[3], {run, order: 3, group: 1, value: ((4 + 14) + 24) / 2, relative_time: null});
720
721         const sum = benchmarkResult.iterations.reduce((total, row) => total + row.value, 0);
722         const squareSum = benchmarkResult.iterations.reduce((total, row) => total + row.value * row.value, 0);
723         assert.equal(benchmarkResult.run['mean_cache'], sum / 4);
724         assert.equal(benchmarkResult.run['sum_cache'], sum);
725         assert.equal(benchmarkResult.run['square_sum_cache'], squareSum);
726
727         const someTestResult = await fetchTestRunIterationsForMetric('ASubTest', 'Time');
728         run = someTestResult.run.id;
729         assert.deepEqual(someTestResult.iterations, [
730             {run, order: 0, group: null, value: 31, relative_time: null},
731             {run, order: 1, group: null, value: 32, relative_time: null},
732         ]);
733         assert.equal(someTestResult.run['mean_cache'], 31.5);
734         assert.equal(someTestResult.run['sum_cache'], 63);
735         assert.equal(someTestResult.run['square_sum_cache'], 31 * 31 + 32 * 32);
736     });
737
738     it("should skip subtests missing the matching metric during aggregation", async () => {
739         const reportWithSubtestMissingMatchingMetric = makeReport({
740             "DummyBenchmark": {
741                 "metrics": {"Time": ["Arithmetic"]},
742                 "tests": {
743                     "DOM": {
744                         "metrics": {"Time": ["Total"]},
745                         "tests": {
746                             "ModifyNodes": {"metrics": {"Time": { "current": [[1, 2], [3, 4]] }}},
747                             "TraverseNodes": {"metrics": {"Time": { "current": [[11, 12], [13, 14]] }}}
748                         }
749                     },
750                     "CSS": {"metrics": {"Time": { "current": [[21, 22], [23, 24]] }}},
751                     "AuxiliaryResult": {
752                         "metrics": {"Allocations": ["Total"]},
753                         "tests": {
754                             "SomeSubTest": {
755                                 "metrics": {"Allocations": {"current": [31, 32]}}
756                             },
757                             "OtherSubTest": {
758                                 "metrics": {"Allocations": {"current": [41, 42]}}
759                             }
760                         }
761                     }
762                 }
763             }
764         });
765
766         await reportAfterAddingBuilderAndAggregators(reportWithSubtestMissingMatchingMetric);
767         const benchmarkResult = await fetchTestRunIterationsForMetric('DummyBenchmark', 'Time');
768
769         let run = benchmarkResult.run.id;
770         assert.equal(benchmarkResult.iterations.length, 4);
771         assert.deepEqual(benchmarkResult.iterations[0], {run, order: 0, group: 0, value: ((1 + 11) + 21) / 2, relative_time: null});
772         assert.deepEqual(benchmarkResult.iterations[1], {run, order: 1, group: 0, value: ((2 + 12) + 22) / 2, relative_time: null});
773         assert.deepEqual(benchmarkResult.iterations[2], {run, order: 2, group: 1, value: ((3 + 13) + 23) / 2, relative_time: null});
774         assert.deepEqual(benchmarkResult.iterations[3], {run, order: 3, group: 1, value: ((4 + 14) + 24) / 2, relative_time: null});
775
776         const sum = benchmarkResult.iterations.reduce((total, row) => total + row.value, 0);
777         const squareSum = benchmarkResult.iterations.reduce((total, row) => total + row.value * row.value, 0);
778         assert.equal(benchmarkResult.run['mean_cache'], sum / 4);
779         assert.equal(benchmarkResult.run['sum_cache'], sum);
780         assert.equal(benchmarkResult.run['square_sum_cache'], squareSum);
781
782         const someTestResult = await fetchTestRunIterationsForMetric('AuxiliaryResult', 'Allocations');
783         run = someTestResult.run.id;
784         assert.deepEqual(someTestResult.iterations, [
785             {run, order: 0, group: null, value: 31 + 41, relative_time: null},
786             {run, order: 1, group: null, value: 32 + 42, relative_time: null},
787         ]);
788         assert.equal(someTestResult.run['mean_cache'], (31 + 41 + 32 + 42) / 2);
789         assert.equal(someTestResult.run['sum_cache'], 31 + 41 + 32 + 42);
790         assert.equal(someTestResult.run['square_sum_cache'], Math.pow(31 + 41, 2) + Math.pow(32 + 42, 2));
791     });
792
793     it("should reject a report when there are more than one non-matching aggregators in a subtest", async () => {
794         const reportWithAmbigiousAggregators = {
795             "buildNumber": "123",
796             "buildTime": "2013-02-28T10:12:03.388304",
797             "builderName": "someBuilder",
798             "builderPassword": "somePassword",
799             "platform": "Mountain Lion",
800             "tests": {
801                 "DummyBenchmark": {
802                     "metrics": {"Time": ["Arithmetic"]},
803                     "tests": {
804                         "DOM": {
805                             "metrics": {"Time": ["Total", "Geometric"]},
806                             "tests": {
807                                 "ModifyNodes": {"metrics": {"Time": { "current": [[1, 2], [3, 4]] }}},
808                                 "TraverseNodes": {"metrics": {"Time": { "current": [[11, 12], [13, 14]] }}}
809                             }
810                         },
811                         "CSS": {"metrics": {"Time": { "current": [[21, 22], [23, 24]] }}}
812                     }
813                 }
814             },
815             "revisions": {
816                 "macOS": {
817                     "revision": "10.8.2 12C60"
818                 },
819                 "WebKit": {
820                     "revision": "141977",
821                     "timestamp": "2013-02-06T08:55:20.9Z"
822                 }
823             }};
824
825         const response = await reportAfterAddingBuilderAndAggregatorsWithResponse(reportWithAmbigiousAggregators);
826         assert.equal(response['status'], 'NoMatchingAggregatedValueInSubtest');
827     });
828
829     function reportWithSameSubtestName()
830     {
831         return {
832             "buildNumber": "123",
833             "buildTime": "2013-02-28T10:12:03.388304",
834             "builderName": "someBuilder",
835             "builderPassword": "somePassword",
836             "platform": "Mountain Lion",
837             "tests": {
838                 "Suite1": {
839                     "tests": {
840                         "test1": {
841                             "metrics": {"Time": { "current": [1, 2, 3, 4, 5] }}
842                         },
843                         "test2": {
844                             "metrics": {"Time": { "current": [6, 7, 8, 9, 10] }}
845                         }
846                     }
847                 },
848                 "Suite2": {
849                     "tests": {
850                         "test1": {
851                             "metrics": {"Time": { "current": [11, 12, 13, 14, 15] }}
852                         },
853                         "test2": {
854                             "metrics": {"Time": { "current": [16, 17, 18, 19, 20] }}
855                         }
856                     }
857                 }
858             }
859         };
860     }
861
862     it("should be able to add a report with same subtest name", () => {
863         return reportAfterAddingBuilderAndAggregators(reportWithSameSubtestName());
864     });
865
866     it("should be able to reuse the same test rows", () => {
867         return reportAfterAddingBuilderAndAggregators(reportWithSameSubtestName()).then(() => {
868             return TestServer.database().selectAll('tests');
869         }).then((tests) => {
870             assert.equal(tests.length, 6);
871             let newReport = reportWithSameSubtestName();
872             newReport.buildNumber = "125";
873             newReport.buildTime = "2013-02-28T12:17:24.1";
874             return TestServer.remoteAPI().postJSON('/api/report/', [newReport]);
875         }).then((response) => {
876             assert.equal(response['status'], 'OK');
877             return TestServer.database().selectAll('tests');
878         }).then((tests) => {
879             assert.equal(tests.length, 6);
880         });
881     });
882
883     const reportWithSameSingleValue = {
884         "buildNumber": "123",
885         "buildTime": "2013-02-28T10:12:03.388304",
886         "builderName": "someBuilder",
887         "builderPassword": "somePassword",
888         "platform": "Mountain Lion",
889         "tests": {
890             "suite": {
891                 "metrics": {"Combined": ["Arithmetic"]},
892                 "tests": {
893                     "test1": {
894                         "metrics": {"Combined": { "current": 3 }}
895                     },
896                     "test2": {
897                         "metrics": {"Combined": { "current": 7 }}
898                     }
899                 }
900             },
901         }};
902
903     it("should be able to add a report with single value results", () => {
904         return reportAfterAddingBuilderAndAggregators(reportWithSameSingleValue).then(() => {
905             return fetchTestRunIterationsForMetric('test1', 'Combined');
906         }).then((result) => {
907             const run = result.run;
908             assert.equal(run['iteration_count_cache'], 1);
909             assert.equal(run['mean_cache'], 3);
910             assert.equal(run['sum_cache'], 3);
911             assert.equal(run['square_sum_cache'], 9);
912             return fetchTestRunIterationsForMetric('suite', 'Combined');
913         }).then((result) => {
914             const run = result.run;
915             assert.equal(run['iteration_count_cache'], 1);
916             assert.equal(run['mean_cache'], 5);
917             assert.equal(run['sum_cache'], 5);
918             assert.equal(run['square_sum_cache'], 25);
919         });
920     });
921
922     const reportWithSameValuePairs = {
923         "buildNumber": "123",
924         "buildTime": "2013-02-28T10:12:03.388304",
925         "builderName": "someBuilder",
926         "builderPassword": "somePassword",
927         "platform": "Mountain Lion",
928         "tests": {
929                 "test": {
930                     "metrics": {"FrameRate": { "current": [[[0, 4], [100, 5], [205, 3]]] }}
931                 },
932             },
933         };
934
935     it("should be able to add a report with (relative time, value) pairs", () => {
936         return reportAfterAddingBuilderAndAggregators(reportWithSameValuePairs).then(() => {
937             return fetchTestRunIterationsForMetric('test', 'FrameRate');
938         }).then((result) => {
939             const run = result.run;
940             assert.equal(run['iteration_count_cache'], 3);
941             assert.equal(run['mean_cache'], 4);
942             assert.equal(run['sum_cache'], 12);
943             assert.equal(run['square_sum_cache'], 16 + 25 + 9);
944
945             const runId = run['id'];
946             assert.deepEqual(result.iterations, [
947                 {run: runId, order: 0, group: null, value: 4, relative_time: 0},
948                 {run: runId, order: 1, group: null, value: 5, relative_time: 100},
949                 {run: runId, order: 2, group: null, value: 3, relative_time: 205}]);
950         });
951     });
952
953     const reportsUpdatingDifferentTests = [
954         {
955             "buildNumber": "123",
956             "buildTime": "2013-02-28T10:12:03",
957             "builderName": "someBuilder",
958             "builderPassword": "somePassword",
959             "platform": "Mountain Lion",
960             "tests": {"test1": {"metrics": {"Time": {"current": 3}}}}
961         },
962         {
963             "buildNumber": "124",
964             "buildTime": "2013-02-28T11:31:21",
965             "builderName": "someBuilder",
966             "builderPassword": "somePassword",
967             "platform": "Mountain Lion",
968             "tests": {"test2": {"metrics": {"Time": {"current": 3}}}}
969         },
970         {
971             "buildNumber": "125",
972             "buildTime": "2013-02-28T12:45:34",
973             "builderName": "someBuilder",
974             "builderPassword": "somePassword",
975             "platform": "Mountain Lion",
976             "tests": {"test1": {"metrics": {"Time": {"current": 3}}}}
977         },
978     ];
979
980     it("should update the last modified date of test configurations with new runs", () => {
981         return addBuilderForReport(reportsUpdatingDifferentTests[0]).then(() => {
982             return TestServer.remoteAPI().postJSON('/api/report/', [reportsUpdatingDifferentTests[0]]);
983         }).then((response) => {
984             assert.equal(response['status'], 'OK');
985             return fetchTestConfig('test1', 'Time');
986         }).then((originalConfig) => {
987             return TestServer.remoteAPI().postJSON('/api/report/', [reportsUpdatingDifferentTests[2]]).then(() => {
988                 return fetchTestConfig('test1', 'Time');
989             }).then((config) => {
990                 assert(originalConfig['config_runs_last_modified'] instanceof Date);
991                 assert(config['config_runs_last_modified'] instanceof Date);
992                 assert(+originalConfig['config_runs_last_modified'] < +config['config_runs_last_modified']);
993             });
994         });
995     });
996
997     it("should not update the last modified date of unrelated test configurations", () => {
998         return addBuilderForReport(reportsUpdatingDifferentTests[0]).then(() => {
999             return TestServer.remoteAPI().postJSON('/api/report/', [reportsUpdatingDifferentTests[0]]);
1000         }).then((response) => {
1001             assert.equal(response['status'], 'OK');
1002             return fetchTestConfig('test1', 'Time');
1003         }).then((originalConfig) => {
1004             return TestServer.remoteAPI().postJSON('/api/report/', [reportsUpdatingDifferentTests[1]]).then((response) => {
1005                 assert.equal(response['status'], 'OK');
1006                 return fetchTestConfig('test1', 'Time');
1007             }).then((config) => {
1008                 assert(originalConfig['config_runs_last_modified'] instanceof Date);
1009                 assert(config['config_runs_last_modified'] instanceof Date);
1010                 assert.equal(+originalConfig['config_runs_last_modified'], +config['config_runs_last_modified']);
1011             });
1012         });
1013     });
1014
1015     const reportWithBuildRequest = {
1016         "buildNumber": "123",
1017         "buildTime": "2013-02-28T10:12:03.388304",
1018         "builderName": "someBuilder",
1019         "builderPassword": "somePassword",
1020         "platform": "Mountain Lion",
1021         "buildRequest": "700",
1022         "tests": {
1023             "test": {
1024                 "metrics": {"FrameRate": { "current": [[[0, 4], [100, 5], [205, 3]]] }}
1025             },
1026         },
1027     };
1028
1029     const anotherReportWithSameBuildRequest = {
1030         "buildNumber": "124",
1031         "buildTime": "2013-02-28T10:12:03.388304",
1032         "builderName": "someBuilder",
1033         "builderPassword": "somePassword",
1034         "platform": "Lion",
1035         "buildRequest": "700",
1036         "tests": {
1037             "test": {
1038                 "metrics": {"FrameRate": { "current": [[[0, 4], [100, 5], [205, 3]]] }}
1039             },
1040         },
1041     };
1042
1043     it("should allow to report a build request", () => {
1044         return MockData.addMockData(TestServer.database()).then(() => {
1045             return reportAfterAddingBuilderAndAggregatorsWithResponse(reportWithBuildRequest);
1046         }).then((response) => {
1047             assert.equal(response['status'], 'OK');
1048         });
1049     });
1050
1051     it("should reject the report if the build request in the report has an existing associated build", () => {
1052         return MockData.addMockData(TestServer.database()).then(() => {
1053             return reportAfterAddingBuilderAndAggregatorsWithResponse(reportWithBuildRequest);
1054         }).then((response) => {
1055             assert.equal(response['status'], 'OK');
1056             return TestServer.database().selectRows('builds', {number: '123'});
1057         }).then((results) => {
1058             assert.equal(results.length, 1);
1059             return TestServer.database().selectRows('platforms', {name: 'Mountain Lion'});
1060         }).then((results) => {
1061             assert.equal(results.length, 1);
1062             return TestServer.remoteAPI().postJSON('/api/report/', [anotherReportWithSameBuildRequest]);
1063         }).then((response) => {
1064             assert.equal(response['status'], 'FailedToUpdateBuildRequest');
1065             return TestServer.database().selectRows('builds', {number: '124'});
1066         }).then((results) => {
1067             assert.equal(results.length, 0);
1068             return TestServer.database().selectRows('platforms', {name: 'Lion'});
1069         }).then((results) => {
1070             assert.equal(results.length, 0);
1071         });
1072     });
1073
1074 });