Websites/perf.webkit.org/server-tests/api-report-tests.js

   1 'use strict';
   2
   3 const assert = require('assert');
   4
   5 const TestServer = require('./resources/test-server.js');
   6 const addBuilderForReport = require('./resources/common-operations.js').addBuilderForReport;
   7 const addSlaveForReport = require('./resources/common-operations.js').addSlaveForReport;
   8 const connectToDatabaseInEveryTest = require('./resources/common-operations.js').connectToDatabaseInEveryTest;
   9
  10 describe("/api/report", function () {
  11     this.timeout(1000);
  12     TestServer.inject();
  13     connectToDatabaseInEveryTest();
  14
  15     function emptyReport()
  16     {
  17         return {
  18             "buildNumber": "123",
  19             "buildTime": "2013-02-28T10:12:03.388304",
  20             "builderName": "someBuilder",
  21             "slaveName": "someSlave",
  22             "builderPassword": "somePassword",
  23             "platform": "Mountain Lion",
  24             "tests": {},
  25             "revisions": {
  26                 "OS X": {
  27                     "revision": "10.8.2 12C60"
  28                 },
  29                 "WebKit": {
  30                     "revision": "141977",
  31                     "timestamp": "2013-02-06T08:55:20.9Z"
  32                 }
  33             }
  34         };
  35     }
  36
  37     function emptySlaveReport()
  38     {
  39         return {
  40             "buildNumber": "123",
  41             "buildTime": "2013-02-28T10:12:03.388304",
  42             "builderName": "someBuilder",
  43             "slaveName": "someSlave",
  44             "slavePassword": "otherPassword",
  45             "platform": "Mountain Lion",
  46             "tests": {},
  47             "revisions": {
  48                 "OS X": {
  49                     "revision": "10.8.2 12C60"
  50                 },
  51                 "WebKit": {
  52                     "revision": "141977",
  53                     "timestamp": "2013-02-06T08:55:20.9Z"
  54                 }
  55             }
  56         };
  57     }
  58
  59     it("should reject error when builder name is missing", function (done) {
  60         TestServer.remoteAPI().postJSON('/api/report/', [{"buildTime": "2013-02-28T10:12:03.388304"}]).then(function (response) {
  61             assert.equal(response['status'], 'MissingBuilderName');
  62             done();
  63         }).catch(done);
  64     });
  65
  66     it("should reject error when build time is missing", function (done) {
  67         addBuilderForReport(emptyReport()).then(function () {
  68             return TestServer.remoteAPI().postJSON('/api/report/', [{"builderName": "someBuilder", "builderPassword": "somePassword"}]);
  69         }).then(function (response) {
  70             assert.equal(response['status'], 'MissingBuildTime');
  71             done();
  72         });
  73     });
  74
  75     it("should reject when there are no builders", function (done) {
  76         TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]).then(function (response) {
  77             assert.equal(response['status'], 'BuilderNotFound');
  78             assert.equal(response['failureStored'], false);
  79             assert.equal(response['processedRuns'], 0);
  80             return TestServer.database().selectAll('reports');
  81         }).then(function (reports) {
  82             assert.equal(reports.length, 0);
  83             done();
  84         }).catch(done);
  85     });
  86
  87     it("should reject a report without a builder password", function (done) {
  88         addBuilderForReport(emptyReport()).then(function () {
  89             var report = [{
  90                 "buildNumber": "123",
  91                 "buildTime": "2013-02-28T10:12:03.388304",
  92                 "builderName": "someBuilder",
  93                 "tests": {},
  94                 "revisions": {}}];
  95             return TestServer.remoteAPI().postJSON('/api/report/', report);
  96         }).then(function (response) {
  97             assert.equal(response['status'], 'BuilderNotFound');
  98             assert.equal(response['failureStored'], false);
  99             assert.equal(response['processedRuns'], 0);
 100             return TestServer.database().selectAll('reports');
 101         }).then(function (reports) {
 102             assert.equal(reports.length, 0);
 103             done();
 104         }).catch(done);
 105     });
 106
 107     it("should store a report from a valid builder", function (done) {
 108         addBuilderForReport(emptyReport()).then(function () {
 109             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
 110         }).then(function (response) {
 111             assert.equal(response['status'], 'OK');
 112             assert.equal(response['failureStored'], false);
 113             assert.equal(response['processedRuns'], 1);
 114             return TestServer.database().selectAll('reports');
 115         }).then(function (reports) {
 116             assert.equal(reports.length, 1);
 117             const submittedContent = emptyReport();
 118             const storedContent = JSON.parse(reports[0]['content']);
 119
 120             delete submittedContent['builderPassword'];
 121             delete submittedContent['tests'];
 122             delete storedContent['tests'];
 123             assert.deepEqual(storedContent, submittedContent);
 124
 125             done();
 126         }).catch(done);
 127     });
 128
 129     it("should treat the slave password as the builder password if there is no matching slave", function (done) {
 130         let report = emptyReport();
 131         report['slavePassword'] = report['builderPassword'];
 132         delete report['builderPassword'];
 133
 134         addSlaveForReport(report).then(function () {
 135             return TestServer.remoteAPI().postJSON('/api/report/', [report]);
 136         }).then(function (response) {
 137             assert.equal(response['status'], 'OK');
 138             assert.equal(response['failureStored'], false);
 139             assert.equal(response['processedRuns'], 1);
 140             return TestServer.database().selectAll('reports');
 141         }).then(function (reports) {
 142             assert.equal(reports.length, 1);
 143             const storedContent = JSON.parse(reports[0]['content']);
 144
 145             delete report['slavePassword'];
 146             delete report['tests'];
 147             delete storedContent['tests'];
 148             assert.deepEqual(storedContent, report);
 149
 150             done();
 151         }).catch(done);
 152     });
 153
 154     it("should store a report from a valid slave", function (done) {
 155         addSlaveForReport(emptySlaveReport()).then(function () {
 156             return TestServer.remoteAPI().postJSON('/api/report/', [emptySlaveReport()]);
 157         }).then(function (response) {
 158             assert.equal(response['status'], 'OK');
 159             assert.equal(response['failureStored'], false);
 160             assert.equal(response['processedRuns'], 1);
 161             return TestServer.database().selectAll('reports');
 162         }).then(function (reports) {
 163             assert.equal(reports.length, 1);
 164             const submittedContent = emptySlaveReport();
 165             const storedContent = JSON.parse(reports[0]['content']);
 166
 167             delete submittedContent['slavePassword'];
 168             delete submittedContent['tests'];
 169             delete storedContent['tests'];
 170             assert.deepEqual(storedContent, submittedContent);
 171
 172             done();
 173         }).catch(done);
 174     });
 175
 176     it("should store the builder name but not the builder password", function (done) {
 177         addBuilderForReport(emptyReport()).then(function () {
 178             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
 179         }).then(function (response) {
 180             return TestServer.database().selectAll('reports');
 181         }).then(function (reports) {
 182             assert.equal(reports.length, 1);
 183             const storedContent = JSON.parse(reports[0]['content']);
 184             assert.equal(storedContent['builderName'], emptyReport()['builderName']);
 185             assert(!('builderPassword' in storedContent));
 186             done();
 187         }).catch(done);
 188     });
 189
 190     it("should add a slave if there isn't one and the report was authenticated by a builder", function (done) {
 191         addBuilderForReport(emptyReport()).then(function () {
 192             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
 193         }).then(function (response) {
 194             return TestServer.database().selectAll('build_slaves');
 195         }).then(function (slaves) {
 196             assert.equal(slaves.length, 1);
 197             assert.equal(slaves[0]['name'], emptyReport()['slaveName']);
 198             done();
 199         }).catch(done);
 200     });
 201
 202     it("should add a builder if there isn't one and the report was authenticated by a slave", function (done) {
 203         addSlaveForReport(emptySlaveReport()).then(function () {
 204             return TestServer.remoteAPI().postJSON('/api/report/', [emptySlaveReport()]);
 205         }).then(function (response) {
 206             return TestServer.database().selectAll('builders');
 207         }).then(function (builders) {
 208             assert.equal(builders.length, 1);
 209             assert.equal(builders[0]['name'], emptyReport()['builderName']);
 210             done();
 211         }).catch(done);
 212     });
 213
 214     it("should add a build", function (done) {
 215         addBuilderForReport(emptyReport()).then(function () {
 216             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
 217         }).then(function () {
 218             return TestServer.database().selectAll('builds');
 219         }).then(function (builds) {
 220             assert.strictEqual(builds[0]['number'], 123);
 221             done();
 222         }).catch(done);
 223     });
 224
 225     it("should add the platform", function (done) {
 226         addBuilderForReport(emptyReport()).then(function () {
 227             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
 228         }).then(function () {
 229             return TestServer.database().selectAll('platforms');
 230         }).then(function (platforms) {
 231             assert.equal(platforms.length, 1);
 232             assert.equal(platforms[0]['name'], 'Mountain Lion');
 233             done();
 234         }).catch(done);
 235     });
 236
 237     it("should add repositories and build revisions", function (done) {
 238         addBuilderForReport(emptyReport()).then(function () {
 239             return TestServer.remoteAPI().postJSON('/api/report/', [emptyReport()]);
 240         }).then(function (response) {
 241             const db = TestServer.database();
 242             return Promise.all([
 243                 db.selectAll('repositories'),
 244                 db.selectAll('commits'),
 245                 db.selectAll('build_commits', 'build_commit'),
 246             ]);
 247         }).then(function (result) {
 248             const repositories = result[0];
 249             const commits = result[1];
 250             const buildCommitsRelations = result[2];
 251             assert.equal(repositories.length, 2);
 252             assert.deepEqual(repositories.map(function (row) { return row['name']; }).sort(), ['OS X', 'WebKit']);
 253
 254             assert.equal(commits.length, 2);
 255             assert.equal(buildCommitsRelations.length, 2);
 256             assert.equal(buildCommitsRelations[0]['build_commit'], commits[0]['id']);
 257             assert.equal(buildCommitsRelations[1]['build_commit'], commits[1]['id']);
 258             assert.equal(buildCommitsRelations[0]['commit_build'], buildCommitsRelations[1]['commit_build']);
 259
 260             let repositoryIdToName = {};
 261             for (let repository of repositories)
 262                 repositoryIdToName[repository['id']] = repository['name'];
 263
 264             let repositoryNameToRevisionRow = {};
 265             for (let commit of commits)
 266                 repositoryNameToRevisionRow[repositoryIdToName[commit['repository']]] = commit;
 267
 268             assert.equal(repositoryNameToRevisionRow['OS X']['revision'], '10.8.2 12C60');
 269             assert.equal(repositoryNameToRevisionRow['WebKit']['revision'], '141977');
 270             assert.equal(repositoryNameToRevisionRow['WebKit']['time'].toString(),
 271                 new Date('2013-02-06 08:55:20.9').toString());
 272             done();
 273         }).catch(done);
 274     });
 275
 276     it("should not create a duplicate build for the same build number if build times are close", function (done) {
 277         let firstReport = emptyReport();
 278         firstReport['buildTime'] = '2013-02-28T10:12:04';
 279         let secondReport = emptyReport();
 280         secondReport['buildTime'] = '2013-02-28T10:22:03';
 281
 282         addBuilderForReport(emptyReport()).then(function () {
 283             return TestServer.remoteAPI().postJSON('/api/report/', [firstReport]);
 284         }).then(function (response) {
 285             assert.equal(response['status'], 'OK');
 286             return TestServer.database().selectAll('builds');
 287         }).then(function (builds) {
 288             assert.equal(builds.length, 1);
 289             return TestServer.remoteAPI().postJSON('/api/report/', [secondReport]);
 290         }).then(function (response) {
 291             assert.equal(response['status'], 'OK');
 292             return TestServer.database().selectAll('builds');
 293         }).then(function (builds) {
 294             assert.equal(builds.length, 1);
 295             done();
 296         }).catch(done);
 297     });
 298
 299     it("should create distinct builds for the same build number if build times are far apart", function (done) {
 300         let firstReport = emptyReport();
 301         firstReport['buildTime'] = '2013-02-28T10:12:03';
 302         let secondReport = emptyReport();
 303         secondReport['buildTime'] = '2014-01-20T22:23:34';
 304
 305         addBuilderForReport(emptyReport()).then(function () {
 306             return TestServer.remoteAPI().postJSON('/api/report/', [firstReport]);
 307         }).then(function (response) {
 308             assert.equal(response['status'], 'OK');
 309             return TestServer.database().selectAll('builds');
 310         }).then(function (builds) {
 311             assert.equal(builds.length, 1);
 312             return TestServer.remoteAPI().postJSON('/api/report/', [secondReport]);
 313         }).then(function (response) {
 314             assert.equal(response['status'], 'OK');
 315             return TestServer.database().selectAll('builds');
 316         }).then(function (builds) {
 317             assert.equal(builds.length, 2);
 318             done();
 319         }).catch(done);
 320     });
 321
 322     it("should reject a report with mismatching revision info", function (done) {
 323         let firstReport = emptyReport();
 324         firstReport['revisions'] = {
 325             "WebKit": {
 326                 "revision": "141977",
 327                 "timestamp": "2013-02-06T08:55:20.96Z"
 328             }
 329         };
 330
 331         let secondReport = emptyReport();
 332         secondReport['revisions'] = {
 333             "WebKit": {
 334                 "revision": "150000",
 335                 "timestamp": "2013-05-13T10:50:29.6Z"
 336             }
 337         };
 338
 339         addBuilderForReport(firstReport).then(function () {
 340             return TestServer.remoteAPI().postJSON('/api/report/', [firstReport]);
 341         }).then(function (response) {
 342             assert.equal(response['status'], 'OK');
 343             return TestServer.database().selectAll('builds');
 344         }).then(function (builds) {
 345             assert.equal(builds.length, 1);
 346             return TestServer.remoteAPI().postJSON('/api/report/', [secondReport]);
 347         }).then(function (response) {
 348             assert.equal(response['status'], 'MismatchingCommitRevision');
 349             assert(JSON.stringify(response).indexOf('141977') >= 0);
 350             assert(JSON.stringify(response).indexOf('150000') >= 0);
 351             assert.equal(response['failureStored'], true);
 352             assert.equal(response['processedRuns'], 0);
 353             done();
 354         }).catch(done);
 355     });
 356
 357     const reportWithTwoLevelsOfAggregations = {
 358         "buildNumber": "123",
 359         "buildTime": "2013-02-28T10:12:03.388304",
 360         "builderName": "someBuilder",
 361         "builderPassword": "somePassword",
 362         "platform": "Mountain Lion",
 363         "tests": {
 364             "DummyPageLoading": {
 365                 "metrics": {"Time": { "aggregators" : ["Arithmetic"], "current": [300, 310, 320, 330] }},
 366                 "tests": {
 367                     "apple.com": {
 368                         "metrics": {"Time": { "current": [500, 510, 520, 530] }},
 369                         "url": "http://www.apple.com"
 370                     },
 371                     "webkit.org": {
 372                         "metrics": {"Time": { "current": [100, 110, 120, 130] }},
 373                         "url": "http://www.webkit.org"
 374                     }
 375                 }
 376             },
 377             "DummyBenchmark": {
 378                 "metrics": {"Time": ["Arithmetic"]},
 379                 "tests": {
 380                     "DOM": {
 381                         "metrics": {"Time": ["Geometric", "Arithmetic"]},
 382                         "tests": {
 383                             "ModifyNodes": {"metrics": {"Time": { "current": [[11, 12, 13, 14, 15], [16, 17, 18, 19, 20], [21, 22, 23, 24, 25]] }}},
 384                             "TraverseNodes": {"metrics": {"Time": { "current": [[31, 32, 33, 34, 35], [36, 37, 38, 39, 40], [41, 42, 43, 44, 45]] }}}
 385                         }
 386                     },
 387                     "CSS": {"metrics": {"Time": { "current": [[101, 102, 103, 104, 105], [106, 107, 108, 109, 110], [111, 112, 113, 114, 115]] }}}
 388                 }
 389             }
 390         },
 391         "revisions": {
 392             "OS X": {
 393                 "revision": "10.8.2 12C60"
 394             },
 395             "WebKit": {
 396                 "revision": "141977",
 397                 "timestamp": "2013-02-06T08:55:20.9Z"
 398             }
 399         }};
 400
 401     function reportAfterAddingBuilderAndAggregators(report)
 402     {
 403         return addBuilderForReport(report).then(function () {
 404             const db = TestServer.database();
 405             return Promise.all([
 406                 db.insert('aggregators', {name: 'Arithmetic'}),
 407                 db.insert('aggregators', {name: 'Geometric'}),
 408             ]);
 409         }).then(function () {
 410             return TestServer.remoteAPI().postJSON('/api/report/', [report]);
 411         }).then(function (response) {
 412             assert.equal(response['status'], 'OK');
 413             assert.equal(response['failureStored'], false);
 414             return response;
 415         });
 416     }
 417
 418     function fetchRunForMetric(testName, metricName,callback) {
 419         queryAndFetchAll('SELECT * FROM test_runs WHERE run_config IN'
 420             + '(SELECT config_id FROM test_configurations, test_metrics, tests WHERE config_metric = metric_id AND metric_test = test_id AND'
 421             + 'test_name = $1 AND metric_name = $2)',
 422             ['Arithmetic', 'values.reduce(function (a, b) { return a + b; }) / values.length'], function () {
 423             queryAndFetchAll('INSERT INTO aggregators (aggregator_name, aggregator_definition) values ($1, $2)',
 424                 ['Geometric', 'Math.pow(values.reduce(function (a, b) { return a * b; }), 1 / values.length)'], callback);
 425         });
 426     }
 427
 428     it("should accept a report with aggregators", function (done) {
 429         reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(function () {
 430             done();
 431         }).catch(done);
 432     });
 433
 434     it("should add tests", function (done) {
 435         reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(function () {
 436             return TestServer.database().selectAll('tests');
 437         }).then(function (tests) {
 438             assert.deepEqual(tests.map(function (row) { return row['name']; }).sort(),
 439                 ['CSS', 'DOM', 'DummyBenchmark', 'DummyPageLoading', 'ModifyNodes', 'TraverseNodes', 'apple.com', 'webkit.org']);
 440             done();
 441         }).catch(done);
 442     });
 443
 444     it("should add metrics", function (done) {
 445         reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(function () {
 446             return TestServer.database().query('SELECT * FROM tests, test_metrics LEFT JOIN aggregators ON metric_aggregator = aggregator_id WHERE metric_test = test_id');
 447         }).then(function (result) {
 448             let testNameToMetrics = {};
 449             result.rows.forEach(function (row) {
 450                 if (!(row['test_name'] in testNameToMetrics))
 451                     testNameToMetrics[row['test_name']] = new Array;
 452                 testNameToMetrics[row['test_name']].push([row['metric_name'], row['aggregator_name']]);
 453             });
 454             assert.deepEqual(testNameToMetrics['CSS'], [['Time', null]]);
 455             assert.deepEqual(testNameToMetrics['DOM'].sort(), [['Time', 'Arithmetic'], ['Time', 'Geometric']]);
 456             assert.deepEqual(testNameToMetrics['DummyBenchmark'], [['Time', 'Arithmetic']]);
 457             assert.deepEqual(testNameToMetrics['DummyPageLoading'], [['Time', 'Arithmetic']]);
 458             assert.deepEqual(testNameToMetrics['ModifyNodes'], [['Time', null]]);
 459             assert.deepEqual(testNameToMetrics['TraverseNodes'], [['Time', null]]);
 460             assert.deepEqual(testNameToMetrics['apple.com'], [['Time', null]]);
 461             assert.deepEqual(testNameToMetrics['webkit.org'], [['Time', null]]);
 462             done();
 463         }).catch(done);
 464     });
 465
 466     function fetchTestConfig(testName, metricName)
 467     {
 468         return TestServer.database().query(`SELECT * FROM tests, test_metrics, test_configurations
 469             WHERE test_id = metric_test AND metric_id = config_metric
 470             AND test_name = $1 AND metric_name = $2`, [testName, metricName]).then(function (result) {
 471                 assert.equal(result.rows.length, 1);
 472                 return result.rows[0];
 473             });
 474     }
 475
 476     function fetchTestRunIterationsForMetric(testName, metricName)
 477     {
 478         const db = TestServer.database();
 479         return fetchTestConfig(testName, metricName).then(function (config) {
 480             return db.selectFirstRow('test_runs', {config: config['config_id']});
 481         }).then(function (run) {
 482             return db.selectRows('run_iterations', {run: run['id']}, {sortBy: 'order'}).then(function (iterations) {
 483                 return {run: run, iterations: iterations};
 484             });
 485         });
 486     }
 487
 488     it("should store run values", function (done) {
 489         reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(function () {
 490             return fetchTestRunIterationsForMetric('apple.com', 'Time');
 491         }).then(function (result) {
 492             const run = result.run;
 493             const runId = run['id'];
 494             assert.deepEqual(result.iterations, [
 495                 {run: runId, order: 0, group: null, value: 500, relative_time: null},
 496                 {run: runId, order: 1, group: null, value: 510, relative_time: null},
 497                 {run: runId, order: 2, group: null, value: 520, relative_time: null},
 498                 {run: runId, order: 3, group: null, value: 530, relative_time: null}]);
 499             var sum = 500 + 510 + 520 + 530;
 500             assert.equal(run['mean_cache'], sum / result.iterations.length);
 501             assert.equal(run['sum_cache'], sum);
 502             assert.equal(run['square_sum_cache'], 500 * 500 + 510 * 510 + 520 * 520 + 530 * 530);
 503             return fetchTestRunIterationsForMetric('CSS', 'Time');
 504         }).then(function (result) {
 505             const run = result.run;
 506             const runId = run['id'];
 507             assert.deepEqual(result.iterations, [
 508                 {run: runId, order: 0, group: 0, value: 101, relative_time: null},
 509                 {run: runId, order: 1, group: 0, value: 102, relative_time: null},
 510                 {run: runId, order: 2, group: 0, value: 103, relative_time: null},
 511                 {run: runId, order: 3, group: 0, value: 104, relative_time: null},
 512                 {run: runId, order: 4, group: 0, value: 105, relative_time: null},
 513                 {run: runId, order: 5, group: 1, value: 106, relative_time: null},
 514                 {run: runId, order: 6, group: 1, value: 107, relative_time: null},
 515                 {run: runId, order: 7, group: 1, value: 108, relative_time: null},
 516                 {run: runId, order: 8, group: 1, value: 109, relative_time: null},
 517                 {run: runId, order: 9, group: 1, value: 110, relative_time: null},
 518                 {run: runId, order: 10, group: 2, value: 111, relative_time: null},
 519                 {run: runId, order: 11, group: 2, value: 112, relative_time: null},
 520                 {run: runId, order: 12, group: 2, value: 113, relative_time: null},
 521                 {run: runId, order: 13, group: 2, value: 114, relative_time: null},
 522                 {run: runId, order: 14, group: 2, value: 115, relative_time: null}]);
 523             let sum = 0;
 524             let squareSum = 0;
 525             for (let value = 101; value <= 115; ++value) {
 526                 sum += value;
 527                 squareSum += value * value;
 528             }
 529             assert.equal(run['mean_cache'], sum / result.iterations.length);
 530             assert.equal(run['sum_cache'], sum);
 531             assert.equal(run['square_sum_cache'], squareSum);
 532             done();
 533         }).catch(done);
 534     });
 535
 536     it("should store aggregated run values", function (done) {
 537         reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(function () {
 538             return fetchTestRunIterationsForMetric('DummyPageLoading', 'Time');
 539         }).then(function (result) {
 540             const run = result.run;
 541             const runId = result.run['id'];
 542             const expectedValues = [(500 + 100) / 2, (510 + 110) / 2, (520 + 120) / 2, (530 + 130) / 2];
 543             assert.deepEqual(result.iterations, [
 544                 {run: runId, order: 0, group: null, value: expectedValues[0], relative_time: null},
 545                 {run: runId, order: 1, group: null, value: expectedValues[1], relative_time: null},
 546                 {run: runId, order: 2, group: null, value: expectedValues[2], relative_time: null},
 547                 {run: runId, order: 3, group: null, value: expectedValues[3], relative_time: null}]);
 548             const sum = expectedValues.reduce(function (sum, value) { return sum + value; }, 0);
 549             assert.equal(run['mean_cache'], sum / result.iterations.length);
 550             assert.equal(run['sum_cache'], sum);
 551             assert.equal(run['square_sum_cache'], expectedValues.reduce(function (sum, value) { return sum + value * value; }, 0));
 552             done();
 553         }).catch(done);
 554     });
 555
 556     it("should be able to compute the aggregation of aggregated values", function (done) {
 557         reportAfterAddingBuilderAndAggregators(reportWithTwoLevelsOfAggregations).then(function () {
 558             return fetchTestRunIterationsForMetric('DummyBenchmark', 'Time');
 559         }).then(function (result) {
 560             const run = result.run;
 561             const runId = run['id'];
 562             const expectedIterations = [];
 563             let sum = 0;
 564             let squareSum = 0;
 565             for (let i = 0; i < 15; ++i) {
 566                 const value = i + 1;
 567                 const DOMMean = ((10 + value) + (30 + value)) / 2;
 568                 const expectedValue = (DOMMean + 100 + value) / 2;
 569                 sum += expectedValue;
 570                 squareSum += expectedValue * expectedValue;
 571                 expectedIterations.push({run: runId, order: i, group: Math.floor(i / 5), value: expectedValue, relative_time: null});
 572             }
 573             assert.deepEqual(result.iterations, expectedIterations);
 574             assert.equal(run['mean_cache'], sum / result.iterations.length);
 575             assert.equal(run['sum_cache'], sum);
 576             assert.equal(run['square_sum_cache'], squareSum);
 577             done();
 578         }).catch(done);
 579     });
 580
 581     function reportWithSameSubtestName()
 582     {
 583         return {
 584             "buildNumber": "123",
 585             "buildTime": "2013-02-28T10:12:03.388304",
 586             "builderName": "someBuilder",
 587             "builderPassword": "somePassword",
 588             "platform": "Mountain Lion",
 589             "tests": {
 590                 "Suite1": {
 591                     "tests": {
 592                         "test1": {
 593                             "metrics": {"Time": { "current": [1, 2, 3, 4, 5] }}
 594                         },
 595                         "test2": {
 596                             "metrics": {"Time": { "current": [6, 7, 8, 9, 10] }}
 597                         }
 598                     }
 599                 },
 600                 "Suite2": {
 601                     "tests": {
 602                         "test1": {
 603                             "metrics": {"Time": { "current": [11, 12, 13, 14, 15] }}
 604                         },
 605                         "test2": {
 606                             "metrics": {"Time": { "current": [16, 17, 18, 19, 20] }}
 607                         }
 608                     }
 609                 }
 610             }
 611         };
 612     }
 613
 614     it("should be able to add a report with same subtest name", function (done) {
 615         reportAfterAddingBuilderAndAggregators(reportWithSameSubtestName()).then(function () {
 616             done();
 617         }).catch(done);
 618     });
 619
 620     it("should be able to reuse the same test rows", function (done) {
 621         reportAfterAddingBuilderAndAggregators(reportWithSameSubtestName()).then(function () {
 622             return TestServer.database().selectAll('tests');
 623         }).then(function (tests) {
 624             assert.equal(tests.length, 6);
 625             let newReport = reportWithSameSubtestName();
 626             newReport.buildNumber = "125";
 627             newReport.buildTime = "2013-02-28T12:17:24.1";
 628             return TestServer.remoteAPI().postJSON('/api/report/', [newReport]);
 629         }).then(function (response) {
 630             assert.equal(response['status'], 'OK');
 631             return TestServer.database().selectAll('tests');
 632         }).then(function (tests) {
 633             assert.equal(tests.length, 6);
 634             done();
 635         }).catch(done);
 636     });
 637
 638     const reportWithSameSingleValue = {
 639         "buildNumber": "123",
 640         "buildTime": "2013-02-28T10:12:03.388304",
 641         "builderName": "someBuilder",
 642         "builderPassword": "somePassword",
 643         "platform": "Mountain Lion",
 644         "tests": {
 645             "suite": {
 646                 "metrics": {"Combined": ["Arithmetic"]},
 647                 "tests": {
 648                     "test1": {
 649                         "metrics": {"Combined": { "current": 3 }}
 650                     },
 651                     "test2": {
 652                         "metrics": {"Combined": { "current": 7 }}
 653                     }
 654                 }
 655             },
 656         }};
 657
 658     it("should be able to add a report with single value results", function (done) {
 659         reportAfterAddingBuilderAndAggregators(reportWithSameSingleValue).then(function () {
 660             return fetchTestRunIterationsForMetric('test1', 'Combined');
 661         }).then(function (result) {
 662             const run = result.run;
 663             assert.equal(run['iteration_count_cache'], 1);
 664             assert.equal(run['mean_cache'], 3);
 665             assert.equal(run['sum_cache'], 3);
 666             assert.equal(run['square_sum_cache'], 9);
 667             return fetchTestRunIterationsForMetric('suite', 'Combined');
 668         }).then(function (result) {
 669             const run = result.run;
 670             assert.equal(run['iteration_count_cache'], 1);
 671             assert.equal(run['mean_cache'], 5);
 672             assert.equal(run['sum_cache'], 5);
 673             assert.equal(run['square_sum_cache'], 25);
 674             done();
 675         }).catch(done);
 676     });
 677
 678     const reportWithSameValuePairs = {
 679         "buildNumber": "123",
 680         "buildTime": "2013-02-28T10:12:03.388304",
 681         "builderName": "someBuilder",
 682         "builderPassword": "somePassword",
 683         "platform": "Mountain Lion",
 684         "tests": {
 685                 "test": {
 686                     "metrics": {"FrameRate": { "current": [[[0, 4], [100, 5], [205, 3]]] }}
 687                 },
 688             },
 689         };
 690
 691     it("should be able to add a report with (relative time, value) pairs", function (done) {
 692         reportAfterAddingBuilderAndAggregators(reportWithSameValuePairs).then(function () {
 693             return fetchTestRunIterationsForMetric('test', 'FrameRate');
 694         }).then(function (result) {
 695             const run = result.run;
 696             assert.equal(run['iteration_count_cache'], 3);
 697             assert.equal(run['mean_cache'], 4);
 698             assert.equal(run['sum_cache'], 12);
 699             assert.equal(run['square_sum_cache'], 16 + 25 + 9);
 700
 701             const runId = run['id'];
 702             assert.deepEqual(result.iterations, [
 703                 {run: runId, order: 0, group: null, value: 4, relative_time: 0},
 704                 {run: runId, order: 1, group: null, value: 5, relative_time: 100},
 705                 {run: runId, order: 2, group: null, value: 3, relative_time: 205}]);
 706             done();
 707         }).catch(done);
 708     });
 709
 710     const reportsUpdatingDifferentTests = [
 711         {
 712             "buildNumber": "123",
 713             "buildTime": "2013-02-28T10:12:03",
 714             "builderName": "someBuilder",
 715             "builderPassword": "somePassword",
 716             "platform": "Mountain Lion",
 717             "tests": {"test1": {"metrics": {"Time": {"current": 3}}}}
 718         },
 719         {
 720             "buildNumber": "124",
 721             "buildTime": "2013-02-28T11:31:21",
 722             "builderName": "someBuilder",
 723             "builderPassword": "somePassword",
 724             "platform": "Mountain Lion",
 725             "tests": {"test2": {"metrics": {"Time": {"current": 3}}}}
 726         },
 727         {
 728             "buildNumber": "125",
 729             "buildTime": "2013-02-28T12:45:34",
 730             "builderName": "someBuilder",
 731             "builderPassword": "somePassword",
 732             "platform": "Mountain Lion",
 733             "tests": {"test1": {"metrics": {"Time": {"current": 3}}}}
 734         },
 735     ];
 736
 737     it("should update the last modified date of test configurations with new runs", function (done) {
 738         addBuilderForReport(reportsUpdatingDifferentTests[0]).then(function () {
 739             return TestServer.remoteAPI().postJSON('/api/report/', [reportsUpdatingDifferentTests[0]]);
 740         }).then(function (response) {
 741             assert.equal(response['status'], 'OK');
 742             return fetchTestConfig('test1', 'Time');
 743         }).then(function (originalConfig) {
 744             return TestServer.remoteAPI().postJSON('/api/report/', [reportsUpdatingDifferentTests[2]]).then(function () {
 745                 return fetchTestConfig('test1', 'Time');
 746             }).then(function (config) {
 747                 assert(originalConfig['config_runs_last_modified'] instanceof Date);
 748                 assert(config['config_runs_last_modified'] instanceof Date);
 749                 assert(+originalConfig['config_runs_last_modified'] < +config['config_runs_last_modified']);
 750                 done();
 751             });
 752         }).catch(done);
 753     });
 754
 755     it("should not update the last modified date of unrelated test configurations", function (done) {
 756         addBuilderForReport(reportsUpdatingDifferentTests[0]).then(function () {
 757             return TestServer.remoteAPI().postJSON('/api/report/', [reportsUpdatingDifferentTests[0]]);
 758         }).then(function (response) {
 759             assert.equal(response['status'], 'OK');
 760             return fetchTestConfig('test1', 'Time');
 761         }).then(function (originalConfig) {
 762             return TestServer.remoteAPI().postJSON('/api/report/', [reportsUpdatingDifferentTests[1]]).then(function (response) {
 763                 assert.equal(response['status'], 'OK');
 764                 return fetchTestConfig('test1', 'Time');
 765             }).then(function (config) {
 766                 assert(originalConfig['config_runs_last_modified'] instanceof Date);
 767                 assert(config['config_runs_last_modified'] instanceof Date);
 768                 assert.equal(+originalConfig['config_runs_last_modified'], +config['config_runs_last_modified']);
 769                 done();
 770             });
 771         }).catch(done);
 772     });
 773 });