Websites/perf.webkit.org/server-tests/api-measurement-set-tests.js

   1 'use strict';
   2
   3 const assert = require('assert');
   4
   5 require('../tools/js/v3-models.js');
   6
   7 const MockData = require('./resources/mock-data.js');
   8 const TestServer = require('./resources/test-server.js');
   9 const addBuilderForReport = require('./resources/common-operations.js').addBuilderForReport;
  10 const connectToDatabaseInEveryTest = require('./resources/common-operations.js').connectToDatabaseInEveryTest;
  11
  12 describe("/api/measurement-set", function () {
  13     this.timeout(1000);
  14     TestServer.inject();
  15     connectToDatabaseInEveryTest();
  16
  17     beforeEach(function () {
  18         MockData.resetV3Models();
  19     });
  20
  21     function queryPlatformAndMetric(platformName, metricName)
  22     {
  23         const db = TestServer.database();
  24         return Promise.all([
  25             db.selectFirstRow('platforms', {name: 'Mountain Lion'}),
  26             db.selectFirstRow('test_metrics', {name: 'Time'}),
  27         ]).then(function (result) {
  28             return {platformId: result[0]['id'], metricId: result[1]['id']};
  29         });
  30     }
  31
  32     function format(formatMap, row)
  33     {
  34         var result = {};
  35         for (var i = 0; i < formatMap.length; i++) {
  36             var key = formatMap[i];
  37             if (key == 'id' || key == 'build' || key == 'builder')
  38                 continue;
  39             result[key] = row[i];
  40         }
  41         return result;
  42     }
  43
  44     let clusterStart = TestServer.testConfig().clusterStart;
  45     clusterStart = +Date.UTC(clusterStart[0], clusterStart[1] - 1, clusterStart[2], clusterStart[3], clusterStart[4]);
  46
  47     let clusterSize = TestServer.testConfig().clusterSize;
  48     const DAY = 24 * 3600 * 1000;
  49     const YEAR = 365.24 * DAY;
  50     const MONTH = 30 * DAY;
  51     clusterSize = clusterSize[0] * YEAR + clusterSize[1] * MONTH + clusterSize[2] * DAY;
  52
  53     function clusterTime(index) { return new Date(clusterStart + clusterSize * index); }
  54
  55     const reportWithBuildTime = [{
  56         "buildNumber": "123",
  57         "buildTime": clusterTime(7.8).toISOString(),
  58         "builderName": "someBuilder",
  59         "builderPassword": "somePassword",
  60         "platform": "Mountain Lion",
  61         "tests": {
  62             "Suite": {
  63                 "tests": {
  64                     "test1": {
  65                         "metrics": {"Time": { "current": [1, 2, 3, 4, 5] }}
  66                     },
  67                 }
  68             },
  69         }}];
  70     reportWithBuildTime.startTime = +clusterTime(7);
  71
  72     const reportWithRevision = [{
  73         "buildNumber": "124",
  74         "buildTime": "2013-02-28T15:34:51",
  75         "revisions": {
  76             "WebKit": {
  77                 "revision": "144000",
  78                 "timestamp": clusterTime(10.35645364537).toISOString(),
  79             },
  80         },
  81         "builderName": "someBuilder",
  82         "builderPassword": "somePassword",
  83         "platform": "Mountain Lion",
  84         "tests": {
  85             "Suite": {
  86                 "tests": {
  87                     "test1": {
  88                         "metrics": {"Time": { "current": [11, 12, 13, 14, 15] }}
  89                     }
  90                 }
  91             },
  92         }}];
  93
  94     const reportWithNewRevision = [{
  95         "buildNumber": "125",
  96         "buildTime": "2013-02-28T21:45:17",
  97         "revisions": {
  98             "WebKit": {
  99                 "revision": "160609",
 100                 "timestamp": clusterTime(12.1).toISOString()
 101             },
 102         },
 103         "builderName": "someBuilder",
 104         "builderPassword": "somePassword",
 105         "platform": "Mountain Lion",
 106         "tests": {
 107             "Suite": {
 108                 "tests": {
 109                     "test1": {
 110                         "metrics": {"Time": { "current": [16, 17, 18, 19, 20] }}
 111                     }
 112                 }
 113             },
 114         }}];
 115
 116     const reportWithAncentRevision = [{
 117         "buildNumber": "126",
 118         "buildTime": "2013-02-28T23:07:25",
 119         "revisions": {
 120             "WebKit": {
 121                 "revision": "137793",
 122                 "timestamp": clusterTime(1.8).toISOString()
 123             },
 124         },
 125         "builderName": "someBuilder",
 126         "builderPassword": "somePassword",
 127         "platform": "Mountain Lion",
 128         "tests": {
 129             "Suite": {
 130                 "tests": {
 131                     "test1": {
 132                         "metrics": {"Time": { "current": [21, 22, 23, 24, 25] }}
 133                     }
 134                 }
 135             },
 136         }}];
 137
 138     it("should reject when platform ID is missing", function (done) {
 139         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 140             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 141         }).then(function (response) {
 142             assert.equal(response['status'], 'OK');
 143             return queryPlatformAndMetric('Mountain Lion', 'Time');
 144         }).then(function (result) {
 145             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?metric=${result.metricId}`);
 146         }).then(function (response) {
 147             assert.equal(response['status'], 'AmbiguousRequest');
 148             done();
 149         }).catch(done);
 150     });
 151
 152     it("should reject when metric ID is missing", function (done) {
 153         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 154             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 155         }).then(function (response) {
 156             assert.equal(response['status'], 'OK');
 157             return queryPlatformAndMetric('Mountain Lion', 'Time');
 158         }).then(function (result) {
 159             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}`);
 160         }).then(function (response) {
 161             assert.equal(response['status'], 'AmbiguousRequest');
 162             done();
 163         }).catch(done);
 164     });
 165
 166     it("should reject an invalid platform name", function (done) {
 167         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 168             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 169         }).then(function (response) {
 170             assert.equal(response['status'], 'OK');
 171             return queryPlatformAndMetric('Mountain Lion', 'Time');
 172         }).then(function (result) {
 173             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}a&metric=${result.metricId}`);
 174         }).then(function (response) {
 175             assert.equal(response['status'], 'InvalidPlatform');
 176             done();
 177         }).catch(done);
 178     });
 179
 180     it("should reject an invalid metric name", function (done) {
 181         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 182             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 183         }).then(function (response) {
 184             assert.equal(response['status'], 'OK');
 185             return queryPlatformAndMetric('Mountain Lion', 'Time');
 186         }).then(function (result) {
 187             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}b`);
 188         }).then(function (response) {
 189             assert.equal(response['status'], 'InvalidMetric');
 190             done();
 191         }).catch(done);
 192     });
 193
 194     it("should be able to return an empty report", function (done) {
 195         const db = TestServer.database();
 196         Promise.all([
 197             db.insert('tests', {id: 1, name: 'SomeTest'}),
 198             db.insert('tests', {id: 2, name: 'SomeOtherTest'}),
 199             db.insert('tests', {id: 3, name: 'ChildTest', parent: 1}),
 200             db.insert('tests', {id: 4, name: 'GrandChild', parent: 3}),
 201             db.insert('aggregators', {id: 200, name: 'Total'}),
 202             db.insert('test_metrics', {id: 5, test: 1, name: 'Time'}),
 203             db.insert('test_metrics', {id: 6, test: 2, name: 'Time', aggregator: 200}),
 204             db.insert('test_metrics', {id: 7, test: 2, name: 'Malloc', aggregator: 200}),
 205             db.insert('test_metrics', {id: 8, test: 3, name: 'Time'}),
 206             db.insert('test_metrics', {id: 9, test: 4, name: 'Time'}),
 207             db.insert('platforms', {id: 23, name: 'iOS 9 iPhone 5s'}),
 208             db.insert('platforms', {id: 46, name: 'Trunk Mavericks'}),
 209             db.insert('test_configurations', {id: 101, metric: 5, platform: 46, type: 'current'}),
 210             db.insert('test_configurations', {id: 102, metric: 6, platform: 46, type: 'current'}),
 211             db.insert('test_configurations', {id: 103, metric: 7, platform: 46, type: 'current'}),
 212             db.insert('test_configurations', {id: 104, metric: 8, platform: 46, type: 'current'}),
 213             db.insert('test_configurations', {id: 105, metric: 9, platform: 46, type: 'current'}),
 214             db.insert('test_configurations', {id: 106, metric: 5, platform: 23, type: 'current'}),
 215             db.insert('test_configurations', {id: 107, metric: 5, platform: 23, type: 'baseline'}),
 216         ]).then(function () {
 217             return TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=46&metric=5`).then(function (response) {
 218                 assert.equal(response.statusCode, 404);
 219             }, function (error) {
 220                 assert.equal(error, 404);
 221                 done();
 222             });
 223         }).catch(done);
 224     });
 225
 226     it("should be able to retrieve a reported value", function (done) {
 227         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 228             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 229         }).then(function (response) {
 230             assert.equal(response['status'], 'OK');
 231             return queryPlatformAndMetric('Mountain Lion', 'Time');
 232         }).then(function (result) {
 233             return TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 234         }).then(function (response) {
 235             const buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
 236
 237             assert.deepEqual(Object.keys(response).sort(),
 238                 ['clusterCount', 'clusterSize', 'clusterStart',
 239                   'configurations', 'elapsedTime', 'endTime', 'formatMap', 'lastModified', 'startTime', 'status']);
 240             assert.equal(response['status'], 'OK');
 241             assert.equal(response['clusterCount'], 1);
 242             assert.deepEqual(response['formatMap'], [
 243                 'id', 'mean', 'iterationCount', 'sum', 'squareSum', 'markedOutlier',
 244                 'revisions', 'commitTime', 'build', 'buildTime', 'buildNumber', 'builder']);
 245
 246             assert.equal(response['startTime'], reportWithBuildTime.startTime);
 247             assert(typeof(response['lastModified']) == 'number', 'lastModified time should be a numeric');
 248
 249             assert.deepEqual(Object.keys(response['configurations']), ['current']);
 250
 251             var currentRows = response['configurations']['current'];
 252             assert.equal(currentRows.length, 1);
 253             assert.equal(currentRows[0].length, response['formatMap'].length);
 254             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
 255                 mean: 3,
 256                 iterationCount: 5,
 257                 sum: 15,
 258                 squareSum: 55,
 259                 markedOutlier: false,
 260                 revisions: [],
 261                 commitTime: buildTime,
 262                 buildTime: buildTime,
 263                 buildNumber: '123'});
 264             done();
 265         }).catch(done);
 266     });
 267
 268     it("should return return the right IDs for measurement, build, and builder", function (done) {
 269         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 270             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 271         }).then(function (response) {
 272             assert.equal(response['status'], 'OK');
 273             return queryPlatformAndMetric('Mountain Lion', 'Time');
 274         }).then(function (result) {
 275             const db = TestServer.database();
 276             return Promise.all([
 277                 db.selectAll('test_runs'),
 278                 db.selectAll('builds'),
 279                 db.selectAll('builders'),
 280                 TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`),
 281             ]);
 282         }).then(function (result) {
 283             const runs = result[0];
 284             const builds = result[1];
 285             const builders = result[2];
 286             const response = result[3];
 287
 288             assert.equal(runs.length, 1);
 289             assert.equal(builds.length, 1);
 290             assert.equal(builders.length, 1);
 291             const measurementId = runs[0]['id'];
 292             const buildId = builds[0]['id'];
 293             const builderId = builders[0]['id'];
 294
 295             assert.equal(response['configurations']['current'].length, 1);
 296             const measurement = response['configurations']['current'][0];
 297             assert.equal(response['status'], 'OK');
 298
 299             assert.equal(measurement[response['formatMap'].indexOf('id')], measurementId);
 300             assert.equal(measurement[response['formatMap'].indexOf('build')], buildId);
 301             assert.equal(measurement[response['formatMap'].indexOf('builder')], builderId);
 302
 303             done();
 304         }).catch(done);
 305     });
 306
 307     function postReports(reports, callback)
 308     {
 309         if (!reports.length)
 310             return callback();
 311
 312         postJSON('/api/report/', reports[0], function (response) {
 313             assert.equal(response.statusCode, 200);
 314             assert.equal(JSON.parse(response.responseText)['status'], 'OK');
 315
 316             postReports(reports.slice(1), callback);
 317         });
 318     }
 319
 320     function queryPlatformAndMetricWithRepository(platformName, metricName, repositoryName)
 321     {
 322         const db = TestServer.database();
 323         return Promise.all([
 324             db.selectFirstRow('platforms', {name: platformName}),
 325             db.selectFirstRow('test_metrics', {name: metricName}),
 326             db.selectFirstRow('repositories', {name: repositoryName}),
 327         ]).then(function (result) {
 328             return {platformId: result[0]['id'], metricId: result[1]['id'], repositoryId: result[2]['id']};
 329         });
 330     }
 331
 332     it("should order results by commit time", function (done) {
 333         const remote = TestServer.remoteAPI();
 334         let repositoryId;
 335         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 336             return remote.postJSON('/api/report/', reportWithBuildTime);
 337         }).then(function () {
 338             return remote.postJSON('/api/report/', reportWithRevision);
 339         }).then(function () {
 340             return queryPlatformAndMetricWithRepository('Mountain Lion', 'Time', 'WebKit');
 341         }).then(function (result) {
 342             repositoryId = result.repositoryId;
 343             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 344         }).then(function (response) {
 345             const currentRows = response['configurations']['current'];
 346             const buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
 347             const revisionTime = +(new Date(reportWithRevision[0]['revisions']['WebKit']['timestamp']));
 348             const revisionBuildTime = +(new Date(reportWithRevision[0]['buildTime']));
 349
 350             assert.equal(currentRows.length, 2);
 351             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
 352                mean: 13,
 353                iterationCount: 5,
 354                sum: 65,
 355                squareSum: 855,
 356                markedOutlier: false,
 357                revisions: [[1, repositoryId, '144000', revisionTime]],
 358                commitTime: revisionTime,
 359                buildTime: revisionBuildTime,
 360                buildNumber: '124' });
 361             assert.deepEqual(format(response['formatMap'], currentRows[1]), {
 362                 mean: 3,
 363                 iterationCount: 5,
 364                 sum: 15,
 365                 squareSum: 55,
 366                 markedOutlier: false,
 367                 revisions: [],
 368                 commitTime: buildTime,
 369                 buildTime: buildTime,
 370                 buildNumber: '123' });
 371             done();
 372         }).catch(done);
 373     });
 374
 375     it("should order results by build time when commit times are missing", function (done) {
 376         const remote = TestServer.remoteAPI();
 377         let repositoryId;
 378         addBuilderForReport(reportWithBuildTime[0]).then(() => {
 379             const db = TestServer.database();
 380             return Promise.all([
 381                 db.insert('repositories', {'id': 1, 'name': 'macOS'}),
 382                 db.insert('commits', {'id': 2, 'repository': 1, 'revision': 'macOS 16A323', 'order': 0}),
 383                 db.insert('commits', {'id': 3, 'repository': 1, 'revision': 'macOS 16C68', 'order': 1}),
 384             ]);
 385         }).then(() => {
 386             return remote.postJSON('/api/report/', [{
 387                 "buildNumber": "1001",
 388                 "buildTime": '2017-01-19 15:28:01',
 389                 "revisions": {
 390                     "macOS": {
 391                         "revision": "macOS 16C68",
 392                     },
 393                 },
 394                 "builderName": "someBuilder",
 395                 "builderPassword": "somePassword",
 396                 "platform": "Sierra",
 397                 "tests": { "Test": {"metrics": {"Time": { "baseline": [1, 2, 3, 4, 5] } } } },
 398             }]);
 399         }).then(function () {
 400             return remote.postJSON('/api/report/', [{
 401                 "buildNumber": "1002",
 402                 "buildTime": '2017-01-19 19:46:37',
 403                 "revisions": {
 404                     "macOS": {
 405                         "revision": "macOS 16A323",
 406                     },
 407                 },
 408                 "builderName": "someBuilder",
 409                 "builderPassword": "somePassword",
 410                 "platform": "Sierra",
 411                 "tests": { "Test": {"metrics": {"Time": { "baseline": [5, 6, 7, 8, 9] } } } },
 412             }]);
 413         }).then(function () {
 414             return queryPlatformAndMetricWithRepository('Sierra', 'Time', 'macOS');
 415         }).then(function (result) {
 416             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 417         }).then(function (response) {
 418             const currentRows = response['configurations']['baseline'];
 419             assert.equal(currentRows.length, 2);
 420             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
 421                mean: 3,
 422                iterationCount: 5,
 423                sum: 15,
 424                squareSum: 55,
 425                markedOutlier: false,
 426                revisions: [[3, 1, 'macOS 16C68', 0]],
 427                commitTime: +Date.UTC(2017, 0, 19, 15, 28, 1),
 428                buildTime: +Date.UTC(2017, 0, 19, 15, 28, 1),
 429                buildNumber: '1001' });
 430             assert.deepEqual(format(response['formatMap'], currentRows[1]), {
 431                 mean: 7,
 432                 iterationCount: 5,
 433                 sum: 35,
 434                 squareSum: 255,
 435                 markedOutlier: false,
 436                 revisions: [[2, 1, 'macOS 16A323', 0]],
 437                 commitTime: +Date.UTC(2017, 0, 19, 19, 46, 37),
 438                 buildTime: +Date.UTC(2017, 0, 19, 19, 46, 37),
 439                 buildNumber: '1002' });
 440             done();
 441         }).catch(done);
 442     });
 443
 444     function buildNumbers(parsedResult, config)
 445     {
 446         return parsedResult['configurations'][config].map(function (row) {
 447             return format(parsedResult['formatMap'], row)['buildNumber'];
 448         });
 449     }
 450
 451     it("should include one data point after the current time range", function (done) {
 452         const remote = TestServer.remoteAPI();
 453         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 454             return remote.postJSON('/api/report/', reportWithAncentRevision);
 455         }).then(function () {
 456             return remote.postJSON('/api/report/', reportWithNewRevision);
 457         }).then(function () {
 458             return queryPlatformAndMetric('Mountain Lion', 'Time');
 459         }).then(function (result) {
 460             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 461         }).then(function (response) {
 462             assert.equal(response['status'], 'OK');
 463             assert.equal(response['clusterCount'], 2, 'should have two clusters');
 464             assert.deepEqual(buildNumbers(response, 'current'),
 465                 [reportWithAncentRevision[0]['buildNumber'], reportWithNewRevision[0]['buildNumber']]);
 466             done();
 467         }).catch(done);
 468     });
 469
 470     it("should always include one old data point before the current time range", function (done) {
 471         const remote = TestServer.remoteAPI();
 472         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 473             return remote.postJSON('/api/report/', reportWithBuildTime);
 474         }).then(function () {
 475             return remote.postJSON('/api/report/', reportWithAncentRevision);
 476         }).then(function () {
 477             return queryPlatformAndMetric('Mountain Lion', 'Time');
 478         }).then(function (result) {
 479             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 480         }).then(function (response) {
 481             assert.equal(response['clusterCount'], 2, 'should have two clusters');
 482             let currentRows = response['configurations']['current'];
 483             assert.equal(currentRows.length, 2, 'should contain two data points');
 484             assert.deepEqual(buildNumbers(response, 'current'), [reportWithAncentRevision[0]['buildNumber'], reportWithBuildTime[0]['buildNumber']]);
 485             done();
 486         }).catch(done);
 487     });
 488
 489     it("should create cached results", function (done) {
 490         const remote = TestServer.remoteAPI();
 491         let cachePrefix;
 492         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 493             return remote.postJSON('/api/report/', reportWithAncentRevision);
 494         }).then(function () {
 495             return remote.postJSON('/api/report/', reportWithRevision);
 496         }).then(function () {
 497             return remote.postJSON('/api/report/', reportWithNewRevision);
 498         }).then(function () {
 499             return queryPlatformAndMetric('Mountain Lion', 'Time');
 500         }).then(function (result) {
 501             cachePrefix = '/data/measurement-set-' + result.platformId + '-' + result.metricId;
 502             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 503         }).then(function (newResult) {
 504             return remote.getJSONWithStatus(`${cachePrefix}.json`).then(function (cachedResult) {
 505                 assert.deepEqual(newResult, cachedResult);
 506                 return remote.getJSONWithStatus(`${cachePrefix}-${cachedResult['startTime']}.json`);
 507             }).then(function (oldResult) {
 508                 var oldBuildNumbers = buildNumbers(oldResult, 'current');
 509                 var newBuildNumbers = buildNumbers(newResult, 'current');
 510                 assert(oldBuildNumbers.length >= 2, 'The old cluster should contain at least two data points');
 511                 assert(newBuildNumbers.length >= 2, 'The new cluster should contain at least two data points');
 512                 assert.deepEqual(oldBuildNumbers.slice(oldBuildNumbers.length - 2), newBuildNumbers.slice(0, 2),
 513                     'Two conseqcutive clusters should share two data points');
 514                 done();
 515             });
 516         }).catch(done);
 517     });
 518
 519     it("should use lastModified timestamp identical to that in the manifest file", function (done) {
 520         const remote = TestServer.remoteAPI();
 521         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 522             return remote.postJSON('/api/report/', reportWithRevision);
 523         }).then(function () {
 524             return queryPlatformAndMetric('Mountain Lion', 'Time');
 525         }).then(function (result) {
 526             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 527         }).then(function (primaryCluster) {
 528             return remote.getJSONWithStatus('/api/manifest').then(function (content) {
 529                 const manifest = Manifest._didFetchManifest(content);
 530
 531                 const platform = Platform.findByName('Mountain Lion');
 532                 assert.equal(Metric.all().length, 1);
 533                 const metric = Metric.all()[0];
 534                 assert.equal(platform.lastModified(metric), primaryCluster['lastModified']);
 535
 536                 done();
 537             });
 538         }).catch(done);
 539     });
 540
 541 });