Websites/perf.webkit.org/server-tests/api-measurement-set-tests.js

   1 'use strict';
   2
   3 const assert = require('assert');
   4 const crypto = require('crypto');
   5
   6 const TestServer = require('./resources/test-server.js');
   7 const addBuilderForReport = require('./resources/common-operations.js').addBuilderForReport;
   8 const connectToDatabaseInEveryTest = require('./resources/common-operations.js').connectToDatabaseInEveryTest;
   9
  10 describe("/api/measurement-set", function () {
  11     this.timeout(1000);
  12     TestServer.inject();
  13     connectToDatabaseInEveryTest();
  14
  15     function queryPlatformAndMetric(platformName, metricName)
  16     {
  17         const db = TestServer.database();
  18         return Promise.all([
  19             db.selectFirstRow('platforms', {name: 'Mountain Lion'}),
  20             db.selectFirstRow('test_metrics', {name: 'Time'}),
  21         ]).then(function (result) {
  22             return {platformId: result[0]['id'], metricId: result[1]['id']};
  23         });
  24     }
  25
  26     function format(formatMap, row)
  27     {
  28         var result = {};
  29         for (var i = 0; i < formatMap.length; i++) {
  30             var key = formatMap[i];
  31             if (key == 'id' || key == 'build' || key == 'builder')
  32                 continue;
  33             result[key] = row[i];
  34         }
  35         return result;
  36     }
  37
  38     let clusterStart = TestServer.testConfig().clusterStart;
  39     clusterStart = +Date.UTC(clusterStart[0], clusterStart[1] - 1, clusterStart[2], clusterStart[3], clusterStart[4]);
  40
  41     let clusterSize = TestServer.testConfig().clusterSize;
  42     const DAY = 24 * 3600 * 1000;
  43     const YEAR = 365.24 * DAY;
  44     const MONTH = 30 * DAY;
  45     clusterSize = clusterSize[0] * YEAR + clusterSize[1] * MONTH + clusterSize[2] * DAY;
  46
  47     function clusterTime(index) { return new Date(clusterStart + clusterSize * index); }
  48
  49     const reportWithBuildTime = [{
  50         "buildNumber": "123",
  51         "buildTime": clusterTime(7.8).toISOString(),
  52         "builderName": "someBuilder",
  53         "builderPassword": "somePassword",
  54         "platform": "Mountain Lion",
  55         "tests": {
  56             "Suite": {
  57                 "tests": {
  58                     "test1": {
  59                         "metrics": {"Time": { "current": [1, 2, 3, 4, 5] }}
  60                     },
  61                 }
  62             },
  63         }}];
  64     reportWithBuildTime.startTime = +clusterTime(7);
  65
  66     const reportWithRevision = [{
  67         "buildNumber": "124",
  68         "buildTime": "2013-02-28T15:34:51",
  69         "revisions": {
  70             "WebKit": {
  71                 "revision": "144000",
  72                 "timestamp": clusterTime(10.3).toISOString(),
  73             },
  74         },
  75         "builderName": "someBuilder",
  76         "builderPassword": "somePassword",
  77         "platform": "Mountain Lion",
  78         "tests": {
  79             "Suite": {
  80                 "tests": {
  81                     "test1": {
  82                         "metrics": {"Time": { "current": [11, 12, 13, 14, 15] }}
  83                     }
  84                 }
  85             },
  86         }}];
  87
  88     const reportWithNewRevision = [{
  89         "buildNumber": "125",
  90         "buildTime": "2013-02-28T21:45:17",
  91         "revisions": {
  92             "WebKit": {
  93                 "revision": "160609",
  94                 "timestamp": clusterTime(12.1).toISOString()
  95             },
  96         },
  97         "builderName": "someBuilder",
  98         "builderPassword": "somePassword",
  99         "platform": "Mountain Lion",
 100         "tests": {
 101             "Suite": {
 102                 "tests": {
 103                     "test1": {
 104                         "metrics": {"Time": { "current": [16, 17, 18, 19, 20] }}
 105                     }
 106                 }
 107             },
 108         }}];
 109
 110     const reportWithAncentRevision = [{
 111         "buildNumber": "126",
 112         "buildTime": "2013-02-28T23:07:25",
 113         "revisions": {
 114             "WebKit": {
 115                 "revision": "137793",
 116                 "timestamp": clusterTime(1.8).toISOString()
 117             },
 118         },
 119         "builderName": "someBuilder",
 120         "builderPassword": "somePassword",
 121         "platform": "Mountain Lion",
 122         "tests": {
 123             "Suite": {
 124                 "tests": {
 125                     "test1": {
 126                         "metrics": {"Time": { "current": [21, 22, 23, 24, 25] }}
 127                     }
 128                 }
 129             },
 130         }}];
 131
 132     it("should reject when platform ID is missing", function (done) {
 133         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 134             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 135         }).then(function (response) {
 136             assert.equal(response['status'], 'OK');
 137             return queryPlatformAndMetric('Mountain Lion', 'Time');
 138         }).then(function (result) {
 139             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?metric=${result.metricId}`);
 140         }).then(function (response) {
 141             assert.equal(response['status'], 'AmbiguousRequest');
 142             done();
 143         }).catch(done);
 144     });
 145
 146     it("should reject when metric ID is missing", function (done) {
 147         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 148             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 149         }).then(function (response) {
 150             assert.equal(response['status'], 'OK');
 151             return queryPlatformAndMetric('Mountain Lion', 'Time');
 152         }).then(function (result) {
 153             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}`);
 154         }).then(function (response) {
 155             assert.equal(response['status'], 'AmbiguousRequest');
 156             done();
 157         }).catch(done);
 158     });
 159
 160     it("should reject an invalid platform name", function (done) {
 161         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 162             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 163         }).then(function (response) {
 164             assert.equal(response['status'], 'OK');
 165             return queryPlatformAndMetric('Mountain Lion', 'Time');
 166         }).then(function (result) {
 167             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}a&metric=${result.metricId}`);
 168         }).then(function (response) {
 169             assert.equal(response['status'], 'InvalidPlatform');
 170             done();
 171         }).catch(done);
 172     });
 173
 174     it("should reject an invalid metric name", function (done) {
 175         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 176             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 177         }).then(function (response) {
 178             assert.equal(response['status'], 'OK');
 179             return queryPlatformAndMetric('Mountain Lion', 'Time');
 180         }).then(function (result) {
 181             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}b`);
 182         }).then(function (response) {
 183             assert.equal(response['status'], 'InvalidMetric');
 184             done();
 185         }).catch(done);
 186     });
 187
 188     it("should be able to retrieve a reported value", function (done) {
 189         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 190             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 191         }).then(function (response) {
 192             assert.equal(response['status'], 'OK');
 193             return queryPlatformAndMetric('Mountain Lion', 'Time');
 194         }).then(function (result) {
 195             return TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 196         }).then(function (response) {
 197             const buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
 198
 199             assert.deepEqual(Object.keys(response).sort(),
 200                 ['clusterCount', 'clusterSize', 'clusterStart',
 201                   'configurations', 'elapsedTime', 'endTime', 'formatMap', 'lastModified', 'startTime', 'status']);
 202             assert.equal(response['status'], 'OK');
 203             assert.equal(response['clusterCount'], 1);
 204             assert.deepEqual(response['formatMap'], [
 205                 'id', 'mean', 'iterationCount', 'sum', 'squareSum', 'markedOutlier',
 206                 'revisions', 'commitTime', 'build', 'buildTime', 'buildNumber', 'builder']);
 207
 208             assert.equal(response['startTime'], reportWithBuildTime.startTime);
 209             assert(typeof(response['lastModified']) == 'number', 'lastModified time should be a numeric');
 210
 211             assert.deepEqual(Object.keys(response['configurations']), ['current']);
 212
 213             var currentRows = response['configurations']['current'];
 214             assert.equal(currentRows.length, 1);
 215             assert.equal(currentRows[0].length, response['formatMap'].length);
 216             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
 217                 mean: 3,
 218                 iterationCount: 5,
 219                 sum: 15,
 220                 squareSum: 55,
 221                 markedOutlier: false,
 222                 revisions: [],
 223                 commitTime: buildTime,
 224                 buildTime: buildTime,
 225                 buildNumber: '123'});
 226             done();
 227         }).catch(done);
 228     });
 229
 230     it("should return return the right IDs for measurement, build, and builder", function (done) {
 231         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 232             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 233         }).then(function (response) {
 234             assert.equal(response['status'], 'OK');
 235             return queryPlatformAndMetric('Mountain Lion', 'Time');
 236         }).then(function (result) {
 237             const db = TestServer.database();
 238             return Promise.all([
 239                 db.selectAll('test_runs'),
 240                 db.selectAll('builds'),
 241                 db.selectAll('builders'),
 242                 TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`),
 243             ]);
 244         }).then(function (result) {
 245             const runs = result[0];
 246             const builds = result[1];
 247             const builders = result[2];
 248             const response = result[3];
 249
 250             assert.equal(runs.length, 1);
 251             assert.equal(builds.length, 1);
 252             assert.equal(builders.length, 1);
 253             const measurementId = runs[0]['id'];
 254             const buildId = builds[0]['id'];
 255             const builderId = builders[0]['id'];
 256
 257             assert.equal(response['configurations']['current'].length, 1);
 258             const measurement = response['configurations']['current'][0];
 259             assert.equal(response['status'], 'OK');
 260
 261             assert.equal(measurement[response['formatMap'].indexOf('id')], measurementId);
 262             assert.equal(measurement[response['formatMap'].indexOf('build')], buildId);
 263             assert.equal(measurement[response['formatMap'].indexOf('builder')], builderId);
 264
 265             done();
 266         }).catch(done);
 267     });
 268
 269     function postReports(reports, callback)
 270     {
 271         if (!reports.length)
 272             return callback();
 273
 274         postJSON('/api/report/', reports[0], function (response) {
 275             assert.equal(response.statusCode, 200);
 276             assert.equal(JSON.parse(response.responseText)['status'], 'OK');
 277
 278             postReports(reports.slice(1), callback);
 279         });
 280     }
 281
 282     function queryPlatformAndMetricWithRepository(platformName, metricName, repositoryName)
 283     {
 284         const db = TestServer.database();
 285         return Promise.all([
 286             db.selectFirstRow('platforms', {name: platformName}),
 287             db.selectFirstRow('test_metrics', {name: metricName}),
 288             db.selectFirstRow('repositories', {name: repositoryName}),
 289         ]).then(function (result) {
 290             return {platformId: result[0]['id'], metricId: result[1]['id'], repositoryId: result[2]['id']};
 291         });
 292     }
 293
 294     it("should order results by commit time", function (done) {
 295         const remote = TestServer.remoteAPI();
 296         let repositoryId;
 297         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 298             return remote.postJSON('/api/report/', reportWithBuildTime);
 299         }).then(function () {
 300             return remote.postJSON('/api/report/', reportWithRevision);
 301         }).then(function () {
 302             return queryPlatformAndMetricWithRepository('Mountain Lion', 'Time', 'WebKit');
 303         }).then(function (result) {
 304             repositoryId = result.repositoryId;
 305             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 306         }).then(function (response) {
 307             const currentRows = response['configurations']['current'];
 308             const buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
 309             const revisionTime = +(new Date(reportWithRevision[0]['revisions']['WebKit']['timestamp']));
 310             const revisionBuildTime = +(new Date(reportWithRevision[0]['buildTime']));
 311
 312             assert.equal(currentRows.length, 2);
 313             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
 314                mean: 13,
 315                iterationCount: 5,
 316                sum: 65,
 317                squareSum: 855,
 318                markedOutlier: false,
 319                revisions: [[1, repositoryId, '144000', revisionTime]],
 320                commitTime: revisionTime,
 321                buildTime: revisionBuildTime,
 322                buildNumber: '124' });
 323             assert.deepEqual(format(response['formatMap'], currentRows[1]), {
 324                 mean: 3,
 325                 iterationCount: 5,
 326                 sum: 15,
 327                 squareSum: 55,
 328                 markedOutlier: false,
 329                 revisions: [],
 330                 commitTime: buildTime,
 331                 buildTime: buildTime,
 332                 buildNumber: '123' });
 333             done();
 334         }).catch(done);
 335     });
 336
 337     function buildNumbers(parsedResult, config)
 338     {
 339         return parsedResult['configurations'][config].map(function (row) {
 340             return format(parsedResult['formatMap'], row)['buildNumber'];
 341         });
 342     }
 343
 344     it("should include one data point after the current time range", function (done) {
 345         const remote = TestServer.remoteAPI();
 346         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 347             return remote.postJSON('/api/report/', reportWithAncentRevision);
 348         }).then(function () {
 349             return remote.postJSON('/api/report/', reportWithNewRevision);
 350         }).then(function () {
 351             return queryPlatformAndMetric('Mountain Lion', 'Time');
 352         }).then(function (result) {
 353             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 354         }).then(function (response) {
 355             assert.equal(response['status'], 'OK');
 356             assert.equal(response['clusterCount'], 2, 'should have two clusters');
 357             assert.deepEqual(buildNumbers(response, 'current'),
 358                 [reportWithAncentRevision[0]['buildNumber'], reportWithNewRevision[0]['buildNumber']]);
 359             done();
 360         }).catch(done);
 361     });
 362
 363     it("should always include one old data point before the current time range", function (done) {
 364         const remote = TestServer.remoteAPI();
 365         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 366             return remote.postJSON('/api/report/', reportWithBuildTime);
 367         }).then(function () {
 368             return remote.postJSON('/api/report/', reportWithAncentRevision);
 369         }).then(function () {
 370             return queryPlatformAndMetric('Mountain Lion', 'Time');
 371         }).then(function (result) {
 372             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 373         }).then(function (response) {
 374             assert.equal(response['clusterCount'], 2, 'should have two clusters');
 375             let currentRows = response['configurations']['current'];
 376             assert.equal(currentRows.length, 2, 'should contain two data points');
 377             assert.deepEqual(buildNumbers(response, 'current'), [reportWithAncentRevision[0]['buildNumber'], reportWithBuildTime[0]['buildNumber']]);
 378             done();
 379         }).catch(done);
 380     });
 381
 382
 383     it("should create cache results", function (done) {
 384         const remote = TestServer.remoteAPI();
 385         let cachePrefix;
 386         addBuilderForReport(reportWithBuildTime[0]).then(function () {
 387             return remote.postJSON('/api/report/', reportWithAncentRevision);
 388         }).then(function () {
 389             return remote.postJSON('/api/report/', reportWithRevision);
 390         }).then(function () {
 391             return remote.postJSON('/api/report/', reportWithNewRevision);
 392         }).then(function () {
 393             return queryPlatformAndMetric('Mountain Lion', 'Time');
 394         }).then(function (result) {
 395             cachePrefix = '/data/measurement-set-' + result.platformId + '-' + result.metricId;
 396             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 397         }).then(function (newResult) {
 398             return remote.getJSONWithStatus(`${cachePrefix}.json`).then(function (cachedResult) {
 399                 assert.deepEqual(newResult, cachedResult);
 400                 return remote.getJSONWithStatus(`${cachePrefix}-${cachedResult['startTime']}.json`);
 401             }).then(function (oldResult) {
 402                 var oldBuildNumbers = buildNumbers(oldResult, 'current');
 403                 var newBuildNumbers = buildNumbers(newResult, 'current');
 404                 assert(oldBuildNumbers.length >= 2, 'The old cluster should contain at least two data points');
 405                 assert(newBuildNumbers.length >= 2, 'The new cluster should contain at least two data points');
 406                 assert.deepEqual(oldBuildNumbers.slice(oldBuildNumbers.length - 2), newBuildNumbers.slice(0, 2),
 407                     'Two conseqcutive clusters should share two data points');
 408                 done();
 409             });
 410         }).catch(done);
 411     });
 412
 413 });