Websites/perf.webkit.org/server-tests/api-measurement-set-tests.js

   1 'use strict';
   2
   3 const assert = require('assert');
   4
   5 require('../tools/js/v3-models.js');
   6
   7 const MockData = require('./resources/mock-data.js');
   8 const TestServer = require('./resources/test-server.js');
   9 const addBuilderForReport = require('./resources/common-operations.js').addBuilderForReport;
  10 const prepareServerTest = require('./resources/common-operations.js').prepareServerTest;
  11
  12 describe("/api/measurement-set", function () {
  13     prepareServerTest(this);
  14
  15     function queryPlatformAndMetric(platformName, metricName)
  16     {
  17         const db = TestServer.database();
  18         return Promise.all([
  19             db.selectFirstRow('platforms', {name: 'Mountain Lion'}),
  20             db.selectFirstRow('test_metrics', {name: 'Time'}),
  21         ]).then(function (result) {
  22             return {platformId: result[0]['id'], metricId: result[1]['id']};
  23         });
  24     }
  25
  26     function format(formatMap, row)
  27     {
  28         var result = {};
  29         for (var i = 0; i < formatMap.length; i++) {
  30             var key = formatMap[i];
  31             if (key == 'id' || key == 'build' || key == 'builder')
  32                 continue;
  33             result[key] = row[i];
  34         }
  35         return result;
  36     }
  37
  38     let clusterStart = TestServer.testConfig().clusterStart;
  39     clusterStart = +Date.UTC(clusterStart[0], clusterStart[1] - 1, clusterStart[2], clusterStart[3], clusterStart[4]);
  40
  41     let clusterSize = TestServer.testConfig().clusterSize;
  42     const DAY = 24 * 3600 * 1000;
  43     const YEAR = 365.24 * DAY;
  44     const MONTH = 30 * DAY;
  45     clusterSize = clusterSize[0] * YEAR + clusterSize[1] * MONTH + clusterSize[2] * DAY;
  46
  47     function clusterTime(index) { return new Date(clusterStart + clusterSize * index); }
  48
  49     const reportWithBuildTime = [{
  50         "buildNumber": "123",
  51         "buildTime": clusterTime(7.8).toISOString(),
  52         "builderName": "someBuilder",
  53         "builderPassword": "somePassword",
  54         "platform": "Mountain Lion",
  55         "tests": {
  56             "Suite": {
  57                 "tests": {
  58                     "test1": {
  59                         "metrics": {"Time": { "current": [1, 2, 3, 4, 5] }}
  60                     },
  61                 }
  62             },
  63         }}];
  64     reportWithBuildTime.startTime = +clusterTime(7);
  65
  66     const reportWithRevision = [{
  67         "buildNumber": "124",
  68         "buildTime": "2013-02-28T15:34:51",
  69         "revisions": {
  70             "WebKit": {
  71                 "revision": "144000",
  72                 "timestamp": clusterTime(10.35645364537).toISOString(),
  73             },
  74         },
  75         "builderName": "someBuilder",
  76         "builderPassword": "somePassword",
  77         "platform": "Mountain Lion",
  78         "tests": {
  79             "Suite": {
  80                 "tests": {
  81                     "test1": {
  82                         "metrics": {"Time": { "current": [11, 12, 13, 14, 15] }}
  83                     }
  84                 }
  85             },
  86         }}];
  87
  88     const reportWithNewRevision = [{
  89         "buildNumber": "125",
  90         "buildTime": "2013-02-28T21:45:17",
  91         "revisions": {
  92             "WebKit": {
  93                 "revision": "160609",
  94                 "timestamp": clusterTime(12.1).toISOString()
  95             },
  96         },
  97         "builderName": "someBuilder",
  98         "builderPassword": "somePassword",
  99         "platform": "Mountain Lion",
 100         "tests": {
 101             "Suite": {
 102                 "tests": {
 103                     "test1": {
 104                         "metrics": {"Time": { "current": [16, 17, 18, 19, 20] }}
 105                     }
 106                 }
 107             },
 108         }}];
 109
 110     const reportWithAncentRevision = [{
 111         "buildNumber": "126",
 112         "buildTime": "2013-02-28T23:07:25",
 113         "revisions": {
 114             "WebKit": {
 115                 "revision": "137793",
 116                 "timestamp": clusterTime(1.8).toISOString()
 117             },
 118         },
 119         "builderName": "someBuilder",
 120         "builderPassword": "somePassword",
 121         "platform": "Mountain Lion",
 122         "tests": {
 123             "Suite": {
 124                 "tests": {
 125                     "test1": {
 126                         "metrics": {"Time": { "current": [21, 22, 23, 24, 25] }}
 127                     }
 128                 }
 129             },
 130         }}];
 131
 132     it("should reject when platform ID is missing", () => {
 133         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
 134             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 135         }).then((response) => {
 136             assert.equal(response['status'], 'OK');
 137             return queryPlatformAndMetric('Mountain Lion', 'Time');
 138         }).then((result) => {
 139             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?metric=${result.metricId}`);
 140         }).then((response) => {
 141             assert.equal(response['status'], 'AmbiguousRequest');
 142         });
 143     });
 144
 145     it("should reject when metric ID is missing", () => {
 146         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
 147             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 148         }).then((response) => {
 149             assert.equal(response['status'], 'OK');
 150             return queryPlatformAndMetric('Mountain Lion', 'Time');
 151         }).then((result) => {
 152             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}`);
 153         }).then((response) => {
 154             assert.equal(response['status'], 'AmbiguousRequest');
 155         });
 156     });
 157
 158     it("should reject an invalid platform name", () => {
 159         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
 160             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 161         }).then((response) => {
 162             assert.equal(response['status'], 'OK');
 163             return queryPlatformAndMetric('Mountain Lion', 'Time');
 164         }).then((result) => {
 165             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}a&metric=${result.metricId}`);
 166         }).then((response) => {
 167             assert.equal(response['status'], 'InvalidPlatform');
 168         });
 169     });
 170
 171     it("should reject an invalid metric name", () => {
 172         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
 173             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 174         }).then((response) => {
 175             assert.equal(response['status'], 'OK');
 176             return queryPlatformAndMetric('Mountain Lion', 'Time');
 177         }).then((result) => {
 178             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}b`);
 179         }).then((response) => {
 180             assert.equal(response['status'], 'InvalidMetric');
 181         });
 182     });
 183
 184     it("should return 404 when the report is empty", () => {
 185         const db = TestServer.database();
 186         return Promise.all([
 187             db.insert('tests', {id: 1, name: 'SomeTest'}),
 188             db.insert('tests', {id: 2, name: 'SomeOtherTest'}),
 189             db.insert('tests', {id: 3, name: 'ChildTest', parent: 1}),
 190             db.insert('tests', {id: 4, name: 'GrandChild', parent: 3}),
 191             db.insert('aggregators', {id: 200, name: 'Total'}),
 192             db.insert('test_metrics', {id: 5, test: 1, name: 'Time'}),
 193             db.insert('test_metrics', {id: 6, test: 2, name: 'Time', aggregator: 200}),
 194             db.insert('test_metrics', {id: 7, test: 2, name: 'Malloc', aggregator: 200}),
 195             db.insert('test_metrics', {id: 8, test: 3, name: 'Time'}),
 196             db.insert('test_metrics', {id: 9, test: 4, name: 'Time'}),
 197             db.insert('platforms', {id: 23, name: 'iOS 9 iPhone 5s'}),
 198             db.insert('platforms', {id: 46, name: 'Trunk Mavericks'}),
 199             db.insert('test_configurations', {id: 101, metric: 5, platform: 46, type: 'current'}),
 200             db.insert('test_configurations', {id: 102, metric: 6, platform: 46, type: 'current'}),
 201             db.insert('test_configurations', {id: 103, metric: 7, platform: 46, type: 'current'}),
 202             db.insert('test_configurations', {id: 104, metric: 8, platform: 46, type: 'current'}),
 203             db.insert('test_configurations', {id: 105, metric: 9, platform: 46, type: 'current'}),
 204             db.insert('test_configurations', {id: 106, metric: 5, platform: 23, type: 'current'}),
 205             db.insert('test_configurations', {id: 107, metric: 5, platform: 23, type: 'baseline'}),
 206         ]).then(() => {
 207             return TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=46&metric=5`).then((response) => {
 208                 assert(false);
 209             }, (error) => {
 210                 assert.equal(error, 404);
 211             });
 212         });
 213     });
 214
 215     it("should be able to retrieve a reported value", () => {
 216         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
 217             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 218         }).then((response) => {
 219             assert.equal(response['status'], 'OK');
 220             return queryPlatformAndMetric('Mountain Lion', 'Time');
 221         }).then((result) => {
 222             return TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 223         }).then((response) => {
 224             const buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
 225
 226             assert.deepEqual(Object.keys(response).sort(),
 227                 ['clusterCount', 'clusterSize', 'clusterStart',
 228                   'configurations', 'elapsedTime', 'endTime', 'formatMap', 'lastModified', 'startTime', 'status']);
 229             assert.equal(response['status'], 'OK');
 230             assert.equal(response['clusterCount'], 1);
 231             assert.deepEqual(response['formatMap'], [
 232                 'id', 'mean', 'iterationCount', 'sum', 'squareSum', 'markedOutlier',
 233                 'revisions', 'commitTime', 'build', 'buildTime', 'buildNumber', 'builder']);
 234
 235             assert.equal(response['startTime'], reportWithBuildTime.startTime);
 236             assert(typeof(response['lastModified']) == 'number', 'lastModified time should be a numeric');
 237
 238             assert.deepEqual(Object.keys(response['configurations']), ['current']);
 239
 240             const currentRows = response['configurations']['current'];
 241             assert.equal(currentRows.length, 1);
 242             assert.equal(currentRows[0].length, response['formatMap'].length);
 243             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
 244                 mean: 3,
 245                 iterationCount: 5,
 246                 sum: 15,
 247                 squareSum: 55,
 248                 markedOutlier: false,
 249                 revisions: [],
 250                 commitTime: buildTime,
 251                 buildTime: buildTime,
 252                 buildNumber: '123'});
 253         });
 254     });
 255
 256     it("should return return the right IDs for measurement, build, and builder", () => {
 257         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
 258             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
 259         }).then((response) => {
 260             assert.equal(response['status'], 'OK');
 261             return queryPlatformAndMetric('Mountain Lion', 'Time');
 262         }).then((result) => {
 263             const db = TestServer.database();
 264             return Promise.all([
 265                 db.selectAll('test_runs'),
 266                 db.selectAll('builds'),
 267                 db.selectAll('builders'),
 268                 TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`),
 269             ]);
 270         }).then((result) => {
 271             const runs = result[0];
 272             const builds = result[1];
 273             const builders = result[2];
 274             const response = result[3];
 275
 276             assert.equal(runs.length, 1);
 277             assert.equal(builds.length, 1);
 278             assert.equal(builders.length, 1);
 279             const measurementId = runs[0]['id'];
 280             const buildId = builds[0]['id'];
 281             const builderId = builders[0]['id'];
 282
 283             assert.equal(response['configurations']['current'].length, 1);
 284             const measurement = response['configurations']['current'][0];
 285             assert.equal(response['status'], 'OK');
 286
 287             assert.equal(measurement[response['formatMap'].indexOf('id')], measurementId);
 288             assert.equal(measurement[response['formatMap'].indexOf('build')], buildId);
 289             assert.equal(measurement[response['formatMap'].indexOf('builder')], builderId);
 290         });
 291     });
 292
 293     function queryPlatformAndMetricWithRepository(platformName, metricName, repositoryName)
 294     {
 295         const db = TestServer.database();
 296         return Promise.all([
 297             db.selectFirstRow('platforms', {name: platformName}),
 298             db.selectFirstRow('test_metrics', {name: metricName}),
 299             db.selectFirstRow('repositories', {name: repositoryName}),
 300         ]).then((result) => ({platformId: result[0]['id'], metricId: result[1]['id'], repositoryId: result[2]['id']}));
 301     }
 302
 303     it("should order results by commit time", () => {
 304         const remote = TestServer.remoteAPI();
 305         let repositoryId;
 306         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
 307             return remote.postJSON('/api/report/', reportWithBuildTime);
 308         }).then(() => {
 309             return remote.postJSON('/api/report/', reportWithRevision);
 310         }).then(() => {
 311             return queryPlatformAndMetricWithRepository('Mountain Lion', 'Time', 'WebKit');
 312         }).then((result) => {
 313             repositoryId = result.repositoryId;
 314             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 315         }).then((response) => {
 316             const currentRows = response['configurations']['current'];
 317             const buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
 318             const revisionTime = +(new Date(reportWithRevision[0]['revisions']['WebKit']['timestamp']));
 319             const revisionBuildTime = +(new Date(reportWithRevision[0]['buildTime']));
 320
 321             assert.equal(currentRows.length, 2);
 322             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
 323                mean: 13,
 324                iterationCount: 5,
 325                sum: 65,
 326                squareSum: 855,
 327                markedOutlier: false,
 328                revisions: [[1, repositoryId, '144000', revisionTime]],
 329                commitTime: revisionTime,
 330                buildTime: revisionBuildTime,
 331                buildNumber: '124' });
 332             assert.deepEqual(format(response['formatMap'], currentRows[1]), {
 333                 mean: 3,
 334                 iterationCount: 5,
 335                 sum: 15,
 336                 squareSum: 55,
 337                 markedOutlier: false,
 338                 revisions: [],
 339                 commitTime: buildTime,
 340                 buildTime: buildTime,
 341                 buildNumber: '123' });
 342         });
 343     });
 344
 345     it("should order results by build time when commit times are missing", () => {
 346         const remote = TestServer.remoteAPI();
 347         let repositoryId;
 348         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
 349             const db = TestServer.database();
 350             return Promise.all([
 351                 db.insert('repositories', {'id': 1, 'name': 'macOS'}),
 352                 db.insert('commits', {'id': 2, 'repository': 1, 'revision': 'macOS 16A323', 'order': 0}),
 353                 db.insert('commits', {'id': 3, 'repository': 1, 'revision': 'macOS 16C68', 'order': 1}),
 354             ]);
 355         }).then(() => {
 356             return remote.postJSON('/api/report/', [{
 357                 "buildNumber": "1001",
 358                 "buildTime": '2017-01-19 15:28:01',
 359                 "revisions": {
 360                     "macOS": {
 361                         "revision": "macOS 16C68",
 362                     },
 363                 },
 364                 "builderName": "someBuilder",
 365                 "builderPassword": "somePassword",
 366                 "platform": "Sierra",
 367                 "tests": { "Test": {"metrics": {"Time": { "baseline": [1, 2, 3, 4, 5] } } } },
 368             }]);
 369         }).then(() => {
 370             return remote.postJSON('/api/report/', [{
 371                 "buildNumber": "1002",
 372                 "buildTime": '2017-01-19 19:46:37',
 373                 "revisions": {
 374                     "macOS": {
 375                         "revision": "macOS 16A323",
 376                     },
 377                 },
 378                 "builderName": "someBuilder",
 379                 "builderPassword": "somePassword",
 380                 "platform": "Sierra",
 381                 "tests": { "Test": {"metrics": {"Time": { "baseline": [5, 6, 7, 8, 9] } } } },
 382             }]);
 383         }).then(() => {
 384             return queryPlatformAndMetricWithRepository('Sierra', 'Time', 'macOS');
 385         }).then((result) => {
 386             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 387         }).then((response) => {
 388             const currentRows = response['configurations']['baseline'];
 389             assert.equal(currentRows.length, 2);
 390             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
 391                mean: 3,
 392                iterationCount: 5,
 393                sum: 15,
 394                squareSum: 55,
 395                markedOutlier: false,
 396                revisions: [[3, 1, 'macOS 16C68', 0]],
 397                commitTime: +Date.UTC(2017, 0, 19, 15, 28, 1),
 398                buildTime: +Date.UTC(2017, 0, 19, 15, 28, 1),
 399                buildNumber: '1001' });
 400             assert.deepEqual(format(response['formatMap'], currentRows[1]), {
 401                 mean: 7,
 402                 iterationCount: 5,
 403                 sum: 35,
 404                 squareSum: 255,
 405                 markedOutlier: false,
 406                 revisions: [[2, 1, 'macOS 16A323', 0]],
 407                 commitTime: +Date.UTC(2017, 0, 19, 19, 46, 37),
 408                 buildTime: +Date.UTC(2017, 0, 19, 19, 46, 37),
 409                 buildNumber: '1002' });
 410         });
 411     });
 412
 413     function buildNumbers(parsedResult, config)
 414     {
 415         return parsedResult['configurations'][config].map((row) => format(parsedResult['formatMap'], row)['buildNumber']);
 416     }
 417
 418     it("should include one data point after the current time range", () => {
 419         const remote = TestServer.remoteAPI();
 420         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
 421             return remote.postJSON('/api/report/', reportWithAncentRevision);
 422         }).then(() => {
 423             return remote.postJSON('/api/report/', reportWithNewRevision);
 424         }).then(() => {
 425             return queryPlatformAndMetric('Mountain Lion', 'Time');
 426         }).then((result) => {
 427             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 428         }).then((response) => {
 429             assert.equal(response['status'], 'OK');
 430             assert.equal(response['clusterCount'], 2, 'should have two clusters');
 431             assert.deepEqual(buildNumbers(response, 'current'),
 432                 [reportWithAncentRevision[0]['buildNumber'], reportWithNewRevision[0]['buildNumber']]);
 433         });
 434     });
 435
 436     it("should always include one old data point before the current time range", () => {
 437         const remote = TestServer.remoteAPI();
 438         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
 439             return remote.postJSON('/api/report/', reportWithBuildTime);
 440         }).then(() => {
 441             return remote.postJSON('/api/report/', reportWithAncentRevision);
 442         }).then(() => {
 443             return queryPlatformAndMetric('Mountain Lion', 'Time');
 444         }).then((result) => {
 445             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 446         }).then((response) => {
 447             assert.equal(response['clusterCount'], 2, 'should have two clusters');
 448             let currentRows = response['configurations']['current'];
 449             assert.equal(currentRows.length, 2, 'should contain two data points');
 450             assert.deepEqual(buildNumbers(response, 'current'), [reportWithAncentRevision[0]['buildNumber'], reportWithBuildTime[0]['buildNumber']]);
 451         });
 452     });
 453
 454     it("should create cached results", () => {
 455         const remote = TestServer.remoteAPI();
 456         let cachePrefix;
 457         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
 458             return remote.postJSON('/api/report/', reportWithAncentRevision);
 459         }).then(() => {
 460             return remote.postJSON('/api/report/', reportWithRevision);
 461         }).then(() => {
 462             return remote.postJSON('/api/report/', reportWithNewRevision);
 463         }).then(() => {
 464             return queryPlatformAndMetric('Mountain Lion', 'Time');
 465         }).then((result) => {
 466             cachePrefix = '/data/measurement-set-' + result.platformId + '-' + result.metricId;
 467             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 468         }).then((newResult) => {
 469             return remote.getJSONWithStatus(`${cachePrefix}.json`).then((cachedResult) => {
 470                 assert.deepEqual(newResult, cachedResult);
 471                 return remote.getJSONWithStatus(`${cachePrefix}-${cachedResult['startTime']}.json`);
 472             }).then((oldResult) => {
 473                 const oldBuildNumbers = buildNumbers(oldResult, 'current');
 474                 const newBuildNumbers = buildNumbers(newResult, 'current');
 475                 assert(oldBuildNumbers.length >= 2, 'The old cluster should contain at least two data points');
 476                 assert(newBuildNumbers.length >= 2, 'The new cluster should contain at least two data points');
 477                 assert.deepEqual(oldBuildNumbers.slice(oldBuildNumbers.length - 2), newBuildNumbers.slice(0, 2),
 478                     'Two conseqcutive clusters should share two data points');
 479             });
 480         });
 481     });
 482
 483     it("should use lastModified timestamp identical to that in the manifest file", () => {
 484         const remote = TestServer.remoteAPI();
 485         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
 486             return remote.postJSON('/api/report/', reportWithRevision);
 487         }).then(() => {
 488             return queryPlatformAndMetric('Mountain Lion', 'Time');
 489         }).then((result) => {
 490             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
 491         }).then((primaryCluster) => {
 492             return remote.getJSONWithStatus('/api/manifest').then((content) => {
 493                 const manifest = Manifest._didFetchManifest(content);
 494
 495                 const platform = Platform.findByName('Mountain Lion');
 496                 assert.equal(Metric.all().length, 1);
 497                 const metric = Metric.all()[0];
 498                 assert.equal(platform.lastModified(metric), primaryCluster['lastModified']);
 499             });
 500         });
 501     });
 502
 503 });