measurement-sets API can incorrectly order points with OS version without commit...
[WebKit.git] / Websites / perf.webkit.org / server-tests / api-measurement-set-tests.js
1 'use strict';
2
3 const assert = require('assert');
4
5 require('../tools/js/v3-models.js');
6
7 const MockData = require('./resources/mock-data.js');
8 const TestServer = require('./resources/test-server.js');
9 const addBuilderForReport = require('./resources/common-operations.js').addBuilderForReport;
10 const connectToDatabaseInEveryTest = require('./resources/common-operations.js').connectToDatabaseInEveryTest;
11
12 describe("/api/measurement-set", function () {
13     this.timeout(1000);
14     TestServer.inject();
15     connectToDatabaseInEveryTest();
16
17     beforeEach(function () {
18         MockData.resetV3Models();
19     });
20
21     function queryPlatformAndMetric(platformName, metricName)
22     {
23         const db = TestServer.database();
24         return Promise.all([
25             db.selectFirstRow('platforms', {name: 'Mountain Lion'}),
26             db.selectFirstRow('test_metrics', {name: 'Time'}),
27         ]).then(function (result) {
28             return {platformId: result[0]['id'], metricId: result[1]['id']};
29         });
30     }
31
32     function format(formatMap, row)
33     {
34         var result = {};
35         for (var i = 0; i < formatMap.length; i++) {
36             var key = formatMap[i];
37             if (key == 'id' || key == 'build' || key == 'builder')
38                 continue;
39             result[key] = row[i];
40         }
41         return result;
42     }
43
44     let clusterStart = TestServer.testConfig().clusterStart;
45     clusterStart = +Date.UTC(clusterStart[0], clusterStart[1] - 1, clusterStart[2], clusterStart[3], clusterStart[4]);
46
47     let clusterSize = TestServer.testConfig().clusterSize;
48     const DAY = 24 * 3600 * 1000;
49     const YEAR = 365.24 * DAY;
50     const MONTH = 30 * DAY;
51     clusterSize = clusterSize[0] * YEAR + clusterSize[1] * MONTH + clusterSize[2] * DAY;
52
53     function clusterTime(index) { return new Date(clusterStart + clusterSize * index); }
54
55     const reportWithBuildTime = [{
56         "buildNumber": "123",
57         "buildTime": clusterTime(7.8).toISOString(),
58         "builderName": "someBuilder",
59         "builderPassword": "somePassword",
60         "platform": "Mountain Lion",
61         "tests": {
62             "Suite": {
63                 "tests": {
64                     "test1": {
65                         "metrics": {"Time": { "current": [1, 2, 3, 4, 5] }}
66                     },
67                 }
68             },
69         }}];
70     reportWithBuildTime.startTime = +clusterTime(7);
71
72     const reportWithRevision = [{
73         "buildNumber": "124",
74         "buildTime": "2013-02-28T15:34:51",
75         "revisions": {
76             "WebKit": {
77                 "revision": "144000",
78                 "timestamp": clusterTime(10.35645364537).toISOString(),
79             },
80         },
81         "builderName": "someBuilder",
82         "builderPassword": "somePassword",
83         "platform": "Mountain Lion",
84         "tests": {
85             "Suite": {
86                 "tests": {
87                     "test1": {
88                         "metrics": {"Time": { "current": [11, 12, 13, 14, 15] }}
89                     }
90                 }
91             },
92         }}];
93
94     const reportWithNewRevision = [{
95         "buildNumber": "125",
96         "buildTime": "2013-02-28T21:45:17",
97         "revisions": {
98             "WebKit": {
99                 "revision": "160609",
100                 "timestamp": clusterTime(12.1).toISOString()
101             },
102         },
103         "builderName": "someBuilder",
104         "builderPassword": "somePassword",
105         "platform": "Mountain Lion",
106         "tests": {
107             "Suite": {
108                 "tests": {
109                     "test1": {
110                         "metrics": {"Time": { "current": [16, 17, 18, 19, 20] }}
111                     }
112                 }
113             },
114         }}];
115
116     const reportWithAncentRevision = [{
117         "buildNumber": "126",
118         "buildTime": "2013-02-28T23:07:25",
119         "revisions": {
120             "WebKit": {
121                 "revision": "137793",
122                 "timestamp": clusterTime(1.8).toISOString()
123             },
124         },
125         "builderName": "someBuilder",
126         "builderPassword": "somePassword",
127         "platform": "Mountain Lion",
128         "tests": {
129             "Suite": {
130                 "tests": {
131                     "test1": {
132                         "metrics": {"Time": { "current": [21, 22, 23, 24, 25] }}
133                     }
134                 }
135             },
136         }}];
137
138     it("should reject when platform ID is missing", function (done) {
139         addBuilderForReport(reportWithBuildTime[0]).then(function () {
140             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
141         }).then(function (response) {
142             assert.equal(response['status'], 'OK');
143             return queryPlatformAndMetric('Mountain Lion', 'Time');
144         }).then(function (result) {
145             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?metric=${result.metricId}`);
146         }).then(function (response) {
147             assert.equal(response['status'], 'AmbiguousRequest');
148             done();
149         }).catch(done);
150     });
151
152     it("should reject when metric ID is missing", function (done) {
153         addBuilderForReport(reportWithBuildTime[0]).then(function () {
154             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
155         }).then(function (response) {
156             assert.equal(response['status'], 'OK');
157             return queryPlatformAndMetric('Mountain Lion', 'Time');
158         }).then(function (result) {
159             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}`);
160         }).then(function (response) {
161             assert.equal(response['status'], 'AmbiguousRequest');
162             done();
163         }).catch(done);
164     });
165
166     it("should reject an invalid platform name", function (done) {
167         addBuilderForReport(reportWithBuildTime[0]).then(function () {
168             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
169         }).then(function (response) {
170             assert.equal(response['status'], 'OK');
171             return queryPlatformAndMetric('Mountain Lion', 'Time');
172         }).then(function (result) {
173             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}a&metric=${result.metricId}`);
174         }).then(function (response) {
175             assert.equal(response['status'], 'InvalidPlatform');
176             done();
177         }).catch(done);
178     });
179
180     it("should reject an invalid metric name", function (done) {
181         addBuilderForReport(reportWithBuildTime[0]).then(function () {
182             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
183         }).then(function (response) {
184             assert.equal(response['status'], 'OK');
185             return queryPlatformAndMetric('Mountain Lion', 'Time');
186         }).then(function (result) {
187             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}b`);
188         }).then(function (response) {
189             assert.equal(response['status'], 'InvalidMetric');
190             done();
191         }).catch(done);
192     });
193
194     it("should be able to return an empty report", function (done) {
195         const db = TestServer.database();
196         Promise.all([
197             db.insert('tests', {id: 1, name: 'SomeTest'}),
198             db.insert('tests', {id: 2, name: 'SomeOtherTest'}),
199             db.insert('tests', {id: 3, name: 'ChildTest', parent: 1}),
200             db.insert('tests', {id: 4, name: 'GrandChild', parent: 3}),
201             db.insert('aggregators', {id: 200, name: 'Total'}),
202             db.insert('test_metrics', {id: 5, test: 1, name: 'Time'}),
203             db.insert('test_metrics', {id: 6, test: 2, name: 'Time', aggregator: 200}),
204             db.insert('test_metrics', {id: 7, test: 2, name: 'Malloc', aggregator: 200}),
205             db.insert('test_metrics', {id: 8, test: 3, name: 'Time'}),
206             db.insert('test_metrics', {id: 9, test: 4, name: 'Time'}),
207             db.insert('platforms', {id: 23, name: 'iOS 9 iPhone 5s'}),
208             db.insert('platforms', {id: 46, name: 'Trunk Mavericks'}),
209             db.insert('test_configurations', {id: 101, metric: 5, platform: 46, type: 'current'}),
210             db.insert('test_configurations', {id: 102, metric: 6, platform: 46, type: 'current'}),
211             db.insert('test_configurations', {id: 103, metric: 7, platform: 46, type: 'current'}),
212             db.insert('test_configurations', {id: 104, metric: 8, platform: 46, type: 'current'}),
213             db.insert('test_configurations', {id: 105, metric: 9, platform: 46, type: 'current'}),
214             db.insert('test_configurations', {id: 106, metric: 5, platform: 23, type: 'current'}),
215             db.insert('test_configurations', {id: 107, metric: 5, platform: 23, type: 'baseline'}),
216         ]).then(function () {
217             return TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=46&metric=5`).then(function (response) {
218                 assert.equal(response.statusCode, 404);
219             }, function (error) {
220                 assert.equal(error, 404);
221                 done();
222             });
223         }).catch(done);
224     });
225
226     it("should be able to retrieve a reported value", function (done) {
227         addBuilderForReport(reportWithBuildTime[0]).then(function () {
228             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
229         }).then(function (response) {
230             assert.equal(response['status'], 'OK');
231             return queryPlatformAndMetric('Mountain Lion', 'Time');
232         }).then(function (result) {
233             return TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
234         }).then(function (response) {
235             const buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
236
237             assert.deepEqual(Object.keys(response).sort(),
238                 ['clusterCount', 'clusterSize', 'clusterStart',
239                   'configurations', 'elapsedTime', 'endTime', 'formatMap', 'lastModified', 'startTime', 'status']);
240             assert.equal(response['status'], 'OK');
241             assert.equal(response['clusterCount'], 1);
242             assert.deepEqual(response['formatMap'], [
243                 'id', 'mean', 'iterationCount', 'sum', 'squareSum', 'markedOutlier',
244                 'revisions', 'commitTime', 'build', 'buildTime', 'buildNumber', 'builder']);
245
246             assert.equal(response['startTime'], reportWithBuildTime.startTime);
247             assert(typeof(response['lastModified']) == 'number', 'lastModified time should be a numeric');
248
249             assert.deepEqual(Object.keys(response['configurations']), ['current']);
250
251             var currentRows = response['configurations']['current'];
252             assert.equal(currentRows.length, 1);
253             assert.equal(currentRows[0].length, response['formatMap'].length);
254             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
255                 mean: 3,
256                 iterationCount: 5,
257                 sum: 15,
258                 squareSum: 55,
259                 markedOutlier: false,
260                 revisions: [],
261                 commitTime: buildTime,
262                 buildTime: buildTime,
263                 buildNumber: '123'});
264             done();
265         }).catch(done);
266     });
267
268     it("should return return the right IDs for measurement, build, and builder", function (done) {
269         addBuilderForReport(reportWithBuildTime[0]).then(function () {
270             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
271         }).then(function (response) {
272             assert.equal(response['status'], 'OK');
273             return queryPlatformAndMetric('Mountain Lion', 'Time');
274         }).then(function (result) {
275             const db = TestServer.database();
276             return Promise.all([
277                 db.selectAll('test_runs'),
278                 db.selectAll('builds'),
279                 db.selectAll('builders'),
280                 TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`),
281             ]);
282         }).then(function (result) {
283             const runs = result[0];
284             const builds = result[1];
285             const builders = result[2];
286             const response = result[3];
287
288             assert.equal(runs.length, 1);
289             assert.equal(builds.length, 1);
290             assert.equal(builders.length, 1);
291             const measurementId = runs[0]['id'];
292             const buildId = builds[0]['id'];
293             const builderId = builders[0]['id'];
294
295             assert.equal(response['configurations']['current'].length, 1);
296             const measurement = response['configurations']['current'][0];
297             assert.equal(response['status'], 'OK');
298
299             assert.equal(measurement[response['formatMap'].indexOf('id')], measurementId);
300             assert.equal(measurement[response['formatMap'].indexOf('build')], buildId);
301             assert.equal(measurement[response['formatMap'].indexOf('builder')], builderId);
302
303             done();
304         }).catch(done);
305     });
306
307     function postReports(reports, callback)
308     {
309         if (!reports.length)
310             return callback();
311
312         postJSON('/api/report/', reports[0], function (response) {
313             assert.equal(response.statusCode, 200);
314             assert.equal(JSON.parse(response.responseText)['status'], 'OK');
315
316             postReports(reports.slice(1), callback);
317         });
318     }
319
320     function queryPlatformAndMetricWithRepository(platformName, metricName, repositoryName)
321     {
322         const db = TestServer.database();
323         return Promise.all([
324             db.selectFirstRow('platforms', {name: platformName}),
325             db.selectFirstRow('test_metrics', {name: metricName}),
326             db.selectFirstRow('repositories', {name: repositoryName}),
327         ]).then(function (result) {
328             return {platformId: result[0]['id'], metricId: result[1]['id'], repositoryId: result[2]['id']};
329         });
330     }
331
332     it("should order results by commit time", function (done) {
333         const remote = TestServer.remoteAPI();
334         let repositoryId;
335         addBuilderForReport(reportWithBuildTime[0]).then(function () {
336             return remote.postJSON('/api/report/', reportWithBuildTime);
337         }).then(function () {
338             return remote.postJSON('/api/report/', reportWithRevision);
339         }).then(function () {
340             return queryPlatformAndMetricWithRepository('Mountain Lion', 'Time', 'WebKit');
341         }).then(function (result) {
342             repositoryId = result.repositoryId;
343             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
344         }).then(function (response) {
345             const currentRows = response['configurations']['current'];
346             const buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
347             const revisionTime = +(new Date(reportWithRevision[0]['revisions']['WebKit']['timestamp']));
348             const revisionBuildTime = +(new Date(reportWithRevision[0]['buildTime']));
349
350             assert.equal(currentRows.length, 2);
351             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
352                mean: 13,
353                iterationCount: 5,
354                sum: 65,
355                squareSum: 855,
356                markedOutlier: false,
357                revisions: [[1, repositoryId, '144000', revisionTime]],
358                commitTime: revisionTime,
359                buildTime: revisionBuildTime,
360                buildNumber: '124' });
361             assert.deepEqual(format(response['formatMap'], currentRows[1]), {
362                 mean: 3,
363                 iterationCount: 5,
364                 sum: 15,
365                 squareSum: 55,
366                 markedOutlier: false,
367                 revisions: [],
368                 commitTime: buildTime,
369                 buildTime: buildTime,
370                 buildNumber: '123' });
371             done();
372         }).catch(done);
373     });
374
375     it("should order results by build time when commit times are missing", function (done) {
376         const remote = TestServer.remoteAPI();
377         let repositoryId;
378         addBuilderForReport(reportWithBuildTime[0]).then(() => {
379             const db = TestServer.database();
380             return Promise.all([
381                 db.insert('repositories', {'id': 1, 'name': 'macOS'}),
382                 db.insert('commits', {'id': 2, 'repository': 1, 'revision': 'macOS 16A323', 'order': 0}),
383                 db.insert('commits', {'id': 3, 'repository': 1, 'revision': 'macOS 16C68', 'order': 1}),
384             ]);
385         }).then(() => {
386             return remote.postJSON('/api/report/', [{
387                 "buildNumber": "1001",
388                 "buildTime": '2017-01-19 15:28:01',
389                 "revisions": {
390                     "macOS": {
391                         "revision": "macOS 16C68",
392                     },
393                 },
394                 "builderName": "someBuilder",
395                 "builderPassword": "somePassword",
396                 "platform": "Sierra",
397                 "tests": { "Test": {"metrics": {"Time": { "baseline": [1, 2, 3, 4, 5] } } } },
398             }]);
399         }).then(function () {
400             return remote.postJSON('/api/report/', [{
401                 "buildNumber": "1002",
402                 "buildTime": '2017-01-19 19:46:37',
403                 "revisions": {
404                     "macOS": {
405                         "revision": "macOS 16A323",
406                     },
407                 },
408                 "builderName": "someBuilder",
409                 "builderPassword": "somePassword",
410                 "platform": "Sierra",
411                 "tests": { "Test": {"metrics": {"Time": { "baseline": [5, 6, 7, 8, 9] } } } },
412             }]);
413         }).then(function () {
414             return queryPlatformAndMetricWithRepository('Sierra', 'Time', 'macOS');
415         }).then(function (result) {
416             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
417         }).then(function (response) {
418             const currentRows = response['configurations']['baseline'];
419             assert.equal(currentRows.length, 2);
420             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
421                mean: 3,
422                iterationCount: 5,
423                sum: 15,
424                squareSum: 55,
425                markedOutlier: false,
426                revisions: [[3, 1, 'macOS 16C68', 0]],
427                commitTime: +Date.UTC(2017, 0, 19, 15, 28, 1),
428                buildTime: +Date.UTC(2017, 0, 19, 15, 28, 1),
429                buildNumber: '1001' });
430             assert.deepEqual(format(response['formatMap'], currentRows[1]), {
431                 mean: 7,
432                 iterationCount: 5,
433                 sum: 35,
434                 squareSum: 255,
435                 markedOutlier: false,
436                 revisions: [[2, 1, 'macOS 16A323', 0]],
437                 commitTime: +Date.UTC(2017, 0, 19, 19, 46, 37),
438                 buildTime: +Date.UTC(2017, 0, 19, 19, 46, 37),
439                 buildNumber: '1002' });
440             done();
441         }).catch(done);
442     });
443
444     function buildNumbers(parsedResult, config)
445     {
446         return parsedResult['configurations'][config].map(function (row) {
447             return format(parsedResult['formatMap'], row)['buildNumber'];
448         });
449     }
450
451     it("should include one data point after the current time range", function (done) {
452         const remote = TestServer.remoteAPI();
453         addBuilderForReport(reportWithBuildTime[0]).then(function () {
454             return remote.postJSON('/api/report/', reportWithAncentRevision);
455         }).then(function () {
456             return remote.postJSON('/api/report/', reportWithNewRevision);
457         }).then(function () {
458             return queryPlatformAndMetric('Mountain Lion', 'Time');
459         }).then(function (result) {
460             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
461         }).then(function (response) {
462             assert.equal(response['status'], 'OK');
463             assert.equal(response['clusterCount'], 2, 'should have two clusters');
464             assert.deepEqual(buildNumbers(response, 'current'),
465                 [reportWithAncentRevision[0]['buildNumber'], reportWithNewRevision[0]['buildNumber']]);
466             done();
467         }).catch(done);
468     });
469
470     it("should always include one old data point before the current time range", function (done) {
471         const remote = TestServer.remoteAPI();
472         addBuilderForReport(reportWithBuildTime[0]).then(function () {
473             return remote.postJSON('/api/report/', reportWithBuildTime);
474         }).then(function () {
475             return remote.postJSON('/api/report/', reportWithAncentRevision);
476         }).then(function () {
477             return queryPlatformAndMetric('Mountain Lion', 'Time');
478         }).then(function (result) {
479             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
480         }).then(function (response) {
481             assert.equal(response['clusterCount'], 2, 'should have two clusters');
482             let currentRows = response['configurations']['current'];
483             assert.equal(currentRows.length, 2, 'should contain two data points');
484             assert.deepEqual(buildNumbers(response, 'current'), [reportWithAncentRevision[0]['buildNumber'], reportWithBuildTime[0]['buildNumber']]);
485             done();
486         }).catch(done);
487     });
488
489     it("should create cached results", function (done) {
490         const remote = TestServer.remoteAPI();
491         let cachePrefix;
492         addBuilderForReport(reportWithBuildTime[0]).then(function () {
493             return remote.postJSON('/api/report/', reportWithAncentRevision);
494         }).then(function () {
495             return remote.postJSON('/api/report/', reportWithRevision);
496         }).then(function () {
497             return remote.postJSON('/api/report/', reportWithNewRevision);
498         }).then(function () {
499             return queryPlatformAndMetric('Mountain Lion', 'Time');
500         }).then(function (result) {
501             cachePrefix = '/data/measurement-set-' + result.platformId + '-' + result.metricId;
502             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
503         }).then(function (newResult) {
504             return remote.getJSONWithStatus(`${cachePrefix}.json`).then(function (cachedResult) {
505                 assert.deepEqual(newResult, cachedResult);
506                 return remote.getJSONWithStatus(`${cachePrefix}-${cachedResult['startTime']}.json`);
507             }).then(function (oldResult) {
508                 var oldBuildNumbers = buildNumbers(oldResult, 'current');
509                 var newBuildNumbers = buildNumbers(newResult, 'current');
510                 assert(oldBuildNumbers.length >= 2, 'The old cluster should contain at least two data points');
511                 assert(newBuildNumbers.length >= 2, 'The new cluster should contain at least two data points');
512                 assert.deepEqual(oldBuildNumbers.slice(oldBuildNumbers.length - 2), newBuildNumbers.slice(0, 2),
513                     'Two conseqcutive clusters should share two data points');
514                 done();
515             });
516         }).catch(done);
517     });
518
519     it("should use lastModified timestamp identical to that in the manifest file", function (done) {
520         const remote = TestServer.remoteAPI();
521         addBuilderForReport(reportWithBuildTime[0]).then(function () {
522             return remote.postJSON('/api/report/', reportWithRevision);
523         }).then(function () {
524             return queryPlatformAndMetric('Mountain Lion', 'Time');
525         }).then(function (result) {
526             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
527         }).then(function (primaryCluster) {
528             return remote.getJSONWithStatus('/api/manifest').then(function (content) {
529                 const manifest = Manifest._didFetchManifest(content);
530
531                 const platform = Platform.findByName('Mountain Lion');
532                 assert.equal(Metric.all().length, 1);
533                 const metric = Metric.all()[0];
534                 assert.equal(platform.lastModified(metric), primaryCluster['lastModified']);
535
536                 done();
537             });
538         }).catch(done);
539     });
540
541 });