Revision information returned by querying measurement set api with analysis task...
[WebKit.git] / Websites / perf.webkit.org / server-tests / api-measurement-set-tests.js
1 'use strict';
2
3 const assert = require('assert');
4
5 require('../tools/js/v3-models.js');
6
7 const MockData = require('./resources/mock-data.js');
8 const TestServer = require('./resources/test-server.js');
9 const addBuilderForReport = require('./resources/common-operations.js').addBuilderForReport;
10 const prepareServerTest = require('./resources/common-operations.js').prepareServerTest;
11
12 describe("/api/measurement-set", function () {
13     prepareServerTest(this);
14
15     function queryPlatformAndMetric(platformName, metricName)
16     {
17         const db = TestServer.database();
18         return Promise.all([
19             db.selectFirstRow('platforms', {name: 'Mountain Lion'}),
20             db.selectFirstRow('test_metrics', {name: 'Time'}),
21         ]).then(function (result) {
22             return {platformId: result[0]['id'], metricId: result[1]['id']};
23         });
24     }
25
26     function format(formatMap, row)
27     {
28         var result = {};
29         for (var i = 0; i < formatMap.length; i++) {
30             var key = formatMap[i];
31             if (key == 'id' || key == 'build' || key == 'builder')
32                 continue;
33             result[key] = row[i];
34         }
35         return result;
36     }
37
38     let clusterStart = TestServer.testConfig().clusterStart;
39     clusterStart = +Date.UTC(clusterStart[0], clusterStart[1] - 1, clusterStart[2], clusterStart[3], clusterStart[4]);
40
41     let clusterSize = TestServer.testConfig().clusterSize;
42     const DAY = 24 * 3600 * 1000;
43     const YEAR = 365.24 * DAY;
44     const MONTH = 30 * DAY;
45     clusterSize = clusterSize[0] * YEAR + clusterSize[1] * MONTH + clusterSize[2] * DAY;
46
47     function clusterTime(index) { return new Date(clusterStart + clusterSize * index); }
48
49     const reportWithBuildTime = [{
50         "buildNumber": "123",
51         "buildTime": clusterTime(7.8).toISOString(),
52         "builderName": "someBuilder",
53         "builderPassword": "somePassword",
54         "platform": "Mountain Lion",
55         "tests": {
56             "Suite": {
57                 "tests": {
58                     "test1": {
59                         "metrics": {"Time": { "current": [1, 2, 3, 4, 5] }}
60                     },
61                 }
62             },
63         }}];
64     reportWithBuildTime.startTime = +clusterTime(7);
65
66     const reportWithRevision = [{
67         "buildNumber": "124",
68         "buildTime": "2013-02-28T15:34:51Z",
69         "revisions": {
70             "WebKit": {
71                 "revision": "144000",
72                 "timestamp": clusterTime(10.35645364537).toISOString(),
73             },
74         },
75         "builderName": "someBuilder",
76         "builderPassword": "somePassword",
77         "platform": "Mountain Lion",
78         "tests": {
79             "Suite": {
80                 "tests": {
81                     "test1": {
82                         "metrics": {"Time": { "current": [11, 12, 13, 14, 15] }}
83                     }
84                 }
85             },
86         }}];
87
88     const reportWithNewRevision = [{
89         "buildNumber": "125",
90         "buildTime": "2013-02-28T21:45:17Z",
91         "revisions": {
92             "WebKit": {
93                 "revision": "160609",
94                 "timestamp": clusterTime(12.1).toISOString()
95             },
96         },
97         "builderName": "someBuilder",
98         "builderPassword": "somePassword",
99         "platform": "Mountain Lion",
100         "tests": {
101             "Suite": {
102                 "tests": {
103                     "test1": {
104                         "metrics": {"Time": { "current": [16, 17, 18, 19, 20] }}
105                     }
106                 }
107             },
108         }}];
109
110     const reportWithAncentRevision = [{
111         "buildNumber": "126",
112         "buildTime": "2013-02-28T23:07:25Z",
113         "revisions": {
114             "WebKit": {
115                 "revision": "137793",
116                 "timestamp": clusterTime(1.8).toISOString()
117             },
118         },
119         "builderName": "someBuilder",
120         "builderPassword": "somePassword",
121         "platform": "Mountain Lion",
122         "tests": {
123             "Suite": {
124                 "tests": {
125                     "test1": {
126                         "metrics": {"Time": { "current": [21, 22, 23, 24, 25] }}
127                     }
128                 }
129             },
130         }}];
131
132     it("should reject when platform ID is missing", () => {
133         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
134             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
135         }).then((response) => {
136             assert.equal(response['status'], 'OK');
137             return queryPlatformAndMetric('Mountain Lion', 'Time');
138         }).then((result) => {
139             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?metric=${result.metricId}`);
140         }).then((response) => {
141             assert.equal(response['status'], 'AmbiguousRequest');
142         });
143     });
144
145     it("should reject when metric ID is missing", () => {
146         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
147             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
148         }).then((response) => {
149             assert.equal(response['status'], 'OK');
150             return queryPlatformAndMetric('Mountain Lion', 'Time');
151         }).then((result) => {
152             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}`);
153         }).then((response) => {
154             assert.equal(response['status'], 'AmbiguousRequest');
155         });
156     });
157
158     it("should reject an invalid platform name", () => {
159         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
160             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
161         }).then((response) => {
162             assert.equal(response['status'], 'OK');
163             return queryPlatformAndMetric('Mountain Lion', 'Time');
164         }).then((result) => {
165             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}a&metric=${result.metricId}`);
166         }).then((response) => {
167             assert.equal(response['status'], 'InvalidPlatform');
168         });
169     });
170
171     it("should reject an invalid metric name", () => {
172         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
173             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
174         }).then((response) => {
175             assert.equal(response['status'], 'OK');
176             return queryPlatformAndMetric('Mountain Lion', 'Time');
177         }).then((result) => {
178             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}b`);
179         }).then((response) => {
180             assert.equal(response['status'], 'InvalidMetric');
181         });
182     });
183
184     it("should return 404 when the report is empty", () => {
185         const db = TestServer.database();
186         return Promise.all([
187             db.insert('tests', {id: 1, name: 'SomeTest'}),
188             db.insert('tests', {id: 2, name: 'SomeOtherTest'}),
189             db.insert('tests', {id: 3, name: 'ChildTest', parent: 1}),
190             db.insert('tests', {id: 4, name: 'GrandChild', parent: 3}),
191             db.insert('aggregators', {id: 200, name: 'Total'}),
192             db.insert('test_metrics', {id: 5, test: 1, name: 'Time'}),
193             db.insert('test_metrics', {id: 6, test: 2, name: 'Time', aggregator: 200}),
194             db.insert('test_metrics', {id: 7, test: 2, name: 'Malloc', aggregator: 200}),
195             db.insert('test_metrics', {id: 8, test: 3, name: 'Time'}),
196             db.insert('test_metrics', {id: 9, test: 4, name: 'Time'}),
197             db.insert('platforms', {id: 23, name: 'iOS 9 iPhone 5s'}),
198             db.insert('platforms', {id: 46, name: 'Trunk Mavericks'}),
199             db.insert('test_configurations', {id: 101, metric: 5, platform: 46, type: 'current'}),
200             db.insert('test_configurations', {id: 102, metric: 6, platform: 46, type: 'current'}),
201             db.insert('test_configurations', {id: 103, metric: 7, platform: 46, type: 'current'}),
202             db.insert('test_configurations', {id: 104, metric: 8, platform: 46, type: 'current'}),
203             db.insert('test_configurations', {id: 105, metric: 9, platform: 46, type: 'current'}),
204             db.insert('test_configurations', {id: 106, metric: 5, platform: 23, type: 'current'}),
205             db.insert('test_configurations', {id: 107, metric: 5, platform: 23, type: 'baseline'}),
206         ]).then(() => {
207             return TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=46&metric=5`).then((response) => {
208                 assert(false);
209             }, (error) => {
210                 assert.equal(error, 404);
211             });
212         });
213     });
214
215     it("should be able to retrieve a reported value", () => {
216         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
217             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
218         }).then((response) => {
219             assert.equal(response['status'], 'OK');
220             return queryPlatformAndMetric('Mountain Lion', 'Time');
221         }).then((result) => {
222             return TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
223         }).then((response) => {
224             const buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
225
226             assert.deepEqual(Object.keys(response).sort(),
227                 ['clusterCount', 'clusterSize', 'clusterStart',
228                   'configurations', 'elapsedTime', 'endTime', 'formatMap', 'lastModified', 'startTime', 'status']);
229             assert.equal(response['status'], 'OK');
230             assert.equal(response['clusterCount'], 1);
231             assert.deepEqual(response['formatMap'], [
232                 'id', 'mean', 'iterationCount', 'sum', 'squareSum', 'markedOutlier',
233                 'revisions', 'commitTime', 'build', 'buildTime', 'buildNumber', 'builder']);
234
235             assert.equal(response['startTime'], reportWithBuildTime.startTime);
236             assert(typeof(response['lastModified']) == 'number', 'lastModified time should be a numeric');
237
238             assert.deepEqual(Object.keys(response['configurations']), ['current']);
239
240             const currentRows = response['configurations']['current'];
241             assert.equal(currentRows.length, 1);
242             assert.equal(currentRows[0].length, response['formatMap'].length);
243             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
244                 mean: 3,
245                 iterationCount: 5,
246                 sum: 15,
247                 squareSum: 55,
248                 markedOutlier: false,
249                 revisions: [],
250                 commitTime: buildTime,
251                 buildTime: buildTime,
252                 buildNumber: '123'});
253         });
254     });
255
256     it("should return return the right IDs for measurement, build, and builder", () => {
257         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
258             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
259         }).then((response) => {
260             assert.equal(response['status'], 'OK');
261             return queryPlatformAndMetric('Mountain Lion', 'Time');
262         }).then((result) => {
263             const db = TestServer.database();
264             return Promise.all([
265                 db.selectAll('test_runs'),
266                 db.selectAll('builds'),
267                 db.selectAll('builders'),
268                 TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`),
269             ]);
270         }).then((result) => {
271             const runs = result[0];
272             const builds = result[1];
273             const builders = result[2];
274             const response = result[3];
275
276             assert.equal(runs.length, 1);
277             assert.equal(builds.length, 1);
278             assert.equal(builders.length, 1);
279             const measurementId = runs[0]['id'];
280             const buildId = builds[0]['id'];
281             const builderId = builders[0]['id'];
282
283             assert.equal(response['configurations']['current'].length, 1);
284             const measurement = response['configurations']['current'][0];
285             assert.equal(response['status'], 'OK');
286
287             assert.equal(measurement[response['formatMap'].indexOf('id')], measurementId);
288             assert.equal(measurement[response['formatMap'].indexOf('build')], buildId);
289             assert.equal(measurement[response['formatMap'].indexOf('builder')], builderId);
290         });
291     });
292
293     function queryPlatformAndMetricWithRepository(platformName, metricName, repositoryName)
294     {
295         const db = TestServer.database();
296         return Promise.all([
297             db.selectFirstRow('platforms', {name: platformName}),
298             db.selectFirstRow('test_metrics', {name: metricName}),
299             db.selectFirstRow('repositories', {name: repositoryName}),
300         ]).then((result) => ({platformId: result[0]['id'], metricId: result[1]['id'], repositoryId: result[2]['id']}));
301     }
302
303     it("should order results by commit time", () => {
304         const remote = TestServer.remoteAPI();
305         let repositoryId;
306         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
307             return remote.postJSON('/api/report/', reportWithBuildTime);
308         }).then(() => {
309             return remote.postJSON('/api/report/', reportWithRevision);
310         }).then(() => {
311             return queryPlatformAndMetricWithRepository('Mountain Lion', 'Time', 'WebKit');
312         }).then((result) => {
313             repositoryId = result.repositoryId;
314             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
315         }).then((response) => {
316             const currentRows = response['configurations']['current'];
317             const buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
318             const revisionTime = +(new Date(reportWithRevision[0]['revisions']['WebKit']['timestamp']));
319             const revisionBuildTime = +(new Date(reportWithRevision[0]['buildTime']));
320
321             assert.equal(currentRows.length, 2);
322             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
323                mean: 13,
324                iterationCount: 5,
325                sum: 65,
326                squareSum: 855,
327                markedOutlier: false,
328                revisions: [[1, repositoryId, '144000', null, revisionTime]],
329                commitTime: revisionTime,
330                buildTime: revisionBuildTime,
331                buildNumber: '124' });
332             assert.deepEqual(format(response['formatMap'], currentRows[1]), {
333                 mean: 3,
334                 iterationCount: 5,
335                 sum: 15,
336                 squareSum: 55,
337                 markedOutlier: false,
338                 revisions: [],
339                 commitTime: buildTime,
340                 buildTime: buildTime,
341                 buildNumber: '123' });
342         });
343     });
344
345     it("should order results by build time when commit times are missing", () => {
346         const remote = TestServer.remoteAPI();
347         let repositoryId;
348         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
349             const db = TestServer.database();
350             return Promise.all([
351                 db.insert('repositories', {'id': 1, 'name': 'macOS'}),
352                 db.insert('commits', {'id': 2, 'repository': 1, 'revision': 'macOS 16A323', 'order': 0}),
353                 db.insert('commits', {'id': 3, 'repository': 1, 'revision': 'macOS 16C68', 'order': 1}),
354             ]);
355         }).then(() => {
356             return remote.postJSON('/api/report/', [{
357                 "buildNumber": "1001",
358                 "buildTime": '2017-01-19 15:28:01',
359                 "revisions": {
360                     "macOS": {
361                         "revision": "macOS 16C68",
362                     },
363                 },
364                 "builderName": "someBuilder",
365                 "builderPassword": "somePassword",
366                 "platform": "Sierra",
367                 "tests": { "Test": {"metrics": {"Time": { "baseline": [1, 2, 3, 4, 5] } } } },
368             }]);
369         }).then(() => {
370             return remote.postJSON('/api/report/', [{
371                 "buildNumber": "1002",
372                 "buildTime": '2017-01-19 19:46:37',
373                 "revisions": {
374                     "macOS": {
375                         "revision": "macOS 16A323",
376                     },
377                 },
378                 "builderName": "someBuilder",
379                 "builderPassword": "somePassword",
380                 "platform": "Sierra",
381                 "tests": { "Test": {"metrics": {"Time": { "baseline": [5, 6, 7, 8, 9] } } } },
382             }]);
383         }).then(() => {
384             return queryPlatformAndMetricWithRepository('Sierra', 'Time', 'macOS');
385         }).then((result) => {
386             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
387         }).then((response) => {
388             const currentRows = response['configurations']['baseline'];
389             assert.equal(currentRows.length, 2);
390             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
391                mean: 3,
392                iterationCount: 5,
393                sum: 15,
394                squareSum: 55,
395                markedOutlier: false,
396                revisions: [[3, 1, 'macOS 16C68', 1, 0]],
397                commitTime: +Date.UTC(2017, 0, 19, 15, 28, 1),
398                buildTime: +Date.UTC(2017, 0, 19, 15, 28, 1),
399                buildNumber: '1001' });
400             assert.deepEqual(format(response['formatMap'], currentRows[1]), {
401                 mean: 7,
402                 iterationCount: 5,
403                 sum: 35,
404                 squareSum: 255,
405                 markedOutlier: false,
406                 revisions: [[2, 1, 'macOS 16A323', 0, 0]],
407                 commitTime: +Date.UTC(2017, 0, 19, 19, 46, 37),
408                 buildTime: +Date.UTC(2017, 0, 19, 19, 46, 37),
409                 buildNumber: '1002' });
410         });
411     });
412
413     function buildNumbers(parsedResult, config)
414     {
415         return parsedResult['configurations'][config].map((row) => format(parsedResult['formatMap'], row)['buildNumber']);
416     }
417
418     it("should include one data point after the current time range", () => {
419         const remote = TestServer.remoteAPI();
420         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
421             return remote.postJSON('/api/report/', reportWithAncentRevision);
422         }).then(() => {
423             return remote.postJSON('/api/report/', reportWithNewRevision);
424         }).then(() => {
425             return queryPlatformAndMetric('Mountain Lion', 'Time');
426         }).then((result) => {
427             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
428         }).then((response) => {
429             assert.equal(response['status'], 'OK');
430             assert.equal(response['clusterCount'], 2, 'should have two clusters');
431             assert.deepEqual(buildNumbers(response, 'current'),
432                 [reportWithAncentRevision[0]['buildNumber'], reportWithNewRevision[0]['buildNumber']]);
433         });
434     });
435
436     it("should always include one old data point before the current time range", () => {
437         const remote = TestServer.remoteAPI();
438         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
439             return remote.postJSON('/api/report/', reportWithBuildTime);
440         }).then(() => {
441             return remote.postJSON('/api/report/', reportWithAncentRevision);
442         }).then(() => {
443             return queryPlatformAndMetric('Mountain Lion', 'Time');
444         }).then((result) => {
445             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
446         }).then((response) => {
447             assert.equal(response['clusterCount'], 2, 'should have two clusters');
448             let currentRows = response['configurations']['current'];
449             assert.equal(currentRows.length, 2, 'should contain two data points');
450             assert.deepEqual(buildNumbers(response, 'current'), [reportWithAncentRevision[0]['buildNumber'], reportWithBuildTime[0]['buildNumber']]);
451         });
452     });
453
454     it("should create cached results", () => {
455         const remote = TestServer.remoteAPI();
456         let cachePrefix;
457         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
458             return remote.postJSON('/api/report/', reportWithAncentRevision);
459         }).then(() => {
460             return remote.postJSON('/api/report/', reportWithRevision);
461         }).then(() => {
462             return remote.postJSON('/api/report/', reportWithNewRevision);
463         }).then(() => {
464             return queryPlatformAndMetric('Mountain Lion', 'Time');
465         }).then((result) => {
466             cachePrefix = '/data/measurement-set-' + result.platformId + '-' + result.metricId;
467             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
468         }).then((newResult) => {
469             return remote.getJSONWithStatus(`${cachePrefix}.json`).then((cachedResult) => {
470                 assert.deepEqual(newResult, cachedResult);
471                 return remote.getJSONWithStatus(`${cachePrefix}-${cachedResult['startTime']}.json`);
472             }).then((oldResult) => {
473                 const oldBuildNumbers = buildNumbers(oldResult, 'current');
474                 const newBuildNumbers = buildNumbers(newResult, 'current');
475                 assert(oldBuildNumbers.length >= 2, 'The old cluster should contain at least two data points');
476                 assert(newBuildNumbers.length >= 2, 'The new cluster should contain at least two data points');
477                 assert.deepEqual(oldBuildNumbers.slice(oldBuildNumbers.length - 2), newBuildNumbers.slice(0, 2),
478                     'Two conseqcutive clusters should share two data points');
479             });
480         });
481     });
482
483     it("should use lastModified timestamp identical to that in the manifest file", () => {
484         const remote = TestServer.remoteAPI();
485         return addBuilderForReport(reportWithBuildTime[0]).then(() => {
486             return remote.postJSON('/api/report/', reportWithRevision);
487         }).then(() => {
488             return queryPlatformAndMetric('Mountain Lion', 'Time');
489         }).then((result) => {
490             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
491         }).then((primaryCluster) => {
492             return remote.getJSONWithStatus('/api/manifest').then((content) => {
493                 const manifest = Manifest._didFetchManifest(content);
494
495                 const platform = Platform.findByName('Mountain Lion');
496                 assert.equal(Metric.all().length, 1);
497                 const metric = Metric.all()[0];
498                 assert.equal(platform.lastModified(metric), primaryCluster['lastModified']);
499             });
500         });
501     });
502
503     async function reportAfterAddingBuilderAndAggregatorsWithResponse(report)
504     {
505         await addBuilderForReport(report);
506         const db = TestServer.database();
507         await Promise.all([
508             db.insert('aggregators', {name: 'Arithmetic'}),
509             db.insert('aggregators', {name: 'Geometric'}),
510         ]);
511         return await TestServer.remoteAPI().postJSON('/api/report/', [report]);
512     }
513
514     const reportWithBuildRequest = {
515         "buildNumber": "123",
516         "buildTime": "2013-02-28T10:12:03.388304",
517         "builderName": "someBuilder",
518         "builderPassword": "somePassword",
519         "platform": "Mountain Lion",
520         "buildRequest": "700",
521         "tests": {
522             "test": {
523                 "metrics": {"FrameRate": { "current": [[[0, 4], [100, 5], [205, 3]]] }}
524             },
525         },
526         "revisions": {
527             "macOS": {
528                 "revision": "10.8.2 12C60"
529             },
530             "WebKit": {
531                 "revision": "141977",
532                 "timestamp": "2013-02-06T08:55:20.9Z"
533             }
534         }
535     };
536
537     it("should allow to report a build request", async () => {
538         await MockData.addMockData(TestServer.database());
539         let response = await reportAfterAddingBuilderAndAggregatorsWithResponse(reportWithBuildRequest);
540         assert.equal(response['status'], 'OK');
541         response = await TestServer.remoteAPI().getJSONWithStatus('/api/measurement-set/?analysisTask=500');
542         assert.equal(response['status'], 'OK');
543         assert.deepEqual(response['measurements'], [[1, 4, 3, 12, 50, [
544                 ['1', '9', '10.8.2 12C60', null, 0], ['2', '11', '141977', null, 1360140920900]],
545             1, 1362046323388, '123', 1, 1, 'current']]);
546     });
547
548 });