cdeeaf2e35bfb5fff16640dce9500444d58c9c87
[WebKit.git] / Websites / perf.webkit.org / server-tests / api-measurement-set-tests.js
1 'use strict';
2
3 const assert = require('assert');
4
5 const TestServer = require('./resources/test-server.js');
6 const addBuilderForReport = require('./resources/common-operations.js').addBuilderForReport;
7 const connectToDatabaseInEveryTest = require('./resources/common-operations.js').connectToDatabaseInEveryTest;
8
9 describe("/api/measurement-set", function () {
10     this.timeout(1000);
11     TestServer.inject();
12     connectToDatabaseInEveryTest();
13
14     function queryPlatformAndMetric(platformName, metricName)
15     {
16         const db = TestServer.database();
17         return Promise.all([
18             db.selectFirstRow('platforms', {name: 'Mountain Lion'}),
19             db.selectFirstRow('test_metrics', {name: 'Time'}),
20         ]).then(function (result) {
21             return {platformId: result[0]['id'], metricId: result[1]['id']};
22         });
23     }
24
25     function format(formatMap, row)
26     {
27         var result = {};
28         for (var i = 0; i < formatMap.length; i++) {
29             var key = formatMap[i];
30             if (key == 'id' || key == 'build' || key == 'builder')
31                 continue;
32             result[key] = row[i];
33         }
34         return result;
35     }
36
37     let clusterStart = TestServer.testConfig().clusterStart;
38     clusterStart = +Date.UTC(clusterStart[0], clusterStart[1] - 1, clusterStart[2], clusterStart[3], clusterStart[4]);
39
40     let clusterSize = TestServer.testConfig().clusterSize;
41     const DAY = 24 * 3600 * 1000;
42     const YEAR = 365.24 * DAY;
43     const MONTH = 30 * DAY;
44     clusterSize = clusterSize[0] * YEAR + clusterSize[1] * MONTH + clusterSize[2] * DAY;
45
46     function clusterTime(index) { return new Date(clusterStart + clusterSize * index); }
47
48     const reportWithBuildTime = [{
49         "buildNumber": "123",
50         "buildTime": clusterTime(7.8).toISOString(),
51         "builderName": "someBuilder",
52         "builderPassword": "somePassword",
53         "platform": "Mountain Lion",
54         "tests": {
55             "Suite": {
56                 "tests": {
57                     "test1": {
58                         "metrics": {"Time": { "current": [1, 2, 3, 4, 5] }}
59                     },
60                 }
61             },
62         }}];
63     reportWithBuildTime.startTime = +clusterTime(7);
64
65     const reportWithRevision = [{
66         "buildNumber": "124",
67         "buildTime": "2013-02-28T15:34:51",
68         "revisions": {
69             "WebKit": {
70                 "revision": "144000",
71                 "timestamp": clusterTime(10.3).toISOString(),
72             },
73         },
74         "builderName": "someBuilder",
75         "builderPassword": "somePassword",
76         "platform": "Mountain Lion",
77         "tests": {
78             "Suite": {
79                 "tests": {
80                     "test1": {
81                         "metrics": {"Time": { "current": [11, 12, 13, 14, 15] }}
82                     }
83                 }
84             },
85         }}];
86
87     const reportWithNewRevision = [{
88         "buildNumber": "125",
89         "buildTime": "2013-02-28T21:45:17",
90         "revisions": {
91             "WebKit": {
92                 "revision": "160609",
93                 "timestamp": clusterTime(12.1).toISOString()
94             },
95         },
96         "builderName": "someBuilder",
97         "builderPassword": "somePassword",
98         "platform": "Mountain Lion",
99         "tests": {
100             "Suite": {
101                 "tests": {
102                     "test1": {
103                         "metrics": {"Time": { "current": [16, 17, 18, 19, 20] }}
104                     }
105                 }
106             },
107         }}];
108
109     const reportWithAncentRevision = [{
110         "buildNumber": "126",
111         "buildTime": "2013-02-28T23:07:25",
112         "revisions": {
113             "WebKit": {
114                 "revision": "137793",
115                 "timestamp": clusterTime(1.8).toISOString()
116             },
117         },
118         "builderName": "someBuilder",
119         "builderPassword": "somePassword",
120         "platform": "Mountain Lion",
121         "tests": {
122             "Suite": {
123                 "tests": {
124                     "test1": {
125                         "metrics": {"Time": { "current": [21, 22, 23, 24, 25] }}
126                     }
127                 }
128             },
129         }}];
130
131     it("should reject when platform ID is missing", function (done) {
132         addBuilderForReport(reportWithBuildTime[0]).then(function () {
133             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
134         }).then(function (response) {
135             assert.equal(response['status'], 'OK');
136             return queryPlatformAndMetric('Mountain Lion', 'Time');
137         }).then(function (result) {
138             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?metric=${result.metricId}`);
139         }).then(function (response) {
140             assert.equal(response['status'], 'AmbiguousRequest');
141             done();
142         }).catch(done);
143     });
144
145     it("should reject when metric ID is missing", function (done) {
146         addBuilderForReport(reportWithBuildTime[0]).then(function () {
147             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
148         }).then(function (response) {
149             assert.equal(response['status'], 'OK');
150             return queryPlatformAndMetric('Mountain Lion', 'Time');
151         }).then(function (result) {
152             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}`);
153         }).then(function (response) {
154             assert.equal(response['status'], 'AmbiguousRequest');
155             done();
156         }).catch(done);
157     });
158
159     it("should reject an invalid platform name", function (done) {
160         addBuilderForReport(reportWithBuildTime[0]).then(function () {
161             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
162         }).then(function (response) {
163             assert.equal(response['status'], 'OK');
164             return queryPlatformAndMetric('Mountain Lion', 'Time');
165         }).then(function (result) {
166             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}a&metric=${result.metricId}`);
167         }).then(function (response) {
168             assert.equal(response['status'], 'InvalidPlatform');
169             done();
170         }).catch(done);
171     });
172
173     it("should reject an invalid metric name", function (done) {
174         addBuilderForReport(reportWithBuildTime[0]).then(function () {
175             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
176         }).then(function (response) {
177             assert.equal(response['status'], 'OK');
178             return queryPlatformAndMetric('Mountain Lion', 'Time');
179         }).then(function (result) {
180             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}b`);
181         }).then(function (response) {
182             assert.equal(response['status'], 'InvalidMetric');
183             done();
184         }).catch(done);
185     });
186
187     it("should be able to retrieve a reported value", function (done) {
188         addBuilderForReport(reportWithBuildTime[0]).then(function () {
189             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
190         }).then(function (response) {
191             assert.equal(response['status'], 'OK');
192             return queryPlatformAndMetric('Mountain Lion', 'Time');
193         }).then(function (result) {
194             return TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
195         }).then(function (response) {
196             const buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
197
198             assert.deepEqual(Object.keys(response).sort(),
199                 ['clusterCount', 'clusterSize', 'clusterStart',
200                   'configurations', 'elapsedTime', 'endTime', 'formatMap', 'lastModified', 'startTime', 'status']);
201             assert.equal(response['status'], 'OK');
202             assert.equal(response['clusterCount'], 1);
203             assert.deepEqual(response['formatMap'], [
204                 'id', 'mean', 'iterationCount', 'sum', 'squareSum', 'markedOutlier',
205                 'revisions', 'commitTime', 'build', 'buildTime', 'buildNumber', 'builder']);
206
207             assert.equal(response['startTime'], reportWithBuildTime.startTime);
208             assert(typeof(response['lastModified']) == 'number', 'lastModified time should be a numeric');
209
210             assert.deepEqual(Object.keys(response['configurations']), ['current']);
211
212             var currentRows = response['configurations']['current'];
213             assert.equal(currentRows.length, 1);
214             assert.equal(currentRows[0].length, response['formatMap'].length);
215             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
216                 mean: 3,
217                 iterationCount: 5,
218                 sum: 15,
219                 squareSum: 55,
220                 markedOutlier: false,
221                 revisions: [],
222                 commitTime: buildTime,
223                 buildTime: buildTime,
224                 buildNumber: '123'});
225             done();
226         }).catch(done);
227     });
228
229     it("should return return the right IDs for measurement, build, and builder", function (done) {
230         addBuilderForReport(reportWithBuildTime[0]).then(function () {
231             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
232         }).then(function (response) {
233             assert.equal(response['status'], 'OK');
234             return queryPlatformAndMetric('Mountain Lion', 'Time');
235         }).then(function (result) {
236             const db = TestServer.database();
237             return Promise.all([
238                 db.selectAll('test_runs'),
239                 db.selectAll('builds'),
240                 db.selectAll('builders'),
241                 TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`),
242             ]);
243         }).then(function (result) {
244             const runs = result[0];
245             const builds = result[1];
246             const builders = result[2];
247             const response = result[3];
248
249             assert.equal(runs.length, 1);
250             assert.equal(builds.length, 1);
251             assert.equal(builders.length, 1);
252             const measurementId = runs[0]['id'];
253             const buildId = builds[0]['id'];
254             const builderId = builders[0]['id'];
255
256             assert.equal(response['configurations']['current'].length, 1);
257             const measurement = response['configurations']['current'][0];
258             assert.equal(response['status'], 'OK');
259
260             assert.equal(measurement[response['formatMap'].indexOf('id')], measurementId);
261             assert.equal(measurement[response['formatMap'].indexOf('build')], buildId);
262             assert.equal(measurement[response['formatMap'].indexOf('builder')], builderId);
263
264             done();
265         }).catch(done);
266     });
267
268     function postReports(reports, callback)
269     {
270         if (!reports.length)
271             return callback();
272
273         postJSON('/api/report/', reports[0], function (response) {
274             assert.equal(response.statusCode, 200);
275             assert.equal(JSON.parse(response.responseText)['status'], 'OK');
276
277             postReports(reports.slice(1), callback);
278         });
279     }
280
281     function queryPlatformAndMetricWithRepository(platformName, metricName, repositoryName)
282     {
283         const db = TestServer.database();
284         return Promise.all([
285             db.selectFirstRow('platforms', {name: platformName}),
286             db.selectFirstRow('test_metrics', {name: metricName}),
287             db.selectFirstRow('repositories', {name: repositoryName}),
288         ]).then(function (result) {
289             return {platformId: result[0]['id'], metricId: result[1]['id'], repositoryId: result[2]['id']};
290         });
291     }
292
293     it("should order results by commit time", function (done) {
294         const remote = TestServer.remoteAPI();
295         let repositoryId;
296         addBuilderForReport(reportWithBuildTime[0]).then(function () {
297             return remote.postJSON('/api/report/', reportWithBuildTime);
298         }).then(function () {
299             return remote.postJSON('/api/report/', reportWithRevision);
300         }).then(function () {
301             return queryPlatformAndMetricWithRepository('Mountain Lion', 'Time', 'WebKit');
302         }).then(function (result) {
303             repositoryId = result.repositoryId;
304             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
305         }).then(function (response) {
306             const currentRows = response['configurations']['current'];
307             const buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
308             const revisionTime = +(new Date(reportWithRevision[0]['revisions']['WebKit']['timestamp']));
309             const revisionBuildTime = +(new Date(reportWithRevision[0]['buildTime']));
310
311             assert.equal(currentRows.length, 2);
312             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
313                mean: 13,
314                iterationCount: 5,
315                sum: 65,
316                squareSum: 855,
317                markedOutlier: false,
318                revisions: [[1, repositoryId, '144000', revisionTime]],
319                commitTime: revisionTime,
320                buildTime: revisionBuildTime,
321                buildNumber: '124' });
322             assert.deepEqual(format(response['formatMap'], currentRows[1]), {
323                 mean: 3,
324                 iterationCount: 5,
325                 sum: 15,
326                 squareSum: 55,
327                 markedOutlier: false,
328                 revisions: [],
329                 commitTime: buildTime,
330                 buildTime: buildTime,
331                 buildNumber: '123' });
332             done();
333         }).catch(done);
334     });
335
336     function buildNumbers(parsedResult, config)
337     {
338         return parsedResult['configurations'][config].map(function (row) {
339             return format(parsedResult['formatMap'], row)['buildNumber'];
340         });
341     }
342
343     it("should include one data point after the current time range", function (done) {
344         const remote = TestServer.remoteAPI();
345         addBuilderForReport(reportWithBuildTime[0]).then(function () {
346             return remote.postJSON('/api/report/', reportWithAncentRevision);
347         }).then(function () {
348             return remote.postJSON('/api/report/', reportWithNewRevision);
349         }).then(function () {
350             return queryPlatformAndMetric('Mountain Lion', 'Time');
351         }).then(function (result) {
352             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
353         }).then(function (response) {
354             assert.equal(response['status'], 'OK');
355             assert.equal(response['clusterCount'], 2, 'should have two clusters');
356             assert.deepEqual(buildNumbers(response, 'current'),
357                 [reportWithAncentRevision[0]['buildNumber'], reportWithNewRevision[0]['buildNumber']]);
358             done();
359         }).catch(done);
360     });
361
362     it("should always include one old data point before the current time range", function (done) {
363         const remote = TestServer.remoteAPI();
364         addBuilderForReport(reportWithBuildTime[0]).then(function () {
365             return remote.postJSON('/api/report/', reportWithBuildTime);
366         }).then(function () {
367             return remote.postJSON('/api/report/', reportWithAncentRevision);
368         }).then(function () {
369             return queryPlatformAndMetric('Mountain Lion', 'Time');
370         }).then(function (result) {
371             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
372         }).then(function (response) {
373             assert.equal(response['clusterCount'], 2, 'should have two clusters');
374             let currentRows = response['configurations']['current'];
375             assert.equal(currentRows.length, 2, 'should contain two data points');
376             assert.deepEqual(buildNumbers(response, 'current'), [reportWithAncentRevision[0]['buildNumber'], reportWithBuildTime[0]['buildNumber']]);
377             done();
378         }).catch(done);
379     });
380
381
382     it("should create cache results", function (done) {
383         const remote = TestServer.remoteAPI();
384         let cachePrefix;
385         addBuilderForReport(reportWithBuildTime[0]).then(function () {
386             return remote.postJSON('/api/report/', reportWithAncentRevision);
387         }).then(function () {
388             return remote.postJSON('/api/report/', reportWithRevision);
389         }).then(function () {
390             return remote.postJSON('/api/report/', reportWithNewRevision);
391         }).then(function () {
392             return queryPlatformAndMetric('Mountain Lion', 'Time');
393         }).then(function (result) {
394             cachePrefix = '/data/measurement-set-' + result.platformId + '-' + result.metricId;
395             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
396         }).then(function (newResult) {
397             return remote.getJSONWithStatus(`${cachePrefix}.json`).then(function (cachedResult) {
398                 assert.deepEqual(newResult, cachedResult);
399                 return remote.getJSONWithStatus(`${cachePrefix}-${cachedResult['startTime']}.json`);
400             }).then(function (oldResult) {
401                 var oldBuildNumbers = buildNumbers(oldResult, 'current');
402                 var newBuildNumbers = buildNumbers(newResult, 'current');
403                 assert(oldBuildNumbers.length >= 2, 'The old cluster should contain at least two data points');
404                 assert(newBuildNumbers.length >= 2, 'The new cluster should contain at least two data points');
405                 assert.deepEqual(oldBuildNumbers.slice(oldBuildNumbers.length - 2), newBuildNumbers.slice(0, 2),
406                     'Two conseqcutive clusters should share two data points');
407                 done();
408             });
409         }).catch(done);
410     });
411
412 });