Escape builder names in url* and pathFor* methods of BuildbotSyncer
[WebKit-https.git] / Websites / perf.webkit.org / server-tests / api-measurement-set-tests.js
1 'use strict';
2
3 const assert = require('assert');
4 const crypto = require('crypto');
5
6 const TestServer = require('./resources/test-server.js');
7 const addBuilderForReport = require('./resources/common-operations.js').addBuilderForReport;
8 const connectToDatabaseInEveryTest = require('./resources/common-operations.js').connectToDatabaseInEveryTest;
9
10 describe("/api/measurement-set", function () {
11     this.timeout(1000);
12     TestServer.inject();
13     connectToDatabaseInEveryTest();
14
15     function queryPlatformAndMetric(platformName, metricName)
16     {
17         const db = TestServer.database();
18         return Promise.all([
19             db.selectFirstRow('platforms', {name: 'Mountain Lion'}),
20             db.selectFirstRow('test_metrics', {name: 'Time'}),
21         ]).then(function (result) {
22             return {platformId: result[0]['id'], metricId: result[1]['id']};
23         });
24     }
25
26     function format(formatMap, row)
27     {
28         var result = {};
29         for (var i = 0; i < formatMap.length; i++) {
30             var key = formatMap[i];
31             if (key == 'id' || key == 'build' || key == 'builder')
32                 continue;
33             result[key] = row[i];
34         }
35         return result;
36     }
37
38     let clusterStart = TestServer.testConfig().clusterStart;
39     clusterStart = +Date.UTC(clusterStart[0], clusterStart[1] - 1, clusterStart[2], clusterStart[3], clusterStart[4]);
40
41     let clusterSize = TestServer.testConfig().clusterSize;
42     const DAY = 24 * 3600 * 1000;
43     const YEAR = 365.24 * DAY;
44     const MONTH = 30 * DAY;
45     clusterSize = clusterSize[0] * YEAR + clusterSize[1] * MONTH + clusterSize[2] * DAY;
46
47     function clusterTime(index) { return new Date(clusterStart + clusterSize * index); }
48
49     const reportWithBuildTime = [{
50         "buildNumber": "123",
51         "buildTime": clusterTime(7.8).toISOString(),
52         "builderName": "someBuilder",
53         "builderPassword": "somePassword",
54         "platform": "Mountain Lion",
55         "tests": {
56             "Suite": {
57                 "tests": {
58                     "test1": {
59                         "metrics": {"Time": { "current": [1, 2, 3, 4, 5] }}
60                     },
61                 }
62             },
63         }}];
64     reportWithBuildTime.startTime = +clusterTime(7);
65
66     const reportWithRevision = [{
67         "buildNumber": "124",
68         "buildTime": "2013-02-28T15:34:51",
69         "revisions": {
70             "WebKit": {
71                 "revision": "144000",
72                 "timestamp": clusterTime(10.3).toISOString(),
73             },
74         },
75         "builderName": "someBuilder",
76         "builderPassword": "somePassword",
77         "platform": "Mountain Lion",
78         "tests": {
79             "Suite": {
80                 "tests": {
81                     "test1": {
82                         "metrics": {"Time": { "current": [11, 12, 13, 14, 15] }}
83                     }
84                 }
85             },
86         }}];
87
88     const reportWithNewRevision = [{
89         "buildNumber": "125",
90         "buildTime": "2013-02-28T21:45:17",
91         "revisions": {
92             "WebKit": {
93                 "revision": "160609",
94                 "timestamp": clusterTime(12.1).toISOString()
95             },
96         },
97         "builderName": "someBuilder",
98         "builderPassword": "somePassword",
99         "platform": "Mountain Lion",
100         "tests": {
101             "Suite": {
102                 "tests": {
103                     "test1": {
104                         "metrics": {"Time": { "current": [16, 17, 18, 19, 20] }}
105                     }
106                 }
107             },
108         }}];
109
110     const reportWithAncentRevision = [{
111         "buildNumber": "126",
112         "buildTime": "2013-02-28T23:07:25",
113         "revisions": {
114             "WebKit": {
115                 "revision": "137793",
116                 "timestamp": clusterTime(1.8).toISOString()
117             },
118         },
119         "builderName": "someBuilder",
120         "builderPassword": "somePassword",
121         "platform": "Mountain Lion",
122         "tests": {
123             "Suite": {
124                 "tests": {
125                     "test1": {
126                         "metrics": {"Time": { "current": [21, 22, 23, 24, 25] }}
127                     }
128                 }
129             },
130         }}];
131
132     it("should reject when platform ID is missing", function (done) {
133         addBuilderForReport(reportWithBuildTime[0]).then(function () {
134             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
135         }).then(function (response) {
136             assert.equal(response['status'], 'OK');
137             return queryPlatformAndMetric('Mountain Lion', 'Time');
138         }).then(function (result) {
139             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?metric=${result.metricId}`);
140         }).then(function (response) {
141             assert.equal(response['status'], 'AmbiguousRequest');
142             done();
143         }).catch(done);
144     });
145
146     it("should reject when metric ID is missing", function (done) {
147         addBuilderForReport(reportWithBuildTime[0]).then(function () {
148             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
149         }).then(function (response) {
150             assert.equal(response['status'], 'OK');
151             return queryPlatformAndMetric('Mountain Lion', 'Time');
152         }).then(function (result) {
153             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}`);
154         }).then(function (response) {
155             assert.equal(response['status'], 'AmbiguousRequest');
156             done();
157         }).catch(done);
158     });
159
160     it("should reject an invalid platform name", function (done) {
161         addBuilderForReport(reportWithBuildTime[0]).then(function () {
162             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
163         }).then(function (response) {
164             assert.equal(response['status'], 'OK');
165             return queryPlatformAndMetric('Mountain Lion', 'Time');
166         }).then(function (result) {
167             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}a&metric=${result.metricId}`);
168         }).then(function (response) {
169             assert.equal(response['status'], 'InvalidPlatform');
170             done();
171         }).catch(done);
172     });
173
174     it("should reject an invalid metric name", function (done) {
175         addBuilderForReport(reportWithBuildTime[0]).then(function () {
176             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
177         }).then(function (response) {
178             assert.equal(response['status'], 'OK');
179             return queryPlatformAndMetric('Mountain Lion', 'Time');
180         }).then(function (result) {
181             return TestServer.remoteAPI().getJSON(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}b`);
182         }).then(function (response) {
183             assert.equal(response['status'], 'InvalidMetric');
184             done();
185         }).catch(done);
186     });
187
188     it("should be able to retrieve a reported value", function (done) {
189         addBuilderForReport(reportWithBuildTime[0]).then(function () {
190             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
191         }).then(function (response) {
192             assert.equal(response['status'], 'OK');
193             return queryPlatformAndMetric('Mountain Lion', 'Time');
194         }).then(function (result) {
195             return TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
196         }).then(function (response) {
197             const buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
198
199             assert.deepEqual(Object.keys(response).sort(),
200                 ['clusterCount', 'clusterSize', 'clusterStart',
201                   'configurations', 'elapsedTime', 'endTime', 'formatMap', 'lastModified', 'startTime', 'status']);
202             assert.equal(response['status'], 'OK');
203             assert.equal(response['clusterCount'], 1);
204             assert.deepEqual(response['formatMap'], [
205                 'id', 'mean', 'iterationCount', 'sum', 'squareSum', 'markedOutlier',
206                 'revisions', 'commitTime', 'build', 'buildTime', 'buildNumber', 'builder']);
207
208             assert.equal(response['startTime'], reportWithBuildTime.startTime);
209             assert(typeof(response['lastModified']) == 'number', 'lastModified time should be a numeric');
210
211             assert.deepEqual(Object.keys(response['configurations']), ['current']);
212
213             var currentRows = response['configurations']['current'];
214             assert.equal(currentRows.length, 1);
215             assert.equal(currentRows[0].length, response['formatMap'].length);
216             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
217                 mean: 3,
218                 iterationCount: 5,
219                 sum: 15,
220                 squareSum: 55,
221                 markedOutlier: false,
222                 revisions: [],
223                 commitTime: buildTime,
224                 buildTime: buildTime,
225                 buildNumber: '123'});
226             done();
227         }).catch(done);
228     });
229
230     it("should return return the right IDs for measurement, build, and builder", function (done) {
231         addBuilderForReport(reportWithBuildTime[0]).then(function () {
232             return TestServer.remoteAPI().postJSON('/api/report/', reportWithBuildTime);
233         }).then(function (response) {
234             assert.equal(response['status'], 'OK');
235             return queryPlatformAndMetric('Mountain Lion', 'Time');
236         }).then(function (result) {
237             const db = TestServer.database();
238             return Promise.all([
239                 db.selectAll('test_runs'),
240                 db.selectAll('builds'),
241                 db.selectAll('builders'),
242                 TestServer.remoteAPI().getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`),
243             ]);
244         }).then(function (result) {
245             const runs = result[0];
246             const builds = result[1];
247             const builders = result[2];
248             const response = result[3];
249
250             assert.equal(runs.length, 1);
251             assert.equal(builds.length, 1);
252             assert.equal(builders.length, 1);
253             const measurementId = runs[0]['id'];
254             const buildId = builds[0]['id'];
255             const builderId = builders[0]['id'];
256
257             assert.equal(response['configurations']['current'].length, 1);
258             const measurement = response['configurations']['current'][0];
259             assert.equal(response['status'], 'OK');
260
261             assert.equal(measurement[response['formatMap'].indexOf('id')], measurementId);
262             assert.equal(measurement[response['formatMap'].indexOf('build')], buildId);
263             assert.equal(measurement[response['formatMap'].indexOf('builder')], builderId);
264
265             done();
266         }).catch(done);
267     });
268
269     function postReports(reports, callback)
270     {
271         if (!reports.length)
272             return callback();
273
274         postJSON('/api/report/', reports[0], function (response) {
275             assert.equal(response.statusCode, 200);
276             assert.equal(JSON.parse(response.responseText)['status'], 'OK');
277
278             postReports(reports.slice(1), callback);
279         });
280     }
281
282     function queryPlatformAndMetricWithRepository(platformName, metricName, repositoryName)
283     {
284         const db = TestServer.database();
285         return Promise.all([
286             db.selectFirstRow('platforms', {name: platformName}),
287             db.selectFirstRow('test_metrics', {name: metricName}),
288             db.selectFirstRow('repositories', {name: repositoryName}),
289         ]).then(function (result) {
290             return {platformId: result[0]['id'], metricId: result[1]['id'], repositoryId: result[2]['id']};
291         });
292     }
293
294     it("should order results by commit time", function (done) {
295         const remote = TestServer.remoteAPI();
296         let repositoryId;
297         addBuilderForReport(reportWithBuildTime[0]).then(function () {
298             return remote.postJSON('/api/report/', reportWithBuildTime);
299         }).then(function () {
300             return remote.postJSON('/api/report/', reportWithRevision);
301         }).then(function () {
302             return queryPlatformAndMetricWithRepository('Mountain Lion', 'Time', 'WebKit');
303         }).then(function (result) {
304             repositoryId = result.repositoryId;
305             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
306         }).then(function (response) {
307             const currentRows = response['configurations']['current'];
308             const buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
309             const revisionTime = +(new Date(reportWithRevision[0]['revisions']['WebKit']['timestamp']));
310             const revisionBuildTime = +(new Date(reportWithRevision[0]['buildTime']));
311
312             assert.equal(currentRows.length, 2);
313             assert.deepEqual(format(response['formatMap'], currentRows[0]), {
314                mean: 13,
315                iterationCount: 5,
316                sum: 65,
317                squareSum: 855,
318                markedOutlier: false,
319                revisions: [[1, repositoryId, '144000', revisionTime]],
320                commitTime: revisionTime,
321                buildTime: revisionBuildTime,
322                buildNumber: '124' });
323             assert.deepEqual(format(response['formatMap'], currentRows[1]), {
324                 mean: 3,
325                 iterationCount: 5,
326                 sum: 15,
327                 squareSum: 55,
328                 markedOutlier: false,
329                 revisions: [],
330                 commitTime: buildTime,
331                 buildTime: buildTime,
332                 buildNumber: '123' });
333             done();
334         }).catch(done);
335     });
336
337     function buildNumbers(parsedResult, config)
338     {
339         return parsedResult['configurations'][config].map(function (row) {
340             return format(parsedResult['formatMap'], row)['buildNumber'];
341         });
342     }
343
344     it("should include one data point after the current time range", function (done) {
345         const remote = TestServer.remoteAPI();
346         addBuilderForReport(reportWithBuildTime[0]).then(function () {
347             return remote.postJSON('/api/report/', reportWithAncentRevision);
348         }).then(function () {
349             return remote.postJSON('/api/report/', reportWithNewRevision);
350         }).then(function () {
351             return queryPlatformAndMetric('Mountain Lion', 'Time');
352         }).then(function (result) {
353             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
354         }).then(function (response) {
355             assert.equal(response['status'], 'OK');
356             assert.equal(response['clusterCount'], 2, 'should have two clusters');
357             assert.deepEqual(buildNumbers(response, 'current'),
358                 [reportWithAncentRevision[0]['buildNumber'], reportWithNewRevision[0]['buildNumber']]);
359             done();
360         }).catch(done);
361     });
362
363     it("should always include one old data point before the current time range", function (done) {
364         const remote = TestServer.remoteAPI();
365         addBuilderForReport(reportWithBuildTime[0]).then(function () {
366             return remote.postJSON('/api/report/', reportWithBuildTime);
367         }).then(function () {
368             return remote.postJSON('/api/report/', reportWithAncentRevision);
369         }).then(function () {
370             return queryPlatformAndMetric('Mountain Lion', 'Time');
371         }).then(function (result) {
372             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
373         }).then(function (response) {
374             assert.equal(response['clusterCount'], 2, 'should have two clusters');
375             let currentRows = response['configurations']['current'];
376             assert.equal(currentRows.length, 2, 'should contain two data points');
377             assert.deepEqual(buildNumbers(response, 'current'), [reportWithAncentRevision[0]['buildNumber'], reportWithBuildTime[0]['buildNumber']]);
378             done();
379         }).catch(done);
380     });
381
382
383     it("should create cache results", function (done) {
384         const remote = TestServer.remoteAPI();
385         let cachePrefix;
386         addBuilderForReport(reportWithBuildTime[0]).then(function () {
387             return remote.postJSON('/api/report/', reportWithAncentRevision);
388         }).then(function () {
389             return remote.postJSON('/api/report/', reportWithRevision);
390         }).then(function () {
391             return remote.postJSON('/api/report/', reportWithNewRevision);
392         }).then(function () {
393             return queryPlatformAndMetric('Mountain Lion', 'Time');
394         }).then(function (result) {
395             cachePrefix = '/data/measurement-set-' + result.platformId + '-' + result.metricId;
396             return remote.getJSONWithStatus(`/api/measurement-set/?platform=${result.platformId}&metric=${result.metricId}`);
397         }).then(function (newResult) {
398             return remote.getJSONWithStatus(`${cachePrefix}.json`).then(function (cachedResult) {
399                 assert.deepEqual(newResult, cachedResult);
400                 return remote.getJSONWithStatus(`${cachePrefix}-${cachedResult['startTime']}.json`);
401             }).then(function (oldResult) {
402                 var oldBuildNumbers = buildNumbers(oldResult, 'current');
403                 var newBuildNumbers = buildNumbers(newResult, 'current');
404                 assert(oldBuildNumbers.length >= 2, 'The old cluster should contain at least two data points');
405                 assert(newBuildNumbers.length >= 2, 'The new cluster should contain at least two data points');
406                 assert.deepEqual(oldBuildNumbers.slice(oldBuildNumbers.length - 2), newBuildNumbers.slice(0, 2),
407                     'Two conseqcutive clusters should share two data points');
408                 done();
409             });
410         }).catch(done);
411     });
412
413 });