Add /api/measurement-set for v3 UI
[WebKit-https.git] / Websites / perf.webkit.org / tests / api-measurement-set.js
1 describe("/api/measurement-set", function () {
2     function addBuilder(report, callback) {
3         queryAndFetchAll('INSERT INTO builders (builder_name, builder_password_hash) values ($1, $2)',
4             [report[0].builderName, sha256(report[0].builderPassword)], callback);
5     }
6
7     function queryPlatformAndMetric(platformName, metricName, callback) {
8         queryAndFetchAll('SELECT * FROM platforms WHERE platform_name = $1', [platformName], function (platformRows) {
9             queryAndFetchAll('SELECT * FROM test_metrics WHERE metric_name = $1', [metricName], function (metricRows) {
10                 callback(platformRows[0]['platform_id'], metricRows[0]['metric_id']);
11             });
12         });
13     }
14
15     function format(formatMap, row) {
16         var result = {};
17         for (var i = 0; i < formatMap.length; i++) {
18             var key = formatMap[i];
19             if (key == 'id' || key == 'build' || key == 'builder')
20                 continue;
21             result[key] = row[i];
22         }
23         return result;
24     }
25
26     var clusterStart = config('clusterStart');
27     clusterStart = +Date.UTC(clusterStart[0], clusterStart[1] - 1, clusterStart[2], clusterStart[3], clusterStart[4]);
28
29     var clusterSize = config('clusterSize');
30     var DAY = 24 * 3600 * 1000;
31     var YEAR = 365.24 * DAY;
32     var MONTH = 30 * DAY;
33     clusterSize = clusterSize[0] * YEAR + clusterSize[1] * MONTH + clusterSize[2] * DAY;
34
35     function clusterTime(index) { return new Date(clusterStart + clusterSize * index); }
36
37     var reportWithBuildTime = [{
38         "buildNumber": "123",
39         "buildTime": clusterTime(7.8).toISOString(),
40         "builderName": "someBuilder",
41         "builderPassword": "somePassword",
42         "platform": "Mountain Lion",
43         "tests": {
44             "Suite": {
45                 "tests": {
46                     "test1": {
47                         "metrics": {"Time": { "current": [1, 2, 3, 4, 5] }}
48                     },
49                 }
50             },
51         }}];
52     reportWithBuildTime.startTime = +clusterTime(7);
53
54     var reportWithRevision = [{
55         "buildNumber": "124",
56         "buildTime": "2013-02-28T15:34:51",
57         "revisions": {
58             "WebKit": {
59                 "revision": "144000",
60                 "timestamp": clusterTime(10.3).toISOString(),
61             },
62         },
63         "builderName": "someBuilder",
64         "builderPassword": "somePassword",
65         "platform": "Mountain Lion",
66         "tests": {
67             "Suite": {
68                 "tests": {
69                     "test1": {
70                         "metrics": {"Time": { "current": [11, 12, 13, 14, 15] }}
71                     }
72                 }
73             },
74         }}];
75
76     var reportWithNewRevision = [{
77         "buildNumber": "125",
78         "buildTime": "2013-02-28T21:45:17",
79         "revisions": {
80             "WebKit": {
81                 "revision": "160609",
82                 "timestamp": clusterTime(12.1).toISOString()
83             },
84         },
85         "builderName": "someBuilder",
86         "builderPassword": "somePassword",
87         "platform": "Mountain Lion",
88         "tests": {
89             "Suite": {
90                 "tests": {
91                     "test1": {
92                         "metrics": {"Time": { "current": [16, 17, 18, 19, 20] }}
93                     }
94                 }
95             },
96         }}];
97
98     var reportWithAncentRevision = [{
99         "buildNumber": "126",
100         "buildTime": "2013-02-28T23:07:25",
101         "revisions": {
102             "WebKit": {
103                 "revision": "137793",
104                 "timestamp": clusterTime(1.8).toISOString()
105             },
106         },
107         "builderName": "someBuilder",
108         "builderPassword": "somePassword",
109         "platform": "Mountain Lion",
110         "tests": {
111             "Suite": {
112                 "tests": {
113                     "test1": {
114                         "metrics": {"Time": { "current": [21, 22, 23, 24, 25] }}
115                     }
116                 }
117             },
118         }}];
119
120     it("should reject when platform ID is missing", function () {
121         addBuilder(reportWithBuildTime, function () {
122             postJSON('/api/report/', reportWithBuildTime, function (response) {
123                 assert.equal(response.statusCode, 200);
124                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
125                 queryPlatformAndMetric('Mountain Lion', 'Time', function (platformId, metricId) {
126                     httpGet('/api/measurement-set/?metric=' + metricId, function (response) {
127                         assert.notEqual(JSON.parse(response.responseText)['status'], 'InvalidMetric');
128                         notifyDone();
129                     });
130                 });
131
132             });
133         });
134     });
135
136     it("should reject when metric ID is missing", function () {
137         addBuilder(reportWithBuildTime, function () {
138             postJSON('/api/report/', reportWithBuildTime, function (response) {
139                 assert.equal(response.statusCode, 200);
140                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
141                 queryPlatformAndMetric('Mountain Lion', 'Time', function (platformId, metricId) {
142                     httpGet('/api/measurement-set/?platform=' + platformId, function (response) {
143                         assert.notEqual(JSON.parse(response.responseText)['status'], 'InvalidPlatform');
144                         notifyDone();
145                     });
146                 });
147
148             });
149         });
150     });
151
152     it("should reject an invalid platform name", function () {
153         addBuilder(reportWithBuildTime, function () {
154             postJSON('/api/report/', reportWithBuildTime, function (response) {
155                 assert.equal(response.statusCode, 200);
156                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
157                 queryPlatformAndMetric('Mountain Lion', 'Time', function (platformId, metricId) {
158                     httpGet('/api/measurement-set/?platform=' + platformId + 'a&metric=' + metricId, function (response) {
159                         assert.equal(JSON.parse(response.responseText)['status'], 'InvalidPlatform');
160                         notifyDone();
161                     });
162                 });
163
164             });
165         });
166     });
167
168     it("should reject an invalid metric name", function () {
169         addBuilder(reportWithBuildTime, function () {
170             postJSON('/api/report/', reportWithBuildTime, function (response) {
171                 assert.equal(response.statusCode, 200);
172                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
173                 queryPlatformAndMetric('Mountain Lion', 'Time', function (platformId, metricId) {
174                     httpGet('/api/measurement-set/?platform=' + platformId + '&metric=' + metricId + 'b', function (response) {
175                         assert.equal(JSON.parse(response.responseText)['status'], 'InvalidMetric');
176                         notifyDone();
177                     });
178                 });
179
180             });
181         });
182     });
183
184     it("should be able to retrieve a reported value", function () {
185         addBuilder(reportWithBuildTime, function () {
186             postJSON('/api/report/', reportWithBuildTime, function (response) {
187                 assert.equal(response.statusCode, 200);
188                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
189                 queryPlatformAndMetric('Mountain Lion', 'Time', function (platformId, metricId) {
190                     httpGet('/api/measurement-set/?platform=' + platformId + '&metric=' + metricId, function (response) {
191                         try {
192                             var paresdResult = JSON.parse(response.responseText);
193                         } catch (error) {
194                             assert.fail(error, null, response.responseText);
195                         }
196
197                         var buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
198
199                         assert.deepEqual(Object.keys(paresdResult).sort(),
200                             ['clusterCount', 'clusterSize', 'clusterStart',
201                               'configurations', 'elapsedTime', 'endTime', 'formatMap', 'lastModified', 'startTime', 'status']);
202                         assert.equal(paresdResult['status'], 'OK');
203                         assert.equal(paresdResult['clusterCount'], 1);
204                         assert.deepEqual(paresdResult['formatMap'], [
205                             'id', 'mean', 'iterationCount', 'sum', 'squareSum', 'markedOutlier',
206                             'revisions', 'commitTime', 'build', 'buildTime', 'buildNumber', 'builder']);
207
208                         assert.equal(paresdResult['startTime'], reportWithBuildTime.startTime);
209
210                         assert.deepEqual(Object.keys(paresdResult['configurations']), ['current']);
211
212                         var currentRows = paresdResult['configurations']['current'];
213                         assert.equal(currentRows.length, 1);
214                         assert.equal(currentRows[0].length, paresdResult['formatMap'].length);
215                         assert.deepEqual(format(paresdResult['formatMap'], currentRows[0]), {
216                             mean: 3,
217                             iterationCount: 5,
218                             sum: 15,
219                             squareSum: 55,
220                             markedOutlier: false,
221                             revisions: [],
222                             commitTime: buildTime,
223                             buildTime: buildTime,
224                             buildNumber: '123'});
225                         notifyDone();
226                     });
227                 });
228
229             });
230         });
231     });
232
233     it("should return return the right IDs for measurement, build, and builder", function () {
234         addBuilder(reportWithBuildTime, function () {
235             postJSON('/api/report/', reportWithBuildTime, function (response) {
236                 assert.equal(response.statusCode, 200);
237                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
238                 queryPlatformAndMetric('Mountain Lion', 'Time', function (platformId, metricId) {
239                     queryAndFetchAll('SELECT * FROM test_runs', [], function (runs) {
240                         assert.equal(runs.length, 1);
241                         var measurementId = runs[0]['run_id'];
242                         queryAndFetchAll('SELECT * FROM builds', [], function (builds) {
243                             assert.equal(builds.length, 1);
244                             var buildId = builds[0]['build_id'];
245                             queryAndFetchAll('SELECT * FROM builders', [], function (builders) {
246                                 assert.equal(builders.length, 1);
247                                 var builderId = builders[0]['builder_id'];
248                                 httpGet('/api/measurement-set/?platform=' + platformId + '&metric=' + metricId, function (response) {
249                                     var paresdResult = JSON.parse(response.responseText);
250                                     assert.equal(paresdResult['configurations']['current'].length, 1);
251                                     var measurement = paresdResult['configurations']['current'][0];
252                                     assert.equal(paresdResult['status'], 'OK');
253                                     assert.equal(measurement[paresdResult['formatMap'].indexOf('id')], measurementId);
254                                     assert.equal(measurement[paresdResult['formatMap'].indexOf('build')], buildId);
255                                     assert.equal(measurement[paresdResult['formatMap'].indexOf('builder')], builderId);
256                                     notifyDone();
257                                 });
258                             });
259                         });
260                     });
261                 });
262             });
263         });
264     });
265
266     function postReports(reports, callback) {
267         if (!reports.length)
268             return callback();
269
270         postJSON('/api/report/', reports[0], function (response) {
271             assert.equal(response.statusCode, 200);
272             assert.equal(JSON.parse(response.responseText)['status'], 'OK');
273
274             postReports(reports.slice(1), callback);
275         });
276     }
277
278     function queryPlatformAndMetricWithRepository(platformName, metricName, repositoryName, callback) {
279         queryPlatformAndMetric(platformName, metricName, function (platformId, metricId) {
280             queryAndFetchAll('SELECT * FROM repositories WHERE repository_name = $1', [repositoryName], function (rows) {
281                 callback(platformId, metricId, rows[0]['repository_id']);
282             });
283         });
284     }
285
286     it("should order results by commit time", function () {
287         addBuilder(reportWithBuildTime, function () {
288             postReports([reportWithBuildTime, reportWithRevision], function () {
289                 queryPlatformAndMetricWithRepository('Mountain Lion', 'Time', 'WebKit', function (platformId, metricId, repositoryId) {
290                     httpGet('/api/measurement-set/?platform=' + platformId + '&metric=' + metricId, function (response) {
291                         var parsedResult = JSON.parse(response.responseText);
292                         assert.equal(parsedResult['status'], 'OK');
293
294                         var buildTime = +(new Date(reportWithBuildTime[0]['buildTime']));
295                         var revisionTime = +(new Date(reportWithRevision[0]['revisions']['WebKit']['timestamp']));
296                         var revisionBuildTime = +(new Date(reportWithRevision[0]['buildTime']));
297
298                         var currentRows = parsedResult['configurations']['current'];
299                         assert.equal(currentRows.length, 2);
300                         assert.deepEqual(format(parsedResult['formatMap'], currentRows[0]), {
301                            mean: 13,
302                            iterationCount: 5,
303                            sum: 65,
304                            squareSum: 855,
305                            markedOutlier: false,
306                            revisions: [[repositoryId, '144000', revisionTime]],
307                            commitTime: revisionTime,
308                            buildTime: revisionBuildTime,
309                            buildNumber: '124' });
310                         assert.deepEqual(format(parsedResult['formatMap'], currentRows[1]), {
311                             mean: 3,
312                             iterationCount: 5,
313                             sum: 15,
314                             squareSum: 55,
315                             markedOutlier: false,
316                             revisions: [],
317                             commitTime: buildTime,
318                             buildTime: buildTime,
319                             buildNumber: '123' });
320                         notifyDone();
321                     });
322                 });                
323             });
324         });
325     });
326
327     function buildNumbers(parsedResult, config) {
328         return parsedResult['configurations'][config].map(function (row) {
329             return format(parsedResult['formatMap'], row)['buildNumber'];
330         });
331     }
332
333     it("should include one data point after the current time range", function () {
334         addBuilder(reportWithBuildTime, function () {
335             postReports([reportWithAncentRevision, reportWithNewRevision], function () {
336                 queryPlatformAndMetricWithRepository('Mountain Lion', 'Time', 'WebKit', function (platformId, metricId, repositoryId) {
337                     httpGet('/api/measurement-set/?platform=' + platformId + '&metric=' + metricId, function (response) {
338                         var parsedResult = JSON.parse(response.responseText);
339                         assert.equal(parsedResult['status'], 'OK');
340                         assert.equal(parsedResult['clusterCount'], 2, 'should have two clusters');
341                         assert.deepEqual(buildNumbers(parsedResult, 'current'),
342                             [reportWithAncentRevision[0]['buildNumber'], reportWithNewRevision[0]['buildNumber']]);
343                         notifyDone();
344                     });
345                 });                
346             });
347         });
348     });
349
350     // FIXME: This test assumes a cluster step of 2-3 months
351     it("should always include one old data point before the current time range", function () {
352         addBuilder(reportWithBuildTime, function () {
353             postReports([reportWithBuildTime, reportWithAncentRevision], function () {
354                 queryPlatformAndMetricWithRepository('Mountain Lion', 'Time', 'WebKit', function (platformId, metricId, repositoryId) {
355                     httpGet('/api/measurement-set/?platform=' + platformId + '&metric=' + metricId, function (response) {
356                         var parsedResult = JSON.parse(response.responseText);
357                         assert.equal(parsedResult['status'], 'OK');
358                         assert.equal(parsedResult['clusterCount'], 2, 'should have two clusters');
359
360                         var currentRows = parsedResult['configurations']['current'];
361                         assert.equal(currentRows.length, 2, 'should contain at least two data points');
362                         assert.deepEqual(buildNumbers(parsedResult, 'current'),
363                             [reportWithAncentRevision[0]['buildNumber'], reportWithBuildTime[0]['buildNumber']]);
364                         notifyDone();
365                     });
366                 });                
367             });
368         });
369     });
370
371     // FIXME: This test assumes a cluster step of 2-3 months
372     it("should create cache results", function () {
373         addBuilder(reportWithBuildTime, function () {
374             postReports([reportWithAncentRevision, reportWithRevision, reportWithNewRevision], function () {
375                 queryPlatformAndMetricWithRepository('Mountain Lion', 'Time', 'WebKit', function (platformId, metricId, repositoryId) {
376                     httpGet('/api/measurement-set/?platform=' + platformId + '&metric=' + metricId, function (response) {
377                         var parsedResult = JSON.parse(response.responseText);
378                         assert.equal(parsedResult['status'], 'OK');
379                         var cachePrefix = '/data/measurement-set-' + platformId + '-' + metricId;
380                         httpGet(cachePrefix + '.json', function (response) {
381                             var parsedCachedResult = JSON.parse(response.responseText);
382                             assert.deepEqual(parsedResult, parsedCachedResult);
383
384                             httpGet(cachePrefix + '-' + parsedResult['startTime'] + '.json', function (response) {
385                                 var parsedOldResult = JSON.parse(response.responseText);
386
387                                 var oldBuildNumbers = buildNumbers(parsedOldResult, 'current');
388                                 var newBuildNumbers = buildNumbers(parsedResult, 'current');
389                                 assert(oldBuildNumbers.length >= 2, 'The old cluster should contain at least two data points');
390                                 assert(newBuildNumbers.length >= 2, 'The new cluster should contain at least two data points');
391                                 assert.deepEqual(oldBuildNumbers.slice(oldBuildNumbers.length - 2), newBuildNumbers.slice(0, 2),
392                                     'Two conseqcutive clusters should share two data points');
393
394                                 notifyDone();
395                             });
396                         });
397                     });
398                 });                
399             });
400         });
401     });
402
403 });