REGRESSION: Searching commits can highlight wrong data points
[WebKit-https.git] / Websites / perf.webkit.org / tests / api-report.js
1 describe("/api/report", function () {
2     var emptyReport = [{
3         "buildNumber": "123",
4         "buildTime": "2013-02-28T10:12:03.388304",
5         "builderName": "someBuilder",
6         "slaveName": "someSlave",
7         "builderPassword": "somePassword",
8         "platform": "Mountain Lion",
9         "tests": {},
10         "revisions": {
11             "OS X": {
12                 "revision": "10.8.2 12C60"
13             },
14             "WebKit": {
15                 "revision": "141977",
16                 "timestamp": "2013-02-06T08:55:20.9Z"
17             }
18         }}];
19
20     var emptySlaveReport = [{
21         "buildNumber": "123",
22         "buildTime": "2013-02-28T10:12:03.388304",
23         "builderName": "someBuilder",
24         "builderPassword": "somePassword",
25         "slaveName": "someSlave",
26         "slavePassword": "otherPassword",
27         "platform": "Mountain Lion",
28         "tests": {},
29         "revisions": {
30             "OS X": {
31                 "revision": "10.8.2 12C60"
32             },
33             "WebKit": {
34                 "revision": "141977",
35                 "timestamp": "2013-02-06T08:55:20.9Z"
36             }
37         }}];
38
39     function addBuilder(report, callback) {
40         queryAndFetchAll('INSERT INTO builders (builder_name, builder_password_hash) values ($1, $2)',
41             [report[0].builderName, sha256(report[0].builderPassword)], callback);
42     }
43
44     function addSlave(report, callback) {
45         queryAndFetchAll('INSERT INTO build_slaves (slave_name, slave_password_hash) values ($1, $2)',
46             [report[0].slaveName, sha256(report[0].slavePassword)], callback);
47     }
48
49     it("should reject error when builder name is missing", function () {
50         postJSON('/api/report/', [{"buildTime": "2013-02-28T10:12:03.388304"}], function (response) {
51             assert.equal(response.statusCode, 200);
52             assert.equal(JSON.parse(response.responseText)['status'], 'MissingBuilderName');
53             notifyDone();
54         });
55     });
56
57     it("should reject error when build time is missing", function () {
58         addBuilder(emptyReport, function () {
59             postJSON('/api/report/', [{"builderName": "someBuilder", "builderPassword": "somePassword"}], function (response) {
60                 assert.equal(response.statusCode, 200);
61                 assert.equal(JSON.parse(response.responseText)['status'], 'MissingBuildTime');
62                 notifyDone();
63             });
64         });
65     });
66
67     it("should reject when there are no builders", function () {
68         postJSON('/api/report/', emptyReport, function (response) {
69             assert.equal(response.statusCode, 200);
70             assert.notEqual(JSON.parse(response.responseText)['status'], 'OK');
71             assert.equal(JSON.parse(response.responseText)['failureStored'], false);
72             assert.equal(JSON.parse(response.responseText)['processedRuns'], 0);
73
74             queryAndFetchAll('SELECT COUNT(*) from reports', [], function (rows) {
75                 assert.equal(rows[0].count, 0);
76                 notifyDone();
77             });
78         });
79     });
80
81     it("should reject a report without a builder password", function () {
82         addBuilder(emptyReport, function () {
83             var report = [{
84                 "buildNumber": "123",
85                 "buildTime": "2013-02-28T10:12:03.388304",
86                 "builderName": "someBuilder",
87                 "tests": {},
88                 "revisions": {}}];
89             postJSON('/api/report/', report, function (response) {
90                 assert.equal(response.statusCode, 200);
91                 assert.notEqual(JSON.parse(response.responseText)['status'], 'OK');
92                 assert.equal(JSON.parse(response.responseText)['failureStored'], false);
93                 assert.equal(JSON.parse(response.responseText)['processedRuns'], 0);
94
95                 queryAndFetchAll('SELECT COUNT(*) from reports', [], function (rows) {
96                     assert.equal(rows[0].count, 0);
97                     notifyDone();
98                 });
99             });
100         });
101     });
102
103     it("should store a report from a valid builder", function () {
104         addBuilder(emptyReport, function () {
105             postJSON('/api/report/', emptyReport, function (response) {
106                 assert.equal(response.statusCode, 200);
107                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
108                 assert.equal(JSON.parse(response.responseText)['failureStored'], false);
109                 assert.equal(JSON.parse(response.responseText)['processedRuns'], 1);
110                 queryAndFetchAll('SELECT COUNT(*) from reports', [], function (rows) {
111                     assert.equal(rows[0].count, 1);
112                     notifyDone();
113                 });
114             });
115         });
116     });
117
118     it("should treat the slave password as the builder password if there is no matching slave", function () {
119         addBuilder(emptyReport, function () {
120             emptyReport[0]['slavePassword'] = emptyReport[0]['builderPassword'];
121             delete emptyReport[0]['builderPassword'];
122             postJSON('/api/report/', emptyReport, function (response) {
123                 emptyReport[0]['builderPassword'] = emptyReport[0]['slavePassword'];
124                 delete emptyReport[0]['slavePassword'];
125
126                 assert.equal(response.statusCode, 200);
127                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
128                 assert.equal(JSON.parse(response.responseText)['failureStored'], false);
129                 assert.equal(JSON.parse(response.responseText)['processedRuns'], 1);
130                 queryAndFetchAll('SELECT COUNT(*) from reports', [], function (rows) {
131                     assert.equal(rows[0].count, 1);
132                     notifyDone();
133                 });
134             });
135         });
136     });
137
138     it("should store a report from a valid slave", function () {
139         addSlave(emptySlaveReport, function () {
140             postJSON('/api/report/', emptySlaveReport, function (response) {
141                 assert.equal(response.statusCode, 200);
142                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
143                 assert.equal(JSON.parse(response.responseText)['failureStored'], false);
144                 assert.equal(JSON.parse(response.responseText)['processedRuns'], 1);
145                 queryAndFetchAll('SELECT COUNT(*) from reports', [], function (rows) {
146                     assert.equal(rows[0].count, 1);
147                     notifyDone();
148                 });
149             });
150         });
151     });
152
153     it("should store the builder name but not the builder password", function () {
154         addBuilder(emptyReport, function () {
155             postJSON('/api/report/', emptyReport, function (response) {
156                 queryAndFetchAll('SELECT report_content from reports', [], function (rows) {
157                     var storedContent = JSON.parse(rows[0].report_content);
158                     assert.equal(storedContent['builderName'], emptyReport[0]['builderName']);
159                     assert(!('builderPassword' in storedContent));
160                     notifyDone();
161                 });
162             });
163         });
164     });
165
166     it("should add a slave if there isn't one and the report was authenticated by a builder", function () {
167         addBuilder(emptyReport, function () {
168             postJSON('/api/report/', emptyReport, function (response) {
169                 queryAndFetchAll('SELECT * from build_slaves', [], function (rows) {
170                     assert.strictEqual(rows[0].slave_name, emptyReport[0].slaveName);
171                     notifyDone();
172                 });
173             });
174         });
175     });
176
177     it("should add a builder if there isn't one and the report was authenticated by a slave", function () {
178         addSlave(emptySlaveReport, function () {
179             postJSON('/api/report/', emptySlaveReport, function (response) {
180                 queryAndFetchAll('SELECT * from builders', [], function (rows) {
181                     assert.strictEqual(rows[0].builder_name, emptyReport[0].builderName);
182                     notifyDone();
183                 });
184             });
185         });
186     });
187
188     it("should add a build", function () {
189         addBuilder(emptyReport, function () {
190             postJSON('/api/report/', emptyReport, function (response) {
191                 queryAndFetchAll('SELECT * from builds', [], function (rows) {
192                     assert.strictEqual(rows[0].build_number, 123);
193                     notifyDone();
194                 });
195             });
196         });
197     });
198
199     it("should add the platform", function () {
200         addBuilder(emptyReport, function () {
201             postJSON('/api/report/', emptyReport, function (response) {
202                 queryAndFetchAll('SELECT * from platforms', [], function (rows) {
203                     assert.strictEqual(rows[0].platform_name, 'Mountain Lion');
204                     notifyDone();
205                 });
206             });
207         });
208     });
209
210     it("should add repositories and build revisions", function () {
211         addBuilder(emptyReport, function () {
212             postJSON('/api/report/', emptyReport, function (response) {
213                 queryAndFetchAll('SELECT * FROM repositories', [], function (rows) {
214                     assert.deepEqual(rows.map(function (row) { return row['repository_name']; }), ['OS X', 'WebKit']);
215
216                     var repositoryIdToName = {};
217                     rows.forEach(function (row) { repositoryIdToName[row['repository_id']] = row['repository_name']; });
218                     queryAndFetchAll('SELECT * FROM build_commits, commits WHERE build_commit = commit_id', [], function (rows) {
219                         var repositoryNameToRevisionRow = {};
220                         rows.forEach(function (row) {
221                             repositoryNameToRevisionRow[repositoryIdToName[row['commit_repository']]] = row;
222                         });
223                         assert.equal(repositoryNameToRevisionRow['OS X']['commit_revision'], '10.8.2 12C60');
224                         assert.equal(repositoryNameToRevisionRow['WebKit']['commit_revision'], '141977');
225                         assert.equal(repositoryNameToRevisionRow['WebKit']['commit_time'].toString(),
226                             new Date('2013-02-06 08:55:20.9').toString());
227                         notifyDone();
228                     });
229                 });
230             });
231         });
232     });
233
234     it("should not create a duplicate build for the same build number if build times are close", function () {
235         addBuilder(emptyReport, function () {
236             emptyReport[0]['buildTime'] = '2013-02-28T10:12:04';
237             postJSON('/api/report/', emptyReport, function (response) {
238                 assert.equal(response.statusCode, 200);
239                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
240
241                 emptyReport[0]['buildTime'] = '2013-02-28T10:22:03';
242                 postJSON('/api/report/', emptyReport, function (response) {
243                     assert.equal(response.statusCode, 200);
244                     assert.equal(JSON.parse(response.responseText)['status'], 'OK');
245
246                     queryAndFetchAll('SELECT * FROM builds', [], function (rows) {
247                         assert.equal(rows.length, 1);
248                         notifyDone();
249                     });
250                 });
251             });
252         });
253     });
254
255     it("should create distinct builds for the same build number if build times are far apart", function () {
256         addBuilder(emptyReport, function () {
257             emptyReport[0]['buildTime'] = '2013-02-28T10:12:03';
258             postJSON('/api/report/', emptyReport, function (response) {
259                 assert.equal(response.statusCode, 200);
260                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
261
262                 queryAndFetchAll('SELECT * FROM builds', [], function (rows) {
263                     assert.equal(rows.length, 1);
264
265                     emptyReport[0]['buildTime'] = '2014-01-20T22:23:34';
266                     postJSON('/api/report/', emptyReport, function (response) {
267                         assert.equal(response.statusCode, 200);
268                         assert.equal(JSON.parse(response.responseText)['status'], 'OK');
269
270                         queryAndFetchAll('SELECT * FROM builds', [], function (rows) {
271                             assert.equal(rows.length, 2);
272                             notifyDone();
273                         });
274                     });
275                 });
276             });
277         });
278     });
279
280     it("should reject a report with mismatching revision info", function () {
281         addBuilder(emptyReport, function () {
282             emptyReport[0]['revisions'] = {
283                 "WebKit": {
284                     "revision": "141977",
285                     "timestamp": "2013-02-06T08:55:20.96Z"
286                 }
287             }
288             postJSON('/api/report/', emptyReport, function (response) {
289                 assert.equal(response.statusCode, 200);
290                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
291
292                 emptyReport[0]['revisions'] = {
293                     "WebKit": {
294                         "revision": "150000",
295                         "timestamp": "2013-05-13T10:50:29.6Z"
296                     }
297                 }
298                 postJSON('/api/report/', emptyReport, function (response) {
299                     assert.equal(response.statusCode, 200);
300                     assert.notEqual(JSON.parse(response.responseText)['status'], 'OK');
301                     assert(response.responseText.indexOf('141977') >= 0);
302                     assert(response.responseText.indexOf('150000') >= 0);
303                     assert.equal(JSON.parse(response.responseText)['failureStored'], true);
304                     assert.equal(JSON.parse(response.responseText)['processedRuns'], 0);
305                     notifyDone();
306                 });
307             });
308         });
309     });
310
311     var reportWithTwoLevelsOfAggregations = [{
312         "buildNumber": "123",
313         "buildTime": "2013-02-28T10:12:03.388304",
314         "builderName": "someBuilder",
315         "builderPassword": "somePassword",
316         "platform": "Mountain Lion",
317         "tests": {
318             "DummyPageLoading": {
319                 "metrics": {"Time": { "aggregators" : ["Arithmetic"], "current": [300, 310, 320, 330] }},
320                 "tests": {
321                     "apple.com": {
322                         "metrics": {"Time": { "current": [500, 510, 520, 530] }},
323                         "url": "http://www.apple.com"
324                     },
325                     "webkit.org": {
326                         "metrics": {"Time": { "current": [100, 110, 120, 130] }},
327                         "url": "http://www.webkit.org"
328                     }
329                 }
330             },
331             "DummyBenchmark": {
332                 "metrics": {"Time": ["Arithmetic"]},
333                 "tests": {
334                     "DOM": {
335                         "metrics": {"Time": ["Geometric", "Arithmetic"]},
336                         "tests": {
337                             "ModifyNodes": {"metrics": {"Time": { "current": [[11, 12, 13, 14, 15], [16, 17, 18, 19, 20], [21, 22, 23, 24, 25]] }}},
338                             "TraverseNodes": {"metrics": {"Time": { "current": [[31, 32, 33, 34, 35], [36, 37, 38, 39, 40], [41, 42, 43, 44, 45]] }}}
339                         }
340                     },
341                     "CSS": {"metrics": {"Time": { "current": [[101, 102, 103, 104, 105], [106, 107, 108, 109, 110], [111, 112, 113, 114, 115]] }}}
342                 }
343             }
344         },
345         "revisions": {
346             "OS X": {
347                 "revision": "10.8.2 12C60"
348             },
349             "WebKit": {
350                 "revision": "141977",
351                 "timestamp": "2013-02-06T08:55:20.9Z"
352             }
353         }}];
354
355     function addBuilderAndMeanAggregators(report, callback) {
356         addBuilder(report, function () {
357             queryAndFetchAll('INSERT INTO aggregators (aggregator_name, aggregator_definition) values ($1, $2)',
358                 ['Arithmetic', 'values.reduce(function (a, b) { return a + b; }) / values.length'], function () {
359                 queryAndFetchAll('INSERT INTO aggregators (aggregator_name, aggregator_definition) values ($1, $2)',
360                     ['Geometric', 'Math.pow(values.reduce(function (a, b) { return a * b; }), 1 / values.length)'], callback);
361             });
362         });
363     }
364
365     function fetchRunForMetric(testName, metricName,callback) {
366         queryAndFetchAll('SELECT * FROM test_runs WHERE run_config IN'
367             + '(SELECT config_id FROM test_configurations, test_metrics, tests WHERE config_metric = metric_id AND metric_test = test_id AND'
368             + 'test_name = $1 AND metric_name = $2)',
369             ['Arithmetic', 'values.reduce(function (a, b) { return a + b; }) / values.length'], function () {
370             queryAndFetchAll('INSERT INTO aggregators (aggregator_name, aggregator_definition) values ($1, $2)',
371                 ['Geometric', 'Math.pow(values.reduce(function (a, b) { return a * b; }), 1 / values.length)'], callback);
372         });
373     }
374
375     it("should reject when aggregators are missing", function () {
376         addBuilder(reportWithTwoLevelsOfAggregations, function () {
377             postJSON('/api/report/', reportWithTwoLevelsOfAggregations, function (response) {
378                 assert.equal(response.statusCode, 200);
379                 assert.equal(JSON.parse(response.responseText)['status'], 'AggregatorNotFound');
380                 assert.equal(JSON.parse(response.responseText)['failureStored'], true);
381                 notifyDone();
382             });
383         });
384     });
385
386     it("should add tests", function () {
387         addBuilderAndMeanAggregators(reportWithTwoLevelsOfAggregations, function () {
388             postJSON('/api/report/', reportWithTwoLevelsOfAggregations, function (response) {
389                 assert.equal(response.statusCode, 200);
390                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
391                 assert.equal(JSON.parse(response.responseText)['failureStored'], false);
392                 queryAndFetchAll('SELECT * FROM tests', [], function (rows) {
393                     assert.deepEqual(rows.map(function (row) { return row['test_name']; }).sort(),
394                         ['CSS', 'DOM', 'DummyBenchmark', 'DummyPageLoading', 'ModifyNodes', 'TraverseNodes', 'apple.com', 'webkit.org']);
395                     emptyReport[0].tests = {};
396                     notifyDone();
397                 });
398             });
399         });
400     });
401
402     it("should add metrics", function () {
403         addBuilderAndMeanAggregators(reportWithTwoLevelsOfAggregations, function () {
404             postJSON('/api/report/', reportWithTwoLevelsOfAggregations, function (response) {
405                 queryAndFetchAll('SELECT * FROM tests, test_metrics LEFT JOIN aggregators ON metric_aggregator = aggregator_id WHERE metric_test = test_id',
406                     [], function (rows) {
407                     var testNameToMetrics = {};
408                     rows.forEach(function (row) {
409                         if (!(row['test_name'] in testNameToMetrics))
410                             testNameToMetrics[row['test_name']] = new Array;
411                         testNameToMetrics[row['test_name']].push([row['metric_name'], row['aggregator_name']]);
412                     });
413                     assert.deepEqual(testNameToMetrics['CSS'], [['Time', null]]);
414                     assert.deepEqual(testNameToMetrics['DOM'].sort(), [['Time', 'Arithmetic'], ['Time', 'Geometric']]);
415                     assert.deepEqual(testNameToMetrics['DummyBenchmark'], [['Time', 'Arithmetic']]);
416                     assert.deepEqual(testNameToMetrics['DummyPageLoading'], [['Time', 'Arithmetic']]);
417                     assert.deepEqual(testNameToMetrics['ModifyNodes'], [['Time', null]]);
418                     assert.deepEqual(testNameToMetrics['TraverseNodes'], [['Time', null]]);
419                     assert.deepEqual(testNameToMetrics['apple.com'], [['Time', null]]);
420                     assert.deepEqual(testNameToMetrics['webkit.org'], [['Time', null]]);
421                     notifyDone();
422                 });
423             });
424         });
425     });
426
427     function fetchTestRunIterationsForMetric(testName, metricName, callback) {
428          queryAndFetchAll('SELECT * FROM tests, test_metrics, test_configurations, test_runs WHERE metric_test = test_id AND config_metric = metric_id'
429             + ' AND run_config = config_id AND test_name = $1 AND metric_name = $2', [testName, metricName], function (runRows) {
430                 assert.equal(runRows.length, 1);
431                 var run = runRows[0];
432                 queryAndFetchAll('SELECT * FROM run_iterations WHERE iteration_run = $1 ORDER BY iteration_order', [run['run_id']], function (iterationRows) {
433                     callback(run, iterationRows);
434                 });
435             });
436     }
437
438     it("should store run values", function () {
439         addBuilderAndMeanAggregators(reportWithTwoLevelsOfAggregations, function () {
440             postJSON('/api/report/', reportWithTwoLevelsOfAggregations, function (response) {
441                 fetchTestRunIterationsForMetric('apple.com', 'Time', function (run, iterations) {
442                     var runId = run['run_id'];
443                     assert.deepEqual(iterations, [
444                         {iteration_run: runId, iteration_order: 0, iteration_group: null, iteration_value: 500, iteration_relative_time: null},
445                         {iteration_run: runId, iteration_order: 1, iteration_group: null, iteration_value: 510, iteration_relative_time: null},
446                         {iteration_run: runId, iteration_order: 2, iteration_group: null, iteration_value: 520, iteration_relative_time: null},
447                         {iteration_run: runId, iteration_order: 3, iteration_group: null, iteration_value: 530, iteration_relative_time: null}]);
448                     var sum = 500 + 510 + 520 + 530;
449                     assert.equal(run['run_mean_cache'], sum / iterations.length);
450                     assert.equal(run['run_sum_cache'], sum);
451                     assert.equal(run['run_square_sum_cache'], 500 * 500 + 510 * 510 + 520 * 520 + 530 * 530);
452
453                     fetchTestRunIterationsForMetric('CSS', 'Time', function (run, iterations) {
454                         var runId = run['run_id'];
455                         assert.deepEqual(iterations, [
456                             {iteration_run: runId, iteration_order: 0, iteration_group: 0, iteration_value: 101, iteration_relative_time: null},
457                             {iteration_run: runId, iteration_order: 1, iteration_group: 0, iteration_value: 102, iteration_relative_time: null},
458                             {iteration_run: runId, iteration_order: 2, iteration_group: 0, iteration_value: 103, iteration_relative_time: null},
459                             {iteration_run: runId, iteration_order: 3, iteration_group: 0, iteration_value: 104, iteration_relative_time: null},
460                             {iteration_run: runId, iteration_order: 4, iteration_group: 0, iteration_value: 105, iteration_relative_time: null},
461                             {iteration_run: runId, iteration_order: 5, iteration_group: 1, iteration_value: 106, iteration_relative_time: null},
462                             {iteration_run: runId, iteration_order: 6, iteration_group: 1, iteration_value: 107, iteration_relative_time: null},
463                             {iteration_run: runId, iteration_order: 7, iteration_group: 1, iteration_value: 108, iteration_relative_time: null},
464                             {iteration_run: runId, iteration_order: 8, iteration_group: 1, iteration_value: 109, iteration_relative_time: null},
465                             {iteration_run: runId, iteration_order: 9, iteration_group: 1, iteration_value: 110, iteration_relative_time: null},
466                             {iteration_run: runId, iteration_order: 10, iteration_group: 2, iteration_value: 111, iteration_relative_time: null},
467                             {iteration_run: runId, iteration_order: 11, iteration_group: 2, iteration_value: 112, iteration_relative_time: null},
468                             {iteration_run: runId, iteration_order: 12, iteration_group: 2, iteration_value: 113, iteration_relative_time: null},
469                             {iteration_run: runId, iteration_order: 13, iteration_group: 2, iteration_value: 114, iteration_relative_time: null},
470                             {iteration_run: runId, iteration_order: 14, iteration_group: 2, iteration_value: 115, iteration_relative_time: null}]);
471                         var sum = 0;
472                         var squareSum = 0;
473                         for (var value = 101; value <= 115; ++value) {
474                             sum += value;
475                             squareSum += value * value;
476                         }
477                         assert.equal(run['run_mean_cache'], sum / iterations.length);
478                         assert.equal(run['run_sum_cache'], sum);
479                         assert.equal(run['run_square_sum_cache'], squareSum);
480                         notifyDone();
481                     });
482                 });
483             });
484         });
485     });
486
487     it("should store aggregated run values", function () {
488         addBuilderAndMeanAggregators(reportWithTwoLevelsOfAggregations, function () {
489             postJSON('/api/report/', reportWithTwoLevelsOfAggregations, function (response) {
490                 fetchTestRunIterationsForMetric('DummyPageLoading', 'Time', function (run, iterations) {
491                     var runId = run['run_id'];
492                     var expectedValues = [(500 + 100) / 2, (510 + 110) / 2, (520 + 120) / 2, (530 + 130) / 2];
493                     assert.deepEqual(iterations, [
494                         {iteration_run: runId, iteration_order: 0, iteration_group: null, iteration_value: expectedValues[0], iteration_relative_time: null},
495                         {iteration_run: runId, iteration_order: 1, iteration_group: null, iteration_value: expectedValues[1], iteration_relative_time: null},
496                         {iteration_run: runId, iteration_order: 2, iteration_group: null, iteration_value: expectedValues[2], iteration_relative_time: null},
497                         {iteration_run: runId, iteration_order: 3, iteration_group: null, iteration_value: expectedValues[3], iteration_relative_time: null}]);
498                     var sum = expectedValues.reduce(function (sum, value) { return sum + value; }, 0);
499                     assert.equal(run['run_mean_cache'], sum / iterations.length);
500                     assert.equal(run['run_sum_cache'], sum);
501                     assert.equal(run['run_square_sum_cache'], expectedValues.reduce(function (sum, value) { return sum + value * value; }, 0));
502                     notifyDone();
503                 });
504             });
505         });
506     });
507
508     it("should be able to compute the aggregation of aggregated values", function () {
509         addBuilderAndMeanAggregators(reportWithTwoLevelsOfAggregations, function () {
510             postJSON('/api/report/', reportWithTwoLevelsOfAggregations, function (response) {
511                 fetchTestRunIterationsForMetric('DummyBenchmark', 'Time', function (run, iterations) {
512                     var expectedIterations = [];
513                     var sum = 0;
514                     var squareSum = 0;
515                     for (var i = 0; i < 15; ++i) {
516                         var value = i + 1;
517                         var DOMMean = ((10 + value) + (30 + value)) / 2;
518                         var expectedValue = (DOMMean + 100 + value) / 2;
519                         sum += expectedValue;
520                         squareSum += expectedValue * expectedValue;
521                         expectedIterations.push({iteration_run: run['run_id'],
522                             iteration_order: i,
523                             iteration_group: Math.floor(i / 5),
524                             iteration_value: expectedValue,
525                             iteration_relative_time: null});
526                     }
527                     assert.deepEqual(iterations, expectedIterations);
528                     assert.equal(run['run_mean_cache'], sum / iterations.length);
529                     assert.equal(run['run_sum_cache'], sum);
530                     assert.equal(run['run_square_sum_cache'], squareSum);
531                     notifyDone();
532                 });
533             });
534         });
535     });
536
537     var reportWithSameSubtestName = [{
538         "buildNumber": "123",
539         "buildTime": "2013-02-28T10:12:03.388304",
540         "builderName": "someBuilder",
541         "builderPassword": "somePassword",
542         "platform": "Mountain Lion",
543         "tests": {
544             "Suite1": {
545                 "tests": {
546                     "test1": {
547                         "metrics": {"Time": { "current": [1, 2, 3, 4, 5] }}
548                     },
549                     "test2": {
550                         "metrics": {"Time": { "current": [6, 7, 8, 9, 10] }}
551                     }
552                 }
553             },
554             "Suite2": {
555                 "tests": {
556                     "test1": {
557                         "metrics": {"Time": { "current": [11, 12, 13, 14, 15] }}
558                     },
559                     "test2": {
560                         "metrics": {"Time": { "current": [16, 17, 18, 19, 20] }}
561                     }
562                 }
563             }
564         }}];
565
566     it("should be able to add a report with same subtest name", function () {
567         addBuilderAndMeanAggregators(reportWithSameSubtestName, function () {
568             postJSON('/api/report/', reportWithSameSubtestName, function (response) {
569                 assert.equal(response.statusCode, 200);
570                 try {
571                     JSON.parse(response.responseText);
572                 } catch (error) {
573                     assert.fail(error, null, response.responseText);
574                 }
575                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
576                 assert.equal(JSON.parse(response.responseText)['failureStored'], false);
577                 notifyDone();
578             });
579         });
580     });
581
582     it("should be able to reuse the same test rows", function () {
583         addBuilderAndMeanAggregators(reportWithSameSubtestName, function () {
584             postJSON('/api/report/', reportWithSameSubtestName, function (response) {
585                 assert.equal(response.statusCode, 200);
586                 queryAndFetchAll('SELECT * FROM tests', [], function (testRows) {
587                    assert.equal(testRows.length, 6);
588                    reportWithSameSubtestName.buildNumber = "125";
589                    reportWithSameSubtestName.buildTime = "2013-02-28T12:17:24.1";
590
591                    postJSON('/api/report/', reportWithSameSubtestName, function (response) {
592                        assert.equal(response.statusCode, 200);
593                        queryAndFetchAll('SELECT * FROM tests', [], function (testRows) {
594                               assert.equal(testRows.length, 6);
595                               notifyDone();
596                           });
597                       });
598                 });
599             });
600         });
601     });
602
603     var reportWithSameSingleValue = [{
604         "buildNumber": "123",
605         "buildTime": "2013-02-28T10:12:03.388304",
606         "builderName": "someBuilder",
607         "builderPassword": "somePassword",
608         "platform": "Mountain Lion",
609         "tests": {
610             "suite": {
611                 "metrics": {"Combined": ["Arithmetic"]},
612                 "tests": {
613                     "test1": {
614                         "metrics": {"Combined": { "current": 3 }}
615                     },
616                     "test2": {
617                         "metrics": {"Combined": { "current": 7 }}
618                     }
619                 }
620             },
621         }}];
622
623     it("should be able to add a report with single value results", function () {
624         addBuilderAndMeanAggregators(reportWithSameSingleValue, function () {
625             postJSON('/api/report/', reportWithSameSingleValue, function (response) {
626                 assert.equal(response.statusCode, 200);
627                 try {
628                     JSON.parse(response.responseText);
629                 } catch (error) {
630                     assert.fail(error, null, response.responseText);
631                 }
632                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
633                 assert.equal(JSON.parse(response.responseText)['failureStored'], false);
634                 fetchTestRunIterationsForMetric('test1', 'Combined', function (run, iterations) {
635                     assert.equal(run['run_iteration_count_cache'], 1);
636                     assert.equal(run['run_mean_cache'], 3);
637                     assert.equal(run['run_sum_cache'], 3);
638                     assert.equal(run['run_square_sum_cache'], 9);
639                     fetchTestRunIterationsForMetric('suite', 'Combined', function (run, iterations) {
640                         assert.equal(run['run_iteration_count_cache'], 1);
641                         assert.equal(run['run_mean_cache'], 5);
642                         assert.equal(run['run_sum_cache'], 5);
643                         assert.equal(run['run_square_sum_cache'], 25);
644                         notifyDone();
645                     });
646                 });
647             });
648         });
649     });
650
651     var reportWithSameValuePairs = [{
652         "buildNumber": "123",
653         "buildTime": "2013-02-28T10:12:03.388304",
654         "builderName": "someBuilder",
655         "builderPassword": "somePassword",
656         "platform": "Mountain Lion",
657         "tests": {
658                 "test": {
659                     "metrics": {"FrameRate": { "current": [[[0, 4], [100, 5], [205, 3]]] }}
660                 },
661             },
662         }];
663
664     it("should be able to add a report with (relative time, value) pairs", function () {
665         addBuilderAndMeanAggregators(reportWithSameValuePairs, function () {
666             postJSON('/api/report/', reportWithSameValuePairs, function (response) {
667                 assert.equal(response.statusCode, 200);
668                 try {
669                     JSON.parse(response.responseText);
670                 } catch (error) {
671                     assert.fail(error, null, response.responseText);
672                 }
673                 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
674                 assert.equal(JSON.parse(response.responseText)['failureStored'], false);
675                 fetchTestRunIterationsForMetric('test', 'FrameRate', function (run, iterations) {
676                     assert.equal(run['run_iteration_count_cache'], 3);
677                     assert.equal(run['run_mean_cache'], 4);
678                     assert.equal(run['run_sum_cache'], 12);
679                     assert.equal(run['run_square_sum_cache'], 16 + 25 + 9);
680                     var runId = run['run_id'];
681                     assert.deepEqual(iterations, [
682                         {iteration_run: runId, iteration_order: 0, iteration_group: null, iteration_value: 4, iteration_relative_time: 0},
683                         {iteration_run: runId, iteration_order: 1, iteration_group: null, iteration_value: 5, iteration_relative_time: 100},
684                         {iteration_run: runId, iteration_order: 2, iteration_group: null, iteration_value: 3, iteration_relative_time: 205}]);
685                     notifyDone();
686                 });
687             });
688         });
689     });
690
691     var reportsUpdatingDifferentTests = [
692         [{
693             "buildNumber": "123",
694             "buildTime": "2013-02-28T10:12:03",
695             "builderName": "someBuilder",
696             "builderPassword": "somePassword",
697             "platform": "Mountain Lion",
698             "tests": {"test1": {"metrics": {"Time": {"current": 3}}}}
699         }],
700         [{
701             "buildNumber": "124",
702             "buildTime": "2013-02-28T11:31:21",
703             "builderName": "someBuilder",
704             "builderPassword": "somePassword",
705             "platform": "Mountain Lion",
706             "tests": {"test2": {"metrics": {"Time": {"current": 3}}}}
707         }],
708         [{
709             "buildNumber": "125",
710             "buildTime": "2013-02-28T12:45:34",
711             "builderName": "someBuilder",
712             "builderPassword": "somePassword",
713             "platform": "Mountain Lion",
714             "tests": {"test1": {"metrics": {"Time": {"current": 3}}}}
715         }],
716     ];
717
718     function fetchTestConfig(testName, metricName, callback) {
719          queryAndFetchAll('SELECT * FROM tests, test_metrics, test_configurations WHERE test_id = metric_test AND metric_id = config_metric'
720             + ' AND test_name = $1 AND metric_name = $2', [testName, metricName], function (runRows) {
721                 assert.equal(runRows.length, 1);
722                 callback(runRows[0]);
723             });
724     }
725
726     it("should update the last modified date of test configurations with new runs", function () {
727         addBuilder(reportsUpdatingDifferentTests[0], function () {
728             postJSON('/api/report/', reportsUpdatingDifferentTests[0], function (response) {
729                 assert.equal(response.statusCode, 200);
730                 fetchTestConfig('test1', 'Time', function (originalConfig) {
731                     postJSON('/api/report/', reportsUpdatingDifferentTests[2], function (response) {
732                         assert.equal(response.statusCode, 200);
733                         fetchTestConfig('test1', 'Time', function (config) {
734                             assert.notEqual(+originalConfig['config_runs_last_modified'], +config['config_runs_last_modified']);
735                             notifyDone();
736                         });
737                     });
738                 });
739             });
740         });
741     });
742
743     it("should update the last modified date of unrelated test configurations", function () {
744         addBuilder(reportsUpdatingDifferentTests[0], function () {
745             postJSON('/api/report/', reportsUpdatingDifferentTests[0], function (response) {
746                 assert.equal(response.statusCode, 200);
747                 fetchTestConfig('test1', 'Time', function (originalConfig) {
748                     postJSON('/api/report/', reportsUpdatingDifferentTests[1], function (response) {
749                         assert.equal(response.statusCode, 200);
750                         fetchTestConfig('test1', 'Time', function (config) {
751                             assert.equal(+originalConfig['config_runs_last_modified'], +config['config_runs_last_modified']);
752                             notifyDone();
753                         });
754                     });
755                 });
756             });
757         });
758     });
759 });