1 describe("/api/report", function () {
4 "buildTime": "2013-02-28T10:12:03.388304",
5 "builderName": "someBuilder",
6 "builderPassword": "somePassword",
7 "platform": "Mountain Lion",
11 "revision": "10.8.2 12C60"
15 "timestamp": "2013-02-06T08:55:20.9Z"
19 function addBuilder(report, callback) {
20 queryAndFetchAll('INSERT INTO builders (builder_name, builder_password_hash) values ($1, $2)',
21 [report[0].builderName, sha256(report[0].builderPassword)], callback);
24 it("should reject error when builder name is missing", function () {
25 postJSON('/api/report/', [{"buildTime": "2013-02-28T10:12:03.388304"}], function (response) {
26 assert.equal(response.statusCode, 200);
27 assert.equal(JSON.parse(response.responseText)['status'], 'MissingBuilderName');
32 it("should reject error when build time is missing", function () {
33 addBuilder(emptyReport, function () {
34 postJSON('/api/report/', [{"builderName": "someBuilder", "builderPassword": "somePassword"}], function (response) {
35 assert.equal(response.statusCode, 200);
36 assert.equal(JSON.parse(response.responseText)['status'], 'MissingBuildTime');
42 it("should reject when there are no builders", function () {
43 postJSON('/api/report/', emptyReport, function (response) {
44 assert.equal(response.statusCode, 200);
45 assert.notEqual(JSON.parse(response.responseText)['status'], 'OK');
46 assert.equal(JSON.parse(response.responseText)['failureStored'], false);
47 assert.equal(JSON.parse(response.responseText)['processedRuns'], 0);
49 queryAndFetchAll('SELECT COUNT(*) from reports', [], function (rows) {
50 assert.equal(rows[0].count, 0);
56 it("should store a report from a valid builder", function () {
57 addBuilder(emptyReport, function () {
58 postJSON('/api/report/', emptyReport, function (response) {
59 assert.equal(response.statusCode, 200);
60 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
61 assert.equal(JSON.parse(response.responseText)['failureStored'], false);
62 assert.equal(JSON.parse(response.responseText)['processedRuns'], 1);
63 queryAndFetchAll('SELECT COUNT(*) from reports', [], function (rows) {
64 assert.equal(rows[0].count, 1);
71 it("should store the builder name but not the builder password", function () {
72 addBuilder(emptyReport, function () {
73 postJSON('/api/report/', emptyReport, function (response) {
74 queryAndFetchAll('SELECT report_content from reports', [], function (rows) {
75 var storedContent = JSON.parse(rows[0].report_content);
76 assert.equal(storedContent['builderName'], emptyReport[0]['builderName']);
77 assert(!('builderPassword' in storedContent));
84 it("should add a build", function () {
85 addBuilder(emptyReport, function () {
86 postJSON('/api/report/', emptyReport, function (response) {
87 queryAndFetchAll('SELECT * from builds', [], function (rows) {
88 assert.strictEqual(rows[0].build_number, 123);
95 it("should add the platform", function () {
96 addBuilder(emptyReport, function () {
97 postJSON('/api/report/', emptyReport, function (response) {
98 queryAndFetchAll('SELECT * from platforms', [], function (rows) {
99 assert.strictEqual(rows[0].platform_name, 'Mountain Lion');
106 it("should add repositories and build revisions", function () {
107 addBuilder(emptyReport, function () {
108 postJSON('/api/report/', emptyReport, function (response) {
109 queryAndFetchAll('SELECT * FROM repositories', [], function (rows) {
110 assert.deepEqual(rows.map(function (row) { return row['repository_name']; }), ['OS X', 'WebKit']);
112 var repositoryIdToName = {};
113 rows.forEach(function (row) { repositoryIdToName[row['repository_id']] = row['repository_name']; });
114 queryAndFetchAll('SELECT * FROM build_revisions', [], function (rows) {
115 var repositoryNameToRevisionRow = {};
116 rows.forEach(function (row) {
117 repositoryNameToRevisionRow[repositoryIdToName[row['revision_repository']]] = row;
119 assert.equal(repositoryNameToRevisionRow['OS X']['revision_value'], '10.8.2 12C60');
120 assert.equal(repositoryNameToRevisionRow['WebKit']['revision_value'], '141977');
121 assert.equal(repositoryNameToRevisionRow['WebKit']['revision_time'].toString(),
122 new Date('2013-02-06 08:55:20.9').toString());
130 it("should not create a duplicate build for the same build number if build times are close", function () {
131 addBuilder(emptyReport, function () {
132 emptyReport[0]['buildTime'] = '2013-02-28T10:12:04';
133 postJSON('/api/report/', emptyReport, function (response) {
134 assert.equal(response.statusCode, 200);
135 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
137 emptyReport[0]['buildTime'] = '2013-02-28T10:22:03';
138 postJSON('/api/report/', emptyReport, function (response) {
139 assert.equal(response.statusCode, 200);
140 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
142 queryAndFetchAll('SELECT * FROM builds', [], function (rows) {
143 assert.equal(rows.length, 1);
151 it("should create distinct builds for the same build number if build times are far apart", function () {
152 addBuilder(emptyReport, function () {
153 emptyReport[0]['buildTime'] = '2013-02-28T10:12:03';
154 postJSON('/api/report/', emptyReport, function (response) {
155 assert.equal(response.statusCode, 200);
156 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
158 queryAndFetchAll('SELECT * FROM builds', [], function (rows) {
159 assert.equal(rows.length, 1);
161 emptyReport[0]['buildTime'] = '2014-01-20T22:23:34';
162 postJSON('/api/report/', emptyReport, function (response) {
163 assert.equal(response.statusCode, 200);
164 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
166 queryAndFetchAll('SELECT * FROM builds', [], function (rows) {
167 assert.equal(rows.length, 2);
176 it("should reject a report with mismatching revision info", function () {
177 addBuilder(emptyReport, function () {
178 emptyReport[0]['revisions'] = {
180 "revision": "141977",
181 "timestamp": "2013-02-06T08:55:20.96Z"
184 postJSON('/api/report/', emptyReport, function (response) {
185 assert.equal(response.statusCode, 200);
186 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
188 emptyReport[0]['revisions'] = {
190 "revision": "150000",
191 "timestamp": "2013-05-13T10:50:29.6Z"
194 postJSON('/api/report/', emptyReport, function (response) {
195 assert.equal(response.statusCode, 200);
196 assert.notEqual(JSON.parse(response.responseText)['status'], 'OK');
197 assert(response.responseText.indexOf('141977') >= 0);
198 assert(response.responseText.indexOf('150000') >= 0);
199 assert.equal(JSON.parse(response.responseText)['failureStored'], true);
200 assert.equal(JSON.parse(response.responseText)['processedRuns'], 0);
207 var reportWithTwoLevelsOfAggregations = [{
208 "buildNumber": "123",
209 "buildTime": "2013-02-28T10:12:03.388304",
210 "builderName": "someBuilder",
211 "builderPassword": "somePassword",
212 "platform": "Mountain Lion",
214 "DummyPageLoading": {
215 "metrics": {"Time": { "aggregators" : ["Arithmetic"], "current": [300, 310, 320, 330] }},
218 "metrics": {"Time": { "current": [500, 510, 520, 530] }},
219 "url": "http://www.apple.com"
222 "metrics": {"Time": { "current": [100, 110, 120, 130] }},
223 "url": "http://www.webkit.org"
228 "metrics": {"Time": ["Arithmetic"]},
231 "metrics": {"Time": ["Geometric", "Arithmetic"]},
233 "ModifyNodes": {"metrics": {"Time": { "current": [[11, 12, 13, 14, 15], [16, 17, 18, 19, 20], [21, 22, 23, 24, 25]] }}},
234 "TraverseNodes": {"metrics": {"Time": { "current": [[31, 32, 33, 34, 35], [36, 37, 38, 39, 40], [41, 42, 43, 44, 45]] }}}
237 "CSS": {"metrics": {"Time": { "current": [[101, 102, 103, 104, 105], [106, 107, 108, 109, 110], [111, 112, 113, 114, 115]] }}}
243 "revision": "10.8.2 12C60"
246 "revision": "141977",
247 "timestamp": "2013-02-06T08:55:20.9Z"
251 function addBuilderAndMeanAggregators(report, callback) {
252 addBuilder(report, function () {
253 queryAndFetchAll('INSERT INTO aggregators (aggregator_name, aggregator_definition) values ($1, $2)',
254 ['Arithmetic', 'values.reduce(function (a, b) { return a + b; }) / values.length'], function () {
255 queryAndFetchAll('INSERT INTO aggregators (aggregator_name, aggregator_definition) values ($1, $2)',
256 ['Geometric', 'Math.pow(values.reduce(function (a, b) { return a * b; }), 1 / values.length)'], callback);
261 function fetchRunForMetric(testName, metricName,callback) {
262 queryAndFetchAll('SELECT * FROM test_runs WHERE run_config IN'
263 + '(SELECT config_id FROM test_configurations, test_metrics, tests WHERE config_metric = metric_id AND metric_test = test_id AND'
264 + 'test_name = $1 AND metric_name = $2)',
265 ['Arithmetic', 'values.reduce(function (a, b) { return a + b; }) / values.length'], function () {
266 queryAndFetchAll('INSERT INTO aggregators (aggregator_name, aggregator_definition) values ($1, $2)',
267 ['Geometric', 'Math.pow(values.reduce(function (a, b) { return a * b; }), 1 / values.length)'], callback);
271 it("should reject when aggregators are missing", function () {
272 addBuilder(reportWithTwoLevelsOfAggregations, function () {
273 postJSON('/api/report/', reportWithTwoLevelsOfAggregations, function (response) {
274 assert.equal(response.statusCode, 200);
275 assert.equal(JSON.parse(response.responseText)['status'], 'AggregatorNotFound');
276 assert.equal(JSON.parse(response.responseText)['failureStored'], true);
282 it("should add tests", function () {
283 addBuilderAndMeanAggregators(reportWithTwoLevelsOfAggregations, function () {
284 postJSON('/api/report/', reportWithTwoLevelsOfAggregations, function (response) {
285 assert.equal(response.statusCode, 200);
286 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
287 assert.equal(JSON.parse(response.responseText)['failureStored'], false);
288 queryAndFetchAll('SELECT * FROM tests', [], function (rows) {
289 assert.deepEqual(rows.map(function (row) { return row['test_name']; }).sort(),
290 ['CSS', 'DOM', 'DummyBenchmark', 'DummyPageLoading', 'ModifyNodes', 'TraverseNodes', 'apple.com', 'webkit.org']);
291 emptyReport[0].tests = {};
298 it("should add metrics", function () {
299 addBuilderAndMeanAggregators(reportWithTwoLevelsOfAggregations, function () {
300 postJSON('/api/report/', reportWithTwoLevelsOfAggregations, function (response) {
301 queryAndFetchAll('SELECT * FROM tests, test_metrics LEFT JOIN aggregators ON metric_aggregator = aggregator_id WHERE metric_test = test_id',
302 [], function (rows) {
303 var testNameToMetrics = {};
304 rows.forEach(function (row) {
305 if (!(row['test_name'] in testNameToMetrics))
306 testNameToMetrics[row['test_name']] = new Array;
307 testNameToMetrics[row['test_name']].push([row['metric_name'], row['aggregator_name']]);
309 assert.deepEqual(testNameToMetrics['CSS'], [['Time', null]]);
310 assert.deepEqual(testNameToMetrics['DOM'].sort(), [['Time', 'Arithmetic'], ['Time', 'Geometric']]);
311 assert.deepEqual(testNameToMetrics['DummyBenchmark'], [['Time', 'Arithmetic']]);
312 assert.deepEqual(testNameToMetrics['DummyPageLoading'], [['Time', 'Arithmetic']]);
313 assert.deepEqual(testNameToMetrics['ModifyNodes'], [['Time', null]]);
314 assert.deepEqual(testNameToMetrics['TraverseNodes'], [['Time', null]]);
315 assert.deepEqual(testNameToMetrics['apple.com'], [['Time', null]]);
316 assert.deepEqual(testNameToMetrics['webkit.org'], [['Time', null]]);
323 function fetchTestRunIterationsForMetric(testName, metricName, callback) {
324 queryAndFetchAll('SELECT * FROM tests, test_metrics, test_configurations, test_runs WHERE metric_test = test_id AND config_metric = metric_id'
325 + ' AND run_config = config_id AND test_name = $1 AND metric_name = $2', [testName, metricName], function (runRows) {
326 assert.equal(runRows.length, 1);
327 var run = runRows[0];
328 queryAndFetchAll('SELECT * FROM run_iterations WHERE iteration_run = $1 ORDER BY iteration_order', [run['run_id']], function (iterationRows) {
329 callback(run, iterationRows);
334 it("should store run values", function () {
335 addBuilderAndMeanAggregators(reportWithTwoLevelsOfAggregations, function () {
336 postJSON('/api/report/', reportWithTwoLevelsOfAggregations, function (response) {
337 fetchTestRunIterationsForMetric('apple.com', 'Time', function (run, iterations) {
338 var runId = run['run_id'];
339 assert.deepEqual(iterations, [
340 {iteration_run: runId, iteration_order: 0, iteration_group: null, iteration_value: 500, iteration_relative_time: null},
341 {iteration_run: runId, iteration_order: 1, iteration_group: null, iteration_value: 510, iteration_relative_time: null},
342 {iteration_run: runId, iteration_order: 2, iteration_group: null, iteration_value: 520, iteration_relative_time: null},
343 {iteration_run: runId, iteration_order: 3, iteration_group: null, iteration_value: 530, iteration_relative_time: null}]);
344 var sum = 500 + 510 + 520 + 530;
345 assert.equal(run['run_mean_cache'], sum / iterations.length);
346 assert.equal(run['run_sum_cache'], sum);
347 assert.equal(run['run_square_sum_cache'], 500 * 500 + 510 * 510 + 520 * 520 + 530 * 530);
349 fetchTestRunIterationsForMetric('CSS', 'Time', function (run, iterations) {
350 var runId = run['run_id'];
351 assert.deepEqual(iterations, [
352 {iteration_run: runId, iteration_order: 0, iteration_group: 0, iteration_value: 101, iteration_relative_time: null},
353 {iteration_run: runId, iteration_order: 1, iteration_group: 0, iteration_value: 102, iteration_relative_time: null},
354 {iteration_run: runId, iteration_order: 2, iteration_group: 0, iteration_value: 103, iteration_relative_time: null},
355 {iteration_run: runId, iteration_order: 3, iteration_group: 0, iteration_value: 104, iteration_relative_time: null},
356 {iteration_run: runId, iteration_order: 4, iteration_group: 0, iteration_value: 105, iteration_relative_time: null},
357 {iteration_run: runId, iteration_order: 5, iteration_group: 1, iteration_value: 106, iteration_relative_time: null},
358 {iteration_run: runId, iteration_order: 6, iteration_group: 1, iteration_value: 107, iteration_relative_time: null},
359 {iteration_run: runId, iteration_order: 7, iteration_group: 1, iteration_value: 108, iteration_relative_time: null},
360 {iteration_run: runId, iteration_order: 8, iteration_group: 1, iteration_value: 109, iteration_relative_time: null},
361 {iteration_run: runId, iteration_order: 9, iteration_group: 1, iteration_value: 110, iteration_relative_time: null},
362 {iteration_run: runId, iteration_order: 10, iteration_group: 2, iteration_value: 111, iteration_relative_time: null},
363 {iteration_run: runId, iteration_order: 11, iteration_group: 2, iteration_value: 112, iteration_relative_time: null},
364 {iteration_run: runId, iteration_order: 12, iteration_group: 2, iteration_value: 113, iteration_relative_time: null},
365 {iteration_run: runId, iteration_order: 13, iteration_group: 2, iteration_value: 114, iteration_relative_time: null},
366 {iteration_run: runId, iteration_order: 14, iteration_group: 2, iteration_value: 115, iteration_relative_time: null}]);
369 for (var value = 101; value <= 115; ++value) {
371 squareSum += value * value;
373 assert.equal(run['run_mean_cache'], sum / iterations.length);
374 assert.equal(run['run_sum_cache'], sum);
375 assert.equal(run['run_square_sum_cache'], squareSum);
383 it("should store aggregated run values", function () {
384 addBuilderAndMeanAggregators(reportWithTwoLevelsOfAggregations, function () {
385 postJSON('/api/report/', reportWithTwoLevelsOfAggregations, function (response) {
386 fetchTestRunIterationsForMetric('DummyPageLoading', 'Time', function (run, iterations) {
387 var runId = run['run_id'];
388 var expectedValues = [(500 + 100) / 2, (510 + 110) / 2, (520 + 120) / 2, (530 + 130) / 2];
389 assert.deepEqual(iterations, [
390 {iteration_run: runId, iteration_order: 0, iteration_group: null, iteration_value: expectedValues[0], iteration_relative_time: null},
391 {iteration_run: runId, iteration_order: 1, iteration_group: null, iteration_value: expectedValues[1], iteration_relative_time: null},
392 {iteration_run: runId, iteration_order: 2, iteration_group: null, iteration_value: expectedValues[2], iteration_relative_time: null},
393 {iteration_run: runId, iteration_order: 3, iteration_group: null, iteration_value: expectedValues[3], iteration_relative_time: null}]);
394 var sum = expectedValues.reduce(function (sum, value) { return sum + value; }, 0);
395 assert.equal(run['run_mean_cache'], sum / iterations.length);
396 assert.equal(run['run_sum_cache'], sum);
397 assert.equal(run['run_square_sum_cache'], expectedValues.reduce(function (sum, value) { return sum + value * value; }, 0));
404 it("should be able to compute the aggregation of aggregated values", function () {
405 addBuilderAndMeanAggregators(reportWithTwoLevelsOfAggregations, function () {
406 postJSON('/api/report/', reportWithTwoLevelsOfAggregations, function (response) {
407 fetchTestRunIterationsForMetric('DummyBenchmark', 'Time', function (run, iterations) {
408 var expectedIterations = [];
411 for (var i = 0; i < 15; ++i) {
413 var DOMMean = ((10 + value) + (30 + value)) / 2;
414 var expectedValue = (DOMMean + 100 + value) / 2;
415 sum += expectedValue;
416 squareSum += expectedValue * expectedValue;
417 expectedIterations.push({iteration_run: run['run_id'],
419 iteration_group: Math.floor(i / 5),
420 iteration_value: expectedValue,
421 iteration_relative_time: null});
423 assert.deepEqual(iterations, expectedIterations);
424 assert.equal(run['run_mean_cache'], sum / iterations.length);
425 assert.equal(run['run_sum_cache'], sum);
426 assert.equal(run['run_square_sum_cache'], squareSum);
433 var reportWithSameSubtestName = [{
434 "buildNumber": "123",
435 "buildTime": "2013-02-28T10:12:03.388304",
436 "builderName": "someBuilder",
437 "builderPassword": "somePassword",
438 "platform": "Mountain Lion",
443 "metrics": {"Time": { "current": [1, 2, 3, 4, 5] }}
446 "metrics": {"Time": { "current": [6, 7, 8, 9, 10] }}
453 "metrics": {"Time": { "current": [11, 12, 13, 14, 15] }}
456 "metrics": {"Time": { "current": [16, 17, 18, 19, 20] }}
462 it("should be able to add a report with same subtest name", function () {
463 addBuilderAndMeanAggregators(reportWithSameSubtestName, function () {
464 postJSON('/api/report/', reportWithSameSubtestName, function (response) {
465 assert.equal(response.statusCode, 200);
467 JSON.parse(response.responseText);
469 assert.fail(error, null, response.responseText);
471 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
472 assert.equal(JSON.parse(response.responseText)['failureStored'], false);
478 it("should be able to reuse the same test rows", function () {
479 addBuilderAndMeanAggregators(reportWithSameSubtestName, function () {
480 postJSON('/api/report/', reportWithSameSubtestName, function (response) {
481 assert.equal(response.statusCode, 200);
482 queryAndFetchAll('SELECT * FROM tests', [], function (testRows) {
483 assert.equal(testRows.length, 6);
484 reportWithSameSubtestName.buildNumber = "125";
485 reportWithSameSubtestName.buildTime = "2013-02-28T12:17:24.1";
487 postJSON('/api/report/', reportWithSameSubtestName, function (response) {
488 assert.equal(response.statusCode, 200);
489 queryAndFetchAll('SELECT * FROM tests', [], function (testRows) {
490 assert.equal(testRows.length, 6);
499 var reportWithSameSingleValue = [{
500 "buildNumber": "123",
501 "buildTime": "2013-02-28T10:12:03.388304",
502 "builderName": "someBuilder",
503 "builderPassword": "somePassword",
504 "platform": "Mountain Lion",
507 "metrics": {"Combined": ["Arithmetic"]},
510 "metrics": {"Combined": { "current": 3 }}
513 "metrics": {"Combined": { "current": 7 }}
519 it("should be able to add a report with single value results", function () {
520 addBuilderAndMeanAggregators(reportWithSameSingleValue, function () {
521 postJSON('/api/report/', reportWithSameSingleValue, function (response) {
522 assert.equal(response.statusCode, 200);
524 JSON.parse(response.responseText);
526 assert.fail(error, null, response.responseText);
528 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
529 assert.equal(JSON.parse(response.responseText)['failureStored'], false);
530 fetchTestRunIterationsForMetric('test1', 'Combined', function (run, iterations) {
531 assert.equal(run['run_iteration_count_cache'], 1);
532 assert.equal(run['run_mean_cache'], 3);
533 assert.equal(run['run_sum_cache'], 3);
534 assert.equal(run['run_square_sum_cache'], 9);
535 fetchTestRunIterationsForMetric('suite', 'Combined', function (run, iterations) {
536 assert.equal(run['run_iteration_count_cache'], 1);
537 assert.equal(run['run_mean_cache'], 5);
538 assert.equal(run['run_sum_cache'], 5);
539 assert.equal(run['run_square_sum_cache'], 25);
547 var reportWithSameValuePairs = [{
548 "buildNumber": "123",
549 "buildTime": "2013-02-28T10:12:03.388304",
550 "builderName": "someBuilder",
551 "builderPassword": "somePassword",
552 "platform": "Mountain Lion",
555 "metrics": {"FrameRate": { "current": [[[0, 4], [100, 5], [205, 3]]] }}
560 it("should be able to add a report with (relative time, value) pairs", function () {
561 addBuilderAndMeanAggregators(reportWithSameValuePairs, function () {
562 postJSON('/api/report/', reportWithSameValuePairs, function (response) {
563 assert.equal(response.statusCode, 200);
565 JSON.parse(response.responseText);
567 assert.fail(error, null, response.responseText);
569 assert.equal(JSON.parse(response.responseText)['status'], 'OK');
570 assert.equal(JSON.parse(response.responseText)['failureStored'], false);
571 fetchTestRunIterationsForMetric('test', 'FrameRate', function (run, iterations) {
572 assert.equal(run['run_iteration_count_cache'], 3);
573 assert.equal(run['run_mean_cache'], 4);
574 assert.equal(run['run_sum_cache'], 12);
575 assert.equal(run['run_square_sum_cache'], 16 + 25 + 9);
576 var runId = run['run_id'];
577 assert.deepEqual(iterations, [
578 {iteration_run: runId, iteration_order: 0, iteration_group: null, iteration_value: 4, iteration_relative_time: 0},
579 {iteration_run: runId, iteration_order: 1, iteration_group: null, iteration_value: 5, iteration_relative_time: 100},
580 {iteration_run: runId, iteration_order: 2, iteration_group: null, iteration_value: 3, iteration_relative_time: 205}]);