+2013-10-21 Ryosuke Niwa <rniwa@webkit.org>
+
+ New flakiness dashboard show test time, modifiers, and flaky tests
+ https://bugs.webkit.org/show_bug.cgi?id=123119
+
+ Reviewed by Tim Horton.
+
+ * api/failing-tests.php: Manually serialize each row in the results to avoid hitting the memory limit.
+ * include/db.php:
+ (Database::query): Added.
+ (Database::fetch_next_row): Added.
+ * include/init-database.sql: Added modifiers and time columns to results table.
+ * include/test-results.php:
+ (store_test_results): Update start_time and end_time to the union of the new interval and the existing interval.
+ (recursively_add_test_results): Handle empty $full_name to eliminate the loop over tests in store_test_results.
+ Also verify that each test name, expected and actual results conform to the specific format to prevent XSS.
+ Also use insert_row instead of select_or_insert_row to avoid issuing an unnecessary SQL query.
+ (format_result): Extracted from format_result_rows. Used in failing-tests.php.
+ * index.html:
+ (TestResultsView): Added _currentBuilderFailureType and _currentBuilderDays.
+ (TestResultsView._createResultCell): Show the test time and the expected result.
+ (TestResultsView._createTestResultRow): Compute the slowest run and also round time to tenth of second for time
+ less than 10s or second if it's more than 10s so that the test time will always be shown in two digits.
+ Also show the bug number and the latest expected result on the left columns after linkifying the bug numbers.
+ (TestResultsView._matchesFailureType): Added. Determines whether results is of a particular failure type.
+ (TestResultsView._populateBuilderPane):
+ (TestResultsView.fetchFailingTestsForBuilder): Store the failure type such as flaky, wrongtestexpectations.
+ (TestResultsView.updateLocationHash):
+ (TestResultsView.loadTestsFromLocationHash):
+ (fetchManifest):
+ * js/dom.js:
+ (element): appendChild if an item is a Node. Otherwise, e.g. integer, create a text node out of toString() call.
+ * main.css: Updated styles.
+
2013-10-18 Ryosuke Niwa <rniwa@webkit.org>
New flakiness dashboard should support showing the failing tests per builder
exit_with_error('BuilderNotFound');
$builder_id = $builder_row['id'];
-$result_rows = $db->query_and_fetch_all(
-'SELECT results.*, builds.*, array_agg((build_revisions.repository, build_revisions.value, build_revisions.time)) AS revisions
+$all_results = $db->query(
+"SELECT results.*, builds.*, array_agg((build_revisions.repository, build_revisions.value, build_revisions.time)) AS revisions
FROM results, builds, build_revisions
WHERE build_revisions.build = builds.id AND results.build = builds.id AND builds.builder = $1
- AND results.actual != $2 AND builds.start_time > now() - interval \'' . $number_of_days . ' days\'
- GROUP BY results.id, builds.id', array($builder_id, 'PASS'));
-if (!$result_rows)
+ AND builds.start_time > now() - interval '$number_of_days days'
+ GROUP BY results.id, builds.id ORDER BY results.test, max(build_revisions.time) DESC", array($builder_id));
+
+if (!$all_results)
exit_with_error('ResultsNotFound');
-exit_with_success(format_result_rows($result_rows));
+// To conserve memory, we serialize tests at a time.
+echo "{\"status\": \"OK\", \"builders\": {\"$builder_id\":{";
+$currentTest = NULL;
+$i = 0;
+while ($result = $db->fetch_next_row($all_results)) {
+ if ($result['test'] != $currentTest) {
+ if ($currentTest)
+ echo '],';
+ $currentTest = $result['test'];
+ echo "\"$currentTest\": [";
+ } else
+ echo ',';
+ echo json_encode(format_result($result), true);
+}
+if ($currentTest)
+ echo ']';
+echo '}}}';
?>
return pg_fetch_all($result);
}
+ function query($query, $params = array()) {
+ if (!$this->connection)
+ return FALSE;
+ return pg_query_params($this->connection, $query, $params);
+ }
+
+ function fetch_next_row($result) {
+ return pg_fetch_assoc($result);
+ }
+
function fetch_table($table_name, $column_to_be_ordered_by = null) {
if (!$this->connection || !ctype_alnum_underscore($table_name) || ($column_to_be_ordered_by && !ctype_alnum_underscore($column_to_be_ordered_by)))
return false;
DROP TABLE results CASCADE;
DROP TABLE tests CASCADE;
-DROP TABLE build_revision_map CASCADE;
-DROP TABLE revisions CASCADE;
+DROP TABLE build_revisions CASCADE;
DROP TABLE builds CASCADE;
DROP TABLE slaves CASCADE;
+DROP TABLE repositories CASCADE;
DROP TABLE builders CASCADE;
CREATE TABLE builders (
start_time timestamp,
end_time timestamp,
slave integer REFERENCES slaves ON DELETE CASCADE,
- fetched bool NOT NULL DEFAULT FALSE,
CONSTRAINT builder_and_build_number_must_be_unique UNIQUE(builder, number));
CREATE INDEX build_builder_index ON builds(builder);
CREATE INDEX build_slave_index ON builds(slave);
test integer REFERENCES tests ON DELETE CASCADE,
build integer REFERENCES builds ON DELETE CASCADE,
expected varchar(64) NOT NULL,
- actual varchar(64) NOT NULL);
+ actual varchar(64) NOT NULL,
+ modifiers varchar(64) NOT NULL,
+ time integer);
CREATE INDEX results_test ON results(test);
CREATE INDEX results_build ON results(build);
$db->begin_transaction();
try {
- foreach ($test_results['tests'] as $name => $subtests)
- recursively_add_test_results($db, $build_id, $subtests, $name);
- $db->query_and_get_affected_rows('UPDATE builds SET (start_time, end_time, slave, fetched) = ($1, $2, $3, TRUE) WHERE id = $4',
+ recursively_add_test_results($db, $build_id, $test_results['tests'], '');
+
+ $db->query_and_get_affected_rows(
+ 'UPDATE builds SET (start_time, end_time, slave) = (least($1, start_time), greatest($2, end_time), $3) WHERE id = $4',
array($start_time->format('Y-m-d H:i:s.u'), $end_time->format('Y-m-d H:i:s.u'), $slave_id, $build_id));
$db->commit_transaction();
} catch (Exception $e) {
function recursively_add_test_results($db, $build_id, $tests, $full_name) {
if (!array_key_exists('expected', $tests) and !array_key_exists('actual', $tests)) {
- foreach ($tests as $name => $subtests)
- recursively_add_test_results($db, $build_id, $subtests, $full_name . '/' . $name);
+ $prefix = $full_name ? $full_name . '/' : '';
+ foreach ($tests as $name => $subtests) {
+ require_format('test_name', $name, '/^[A-Za-z0-9 +_\-\.]+$/');
+ recursively_add_test_results($db, $build_id, $subtests, $prefix . $name);
+ }
return;
}
+ require_format('expected_result', $tests['expected'], '/^[A-Za-z ]+$/');
+ require_format('actual_result', $tests['actual'], '/^[A-Za-z ]+$/');
+ require_format('test_time', $tests['time'], '/^\d*$/');
+ $modifiers = array_get($tests, 'modifiers');
+ if ($modifiers)
+ require_format('test_modifiers', $modifiers, '/^[A-Za-z0-9 \.\/]+$/');
+ else
+ $modifiers = NULL;
+
$test_id = $db->select_or_insert_row('tests', NULL,
array('name' => $full_name), array('name' => $full_name, 'reftest_type' => json_encode(array_get($tests, 'reftest_type'))));
- // FIXME: Either use a transaction or check the return value.
- $db->select_or_insert_row('results', NULL, array('test' => $test_id, 'build' => $build_id,
- 'expected' => $tests['expected'], 'actual' => $tests['actual']));
+ $db->insert_row('results', NULL, array('test' => $test_id, 'build' => $build_id,
+ 'expected' => $tests['expected'], 'actual' => $tests['actual'],
+ 'time' => $tests['time'], 'modifiers' => $tests['modifiers']));
}
date_default_timezone_set('UTC');
return $revisions;
}
+function format_result($result) {
+ return array('buildTime' => strtotime($result['start_time']) * 1000,
+ 'revisions' => parse_revisions_array($result['revisions']),
+ 'slave' => $result['slave'],
+ 'buildNumber' => $result['number'],
+ 'actual' => $result['actual'],
+ 'expected' => $result['expected'],
+ 'time' => $result['time'],
+ 'modifiers' => $result['modifiers']);
+}
+
function format_result_rows($result_rows) {
$builders = array();
foreach ($result_rows as $result) {
array_push(array_ensure_item_has_array(array_ensure_item_has_array($builders, $result['builder']), $result['test']),
- array('buildTime' => strtotime($result['start_time']) * 1000,
- 'revisions' => parse_revisions_array($result['revisions']),
- 'slave' => $result['slave'],
- 'buildNumber' => $result['number'],
- 'actual' => $result['actual'],
- 'expected' => $result['expected']));
+ format_result($result));
}
return array('builders' => $builders);
}
<div id="testView"></div>
<form>
-<label>Show failing tests for</label>
-<select id="builderListView"><option value="">Select builder</option></select>
-<select id="builderDaysView"><option>5</option><option>15</option><option>30</option></select>
+Show
+<label>tests <select id="builderFailureTypeView"><option>failing</option><option>flaky</option><option value="wrongexpectations">has wrong expectations</option></select></label>
+<label>on <select id="builderListView"><option value="">Select builder</option></select></label>
+<label for="builderDaysView">in the last <select id="builderDaysView"><option>5</option><option>15</option><option>30</option></select> days</label>
</form>
<div id="builderFailingTestsView"></div>
this._tooltipContainer = tooltipContainer;
this._tests = [];
this._currentBuilder = null;
+ this._currentBuilderFailureType = null;
+ this._currentBuilderDays = null;
this._oldHash = null;
this._builders = [];
this._slaves = [];
var build = result['buildNumber'];
var actual = result['actual'];
var expected = result['expected'];
- var anchor = element('a', {'href': this._urlFromBuilder('result', master, builder, revision, build)});
+ var timeIfSlow = result.isSlow ? result.roundedTime : '-';
+ var anchor = element('a', {'href': this._urlFromBuilder('result', master, builder, revision, build)}, [timeIfSlow]);
anchor.onmouseenter = function () {
var repositoryById = TestResultsView._repositories;
var formattedRevisions = result.build.formattedRevisions(previousResult ? previousResult.build : null);
element('li', ['Build Time: ' + result.build.formattedBuildTime()]),
element('li', ['Revision: ' + revisionDescription]),
element('li', ['Build: ', element('a', {'href': TestResultsView._urlFromBuilder('build', master, builder, revision, build)}, [build])]),
- element('li', ['Result: ' + actual]),
+ element('li', ['Actual: ' + actual]),
+ element('li', ['Expected: ' + expected]),
])
]));
}
}
TestResultsView._createTestResultRow = function (title, results, builder) {
- for (var i = 0; i < results.length; i++)
- results[i].build = new TestBuild(this._repositories, this._builders, results[i]);
+ var slowestTime = 0;
+ for (var i = 0; i < results.length; i++) {
+ var result = results[i];
+ result.build = new TestBuild(this._repositories, this._builders, result);
+ result.roundedTime = result.time > 10000 ? Math.round(result.time / 1000) : Math.round(result.time / 100) / 10;
+ result.isSlow = result.time > 1000;
+ if (result.isSlow)
+ slowestTime = Math.max(slowestTime, result.roundedTime);
+ }
+ if (slowestTime)
+ slowestTime += 's';
+ else
+ slowestTime = '';
- var sortedResults = results.sort(function (result1, result2) { return result1.build.time() - result2.build.time(); });
+ var sortedResults = results.sort(function (result1, result2) { return result2.build.time() - result1.build.time(); });
var cells = new Array(sortedResults.length);
for (var i = 0; i < sortedResults.length; i++)
cells[i] = this._createResultCell(builder.master, builder.name, sortedResults[i], sortedResults[i - 1]);
- var passCount = cells.filter(function (cell) { return cell.className == 'PASS'; }).length;
- var passingRate = Math.round(passCount / cells.length * 100) + '%';
- return element('tr', [element('th', ['' + title]), element('td', {'class': 'passingRate'}, [passingRate]), element('td', cells)]);
+ var formattedModifiers = sortedResults[0].modifiers.split(' ').map(function (modifier) {
+ if (modifier.indexOf('/') > 0)
+ return element('a', {'href': (modifier.indexOf('http') == 0 ? '' : 'http://') + modifier}, [modifier]);
+ return modifier;
+ });
+
+ return element('tr', [
+ element('th', ['' + title]),
+ element('td', {'class': 'modifiers'}, formattedModifiers),
+ element('td', {'class': 'expected'}, [sortedResults[0].expected]),
+ element('td', {'class': 'slowestTime'}, [slowestTime]),
+ element('td', cells)]);
}
TestResultsView.fetchTest = function (testName) {
this.updateLocationHash();
}
-TestResultsView._populateBuilderPane = function(builderName, results, section) {
+TestResultsView._matchesFailureType = function (results, failureType, tn) {
+ if (!results.length)
+ return false;
+ var latestActualResult = results[0].actual;
+ var latestExpectedResult = results[0].expected;
+ switch (failureType) {
+ case 'failing':
+ return results[0].actual != 'PASS';
+ case 'flaky':
+ var offOneChangeCount = 0;
+ for (var i = 1; i + 1 < results.length; i++) {
+ var previousActual = results[i - 1].actual;
+ var nextActual = results[i + 1].actual;
+ if (previousActual == nextActual && results[i].actual != previousActual)
+ offOneChangeCount++;
+ }
+ return offOneChangeCount; // Heuristics.
+ case 'wrongexpectations':
+ if (latestExpectedResult == latestActualResult)
+ return false;
+ var expectedTokens = latestExpectedResult.split(' ');
+ if (expectedTokens.indexOf(latestActualResult) >= 0)
+ return false;
+ if (latestActualResult == 'TEXT' || latestActualResult == 'TEXT+IMAGE' && expectedTokens.indexOf('FAIL') >= 0)
+ return false;
+ return true;
+ }
+
+ return false;
+}
+
+TestResultsView._populateBuilderPane = function(builderName, failureType, results, section) {
var table = element('table', {'class': 'resultsTable'}, [element('caption', [builderName])]);
var resultsByBuilder = results['builders'];
var resultsByTests;
return;
var builder = this._builders[builderId];
- for (var testId in resultsByTests)
+ for (var testId in resultsByTests) {
+ var results = resultsByTests[testId];
+ if (!results.length || !this._matchesFailureType(results, failureType, this._availableTests[testId].name))
+ continue;
table.appendChild(this._createTestResultRow(this._availableTests[testId].name, resultsByTests[testId], builder));
+ }
section.appendChild(table);
}
-TestResultsView.fetchFailingTestsForBuilder = function (builderName, numberOfDays, doNotUpdateHash) {
+TestResultsView.fetchFailingTestsForBuilder = function (builderName, numberOfDays, failureType, doNotUpdateHash) {
var section = element('section', {'class': 'testResults'}, ['Loading...']);
var container = document.getElementById('builderFailingTestsView');
var self = this;
var xhr = new XMLHttpRequest();
- xhr.open("GET", 'api/failing-tests.php?builder=' + escape(builderName) + '&' + 'days=' + numberOfDays, true);
+ xhr.open("GET", 'api/failing-tests.php?builder=' + escape(builderName) + '&days=' + numberOfDays, true);
xhr.onload = function(event) {
var response = JSON.parse(xhr.response);
section.innerHTML = '';
section.appendChild(text('Failed to load results for ' + builderName + ': ' + response['status']));
return;
}
+ // FIXME: Normalize failureType.
self._currentBuilder = builderName;
+ self._currentBuilderFailureType = failureType;
self._currentBuilderDays = numberOfDays;
- self._populateBuilderPane(builderName, response, section);
+ self._populateBuilderPane(builderName, failureType, response, section);
if (!doNotUpdateHash)
self.updateLocationHash();
}
var params = {
'tests': this._tests.join(','),
'builder': this._currentBuilder,
+ 'builderFailureType': this._currentBuilderFailureType,
'builderDays': this._currentBuilderDays,
};
var hash = '';
document.getElementById('testView').innerHTML = '';
this.fetchTests(parsed['tests'].split(','), doNotUpdateHash);
}
- if (parsed['builder'])
- this.fetchFailingTestsForBuilder(parsed['builder'], parsed['builderDays'] || 5, doNotUpdateHash);
+ if (parsed['builder']) {
+ this.fetchFailingTestsForBuilder(
+ parsed['builder'],
+ parseInt(parsed['builderDays']) || 5,
+ parsed['builderFailureType'] || 'failing', doNotUpdateHash);
+ }
+ return parsed;
}
function fetchManifest(callback) {
builderListView.appendChild(element('option', [text(response['builders'][builderId].name)]));
var builderDaysView = document.getElementById('builderDaysView');
+ var builderFailureTypeView = document.getElementById('builderFailureTypeView');
function updateBuilderView() {
if (builderListView.value)
- TestResultsView.fetchFailingTestsForBuilder(builderListView.value, builderDaysView.value);
+ TestResultsView.fetchFailingTestsForBuilder(builderListView.value, builderDaysView.value, builderFailureTypeView.value);
}
builderListView.addEventListener('change', updateBuilderView);
builderDaysView.addEventListener('change', updateBuilderView);
+ builderFailureTypeView.addEventListener('change', updateBuilderView);
function mapById(items) {
var results = {};
TestResultsView.setBuilders(mapById(response['builders']));
TestResultsView.setSlaves(mapById(response['slaves']));
TestResultsView.setRepositories(mapById(response['repositories']));
- TestResultsView.loadTestsFromLocationHash();
+ // FIXME: Updating location.href shouldn't be TestResultsView's responsibility.
+ var parsedStates = TestResultsView.loadTestsFromLocationHash();
+ if (parsedStates['builder']) {
+ builderListView.value = parsedStates['builder'];
+ builderDaysView.value = parsedStates['builderDays'];
+ builderFailureTypeView.value = parsedStates['builderFailureType'];
+ }
});
function pasteHelper(input, event) {
if (childNodes) {
for (var i = 0; i < childNodes.length; i++) {
- if (typeof(childNodes[i]) === 'string')
- element.appendChild(document.createTextNode(childNodes[i]));
- else
+ if (childNodes[i] instanceof Node)
element.appendChild(childNodes[i]);
+ else
+ element.appendChild(document.createTextNode(childNodes[i]));
}
}
return element;
.resultsTable {
font-size: small;
border-collapse: collapse;
- border: 2px solid #fff;
+ border: 0px solid #fff;
padding: 0;
margin: 0;
+ width: 100%;
}
.resultsTable caption {
.resultsTable td,
.resultsTable th {
- border: 2px solid #fff;
- padding: 0;
+ white-space: pre;
+ border: 0px solid #fff;
+ padding: 0 0.5em;
margin: 0;
}
.resultsTable th,
.resultsTable .passingRate {
+ text-align: left;
font-weight: normal;
padding-right: 10px;
}
padding: 0.2em 0.2em;
}
+.resultsTable tr:hover,
+.resultsTable tr:hover {
+ background-color: #eee;
+}
+
.resultsTable a {
+ color: #00f;
+ text-shadow: none;
+}
+
+.resultsTable a:visited {
+ color: #006;
+}
+
+.resultsTable span a {
display: block;
- width: 1em;
- height: 1.5em;
+ width: 1.4em;
+ height: 2em;
border-radius: 3px;
+ font-size: 0.5em;
+ padding: 0.3em;
+ text-align: center;
+ line-height: 1.9em;
+ color: inherit;
}
.resultsTable .PASS a {
}
.resultsTable .IMAGE a {
- background-color: #3cf;
+ background-color: #fc3;
}
.resultsTable .TEXT.PASS a {
background-color: #f00;
}
+.resultsTable .TIMEOUT a {
+ background-color: #000;
+ color: #fff;
+}
+
.candidateWindow {
z-index: 999;
position: absolute;
git://git.webkit.org / WebKit-https.git / commitdiff