https://bugs.webkit.org/show_bug.cgi?id=123797
Reviewed by Alexey Proskuryakov.
* config.json: Add defaultBuildWaitInterval used by process-builds.php.
* init-database.sql: Add is_process column to builds table so that we can track of "unprocessed" builds.
* public/admin/process-builds.php: Added.
(process_latest_five_builds): Update flakiness states and generate JSONs for the latest five builds.
We go backwards in the time so that we don't end up infinite looping over a single build that fails.
(main): Call processed_builds with an exponential back off.
* public/api/report.php:
(store_results): No need to return build or builder ids.
(main): Don't update flakiness states or generate JSONs.
* public/include/test-results.php:
(add_build): Take the slave name; it doesn't make any sense to create a build without it and later
updating it in store_test_results.
(store_test_results): Once new test results are added, explicitly mark this build as unprocessed.
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@158814
268f45cc-cd09-0410-ab3c-
d52691b4dbfc
+2013-11-06 Ryosuke Niwa <rniwa@webkit.org>
+
+ New flakiness dashboard should generate JSON in the background process
+ https://bugs.webkit.org/show_bug.cgi?id=123797
+
+ Reviewed by Alexey Proskuryakov.
+
+ * config.json: Add defaultBuildWaitInterval used by process-builds.php.
+ * init-database.sql: Add is_process column to builds table so that we can track of "unprocessed" builds.
+
+ * public/admin/process-builds.php: Added.
+ (process_latest_five_builds): Update flakiness states and generate JSONs for the latest five builds.
+ We go backwards in the time so that we don't end up infinite looping over a single build that fails.
+ (main): Call processed_builds with an exponential back off.
+
+ * public/api/report.php:
+ (store_results): No need to return build or builder ids.
+ (main): Don't update flakiness states or generate JSONs.
+
+ * public/include/test-results.php:
+ (add_build): Take the slave name; it doesn't make any sense to create a build without it and later
+ updating it in store_test_results.
+ (store_test_results): Once new test results are added, explicitly mark this build as unprocessed.
+
2013-11-04 Ryosuke Niwa <rniwa@webkit.org>
Build fix. Update is_flaky when it's NULL.
{
"debug": true,
"jsonCacheMaxAge": 600,
+ "defaultBuildWaitInterval": 10,
"dataDirectory": "../data",
"database": {
"host": "localhost",
start_time timestamp,
end_time timestamp,
slave integer REFERENCES slaves ON DELETE CASCADE,
+ is_processed boolean,
CONSTRAINT builder_and_build_number_must_be_unique UNIQUE(builder, number));
CREATE INDEX build_builder_index ON builds(builder);
CREATE INDEX build_slave_index ON builds(slave);
--- /dev/null
+<?php
+
+require_once('../include/test-results.php');
+
+ignore_user_abort(true);
+set_time_limit(0);
+
+function process_latest_five_builds($db) {
+ $build_rows = $db->query_and_fetch_all('SELECT id, builder FROM builds
+ WHERE start_time IS NOT NULL AND is_processed = FALSE ORDER BY end_time DESC LIMIT 5');
+ if (!$build_rows)
+ return FALSE;
+
+ foreach ($build_rows as $row) {
+ echo "Build {$row['id']} for builder {$row['builder']}:\n";
+ echo " Updating flakiness...";
+ flush();
+
+ $start_time = microtime(true);
+ update_flakiness_after_inserting_build($db, $row['id']);
+ $time = microtime(true) - $start_time;
+
+ echo "($time s)\n";
+ echo " Generating JSONs...";
+ flush();
+
+ $start_time = microtime(true);
+ $generator = new ResultsJSONGenerator($db, $row['builder']);
+ $generator->generate('wrongexpectations');
+ $generator->generate('flaky');
+ $time = microtime(true) - $start_time;
+
+ echo "($time s)\n";
+ flush();
+
+ $db->query_and_get_affected_rows('UPDATE builds SET is_processed = TRUE where id = $1', array($row['id']));
+
+ sleep(1);
+ }
+
+ return TRUE;
+}
+
+function main() {
+ $db = new Database;
+ if (!$db->connect()) {
+ echo "Failed to connect to the database";
+ exit(1);
+ }
+
+ $wait = config('defaultBuildWaitInterval');
+ while (1) {
+ if (process_latest_five_builds($db))
+ $wait = max(1, $wait * 0.8);
+ else
+ $wait *= 2;
+ echo "Sleeping $wait s...\n";
+ sleep($wait);
+ }
+}
+
+main();
+
+?>
if (!$builder_id)
exit_with_error('FailedToInsertBuilder', array('master' => $master, 'builderName' => $builder_name));
- $build_id = add_build($db, $builder_id, $build_number);
+ $slave_id = add_slave($db, $_POST['build_slave']);
+ $build_id = add_build($db, $builder_id, $build_number, $slave_id);
if (!$build_id)
exit_with_error('FailedToInsertBuild', array('builderId' => $builder_id, 'buildNumber' => $build_number));
or exit_with_error('FailedToInsertRevision', array('name' => $repository_name, 'data' => $revision_data));
}
- $slave_id = add_slave($db, $_POST['build_slave']);
- if (!store_test_results($db, $test_results, $build_id, $start_time, $end_time, $slave_id))
+ if (!store_test_results($db, $test_results, $build_id, $start_time, $end_time))
exit_with_error('FailedToStoreResults', array('buildId' => $build_id));
-
- return array('build_id' => $build_id, 'builder_id' => $builder_id);
}
function main() {
$json_path = $_FILES['file']['tmp_name'];
$db = connect();
- $builder_and_build = store_results($db, $master, $builder_name, $build_number, $start_time, $end_time, $revisions, $json_path);
- @ob_end_clean();
- ignore_user_abort();
- ob_start();
-
+ store_results($db, $master, $builder_name, $build_number, $start_time, $end_time, $revisions, $json_path);
echo_success();
-
- header('Connection: close');
- header('Content-Length: ' . ob_get_length());
-
- @ob_end_flush();
- flush();
- if (function_exists('fastcgi_finish_request'))
- fastcgi_finish_request();
-
- update_flakiness_after_inserting_build($db, $builder_and_build['build_id']);
-
- $generator = new ResultsJSONGenerator($db, $builder_and_build['builder_id']);
- $generator->generate('wrongexpectations');
- $generator->generate('flaky');
}
main();
return $db->select_or_insert_row('builders', NULL, array('master' => $master, 'name' => $builder_name));
}
-function add_build($db, $builder_id, $build_number) {
- return $db->select_or_insert_row('builds', NULL, array('builder' => $builder_id, 'number' => $build_number));
+function add_build($db, $builder_id, $build_number, $slave_id) {
+ return $db->select_or_insert_row('builds', NULL, array('builder' => $builder_id, 'number' => $build_number, 'slave' => $slave_id));
}
function add_slave($db, $name) {
return json_decode($json_contents, true);
}
-function store_test_results($db, $test_results, $build_id, $start_time, $end_time, $slave_id) {
+function store_test_results($db, $test_results, $build_id, $start_time, $end_time) {
$db->begin_transaction();
try {
recursively_add_test_results($db, $build_id, $test_results['tests'], '');
$db->query_and_get_affected_rows(
- 'UPDATE builds SET (start_time, end_time, slave) = (least($1, start_time), greatest($2, end_time), $3) WHERE id = $4',
- array($start_time->format('Y-m-d H:i:s.u'), $end_time->format('Y-m-d H:i:s.u'), $slave_id, $build_id));
+ 'UPDATE builds SET (start_time, end_time, is_processed) = (least($1, start_time), greatest($2, end_time), FALSE) WHERE id = $3',
+ array($start_time->format('Y-m-d H:i:s.u'), $end_time->format('Y-m-d H:i:s.u'), $build_id));
$db->commit_transaction();
} catch (Exception $e) {
$db->rollback_transaction();
git://git.webkit.org / WebKit-https.git / commitdiff