New flakiness dashboard should generate JSON in the background process

https://bugs.webkit.org/show_bug.cgi?id=123797

Reviewed by Alexey Proskuryakov.

* config.json: Add defaultBuildWaitInterval used by process-builds.php.
* init-database.sql: Add is_process column to builds table so that we can track of "unprocessed" builds.

* public/admin/process-builds.php: Added.
(process_latest_five_builds): Update flakiness states and generate JSONs for the latest five builds.
We go backwards in the time so that we don't end up infinite looping over a single build that fails.
(main): Call processed_builds with an exponential back off.

* public/api/report.php:
(store_results): No need to return build or builder ids.
(main): Don't update flakiness states or generate JSONs.

* public/include/test-results.php:
(add_build): Take the slave name; it doesn't make any sense to create a build without it and later
updating it in store_test_results.
(store_test_results): Once new test results are added, explicitly mark this build as unprocessed.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@158814 268f45cc-cd09-0410-ab3c-d52691b4dbfc

diff --git a/Websites/test-results/ChangeLog b/Websites/test-results/ChangeLog
index 88912d5..86fe51b 100644 (file)
--- a/Websites/test-results/ChangeLog
+++ b/Websites/test-results/ChangeLog
@@ -1,3 +1,27 @@
+2013-11-06  Ryosuke Niwa  <rniwa@webkit.org>
+
+        New flakiness dashboard should generate JSON in the background process
+        https://bugs.webkit.org/show_bug.cgi?id=123797
+
+        Reviewed by Alexey Proskuryakov.
+
+        * config.json: Add defaultBuildWaitInterval used by process-builds.php.
+        * init-database.sql: Add is_process column to builds table so that we can track of "unprocessed" builds.
+
+        * public/admin/process-builds.php: Added.
+        (process_latest_five_builds): Update flakiness states and generate JSONs for the latest five builds.
+        We go backwards in the time so that we don't end up infinite looping over a single build that fails.
+        (main): Call processed_builds with an exponential back off.
+
+        * public/api/report.php:
+        (store_results): No need to return build or builder ids.
+        (main): Don't update flakiness states or generate JSONs.
+
+        * public/include/test-results.php:
+        (add_build): Take the slave name; it doesn't make any sense to create a build without it and later
+        updating it in store_test_results.
+        (store_test_results): Once new test results are added, explicitly mark this build as unprocessed.
+

 2013-11-04  Ryosuke Niwa  <rniwa@webkit.org>
 
         Build fix. Update is_flaky when it's NULL.
 2013-11-04  Ryosuke Niwa  <rniwa@webkit.org>
 
         Build fix. Update is_flaky when it's NULL.

diff --git a/Websites/test-results/config.json b/Websites/test-results/config.json
index 7ba4c50..5e68270 100644 (file)
--- a/Websites/test-results/config.json
+++ b/Websites/test-results/config.json
@@ -1,6 +1,7 @@
 {
     "debug": true,
     "jsonCacheMaxAge": 600,
 {
     "debug": true,
     "jsonCacheMaxAge": 600,

+    "defaultBuildWaitInterval": 10,

     "dataDirectory": "../data",
     "database": {
         "host": "localhost",
     "dataDirectory": "../data",
     "database": {
         "host": "localhost",

diff --git a/Websites/test-results/init-database.sql b/Websites/test-results/init-database.sql
index 2c2e123..0ff6e12 100644 (file)
--- a/Websites/test-results/init-database.sql
+++ b/Websites/test-results/init-database.sql
@@ -29,6 +29,7 @@ CREATE TABLE builds (
     start_time timestamp,
     end_time timestamp,
     slave integer REFERENCES slaves ON DELETE CASCADE,
     start_time timestamp,
     end_time timestamp,
     slave integer REFERENCES slaves ON DELETE CASCADE,

+    is_processed boolean,

     CONSTRAINT builder_and_build_number_must_be_unique UNIQUE(builder, number));
 CREATE INDEX build_builder_index ON builds(builder);
 CREATE INDEX build_slave_index ON builds(slave);
     CONSTRAINT builder_and_build_number_must_be_unique UNIQUE(builder, number));
 CREATE INDEX build_builder_index ON builds(builder);
 CREATE INDEX build_slave_index ON builds(slave);

diff --git a/Websites/test-results/public/admin/process-builds.php b/Websites/test-results/public/admin/process-builds.php
new file mode 100644 (file)
index 0000000..2aa0286
--- /dev/null
+++ b/Websites/test-results/public/admin/process-builds.php
@@ -0,0 +1,64 @@
+<?php
+
+require_once('../include/test-results.php');
+
+ignore_user_abort(true); 
+set_time_limit(0);
+
+function process_latest_five_builds($db) {
+    $build_rows = $db->query_and_fetch_all('SELECT id, builder FROM builds
+        WHERE start_time IS NOT NULL AND is_processed = FALSE ORDER BY end_time DESC LIMIT 5');
+    if (!$build_rows)
+        return FALSE;
+
+    foreach ($build_rows as $row) {
+        echo "Build {$row['id']} for builder {$row['builder']}:\n";
+        echo "    Updating flakiness...";
+        flush();
+
+        $start_time = microtime(true);
+        update_flakiness_after_inserting_build($db, $row['id']);
+        $time = microtime(true) - $start_time;
+
+        echo "($time s)\n";
+        echo "    Generating JSONs...";
+        flush();
+
+        $start_time = microtime(true);
+        $generator = new ResultsJSONGenerator($db, $row['builder']);
+        $generator->generate('wrongexpectations');
+        $generator->generate('flaky');
+        $time = microtime(true) - $start_time;
+
+        echo "($time s)\n";
+        flush();
+
+        $db->query_and_get_affected_rows('UPDATE builds SET is_processed = TRUE where id = $1', array($row['id']));
+
+        sleep(1);
+    }
+
+    return TRUE;
+}
+
+function main() {
+    $db = new Database;
+    if (!$db->connect()) {
+        echo "Failed to connect to the database";
+        exit(1);
+    }
+
+    $wait = config('defaultBuildWaitInterval');
+    while (1) {
+        if (process_latest_five_builds($db))
+            $wait = max(1, $wait * 0.8);
+        else
+            $wait *= 2;
+        echo "Sleeping $wait s...\n";
+        sleep($wait);
+    }
+}
+
+main();
+
+?>

diff --git a/Websites/test-results/public/api/report.php b/Websites/test-results/public/api/report.php
index c52daa2..178c13c 100644 (file)
--- a/Websites/test-results/public/api/report.php
+++ b/Websites/test-results/public/api/report.php
@@ -12,7 +12,8 @@ function store_results($db, $master, $builder_name, $build_number, $start_time,
     if (!$builder_id)
         exit_with_error('FailedToInsertBuilder', array('master' => $master, 'builderName' => $builder_name));
 
     if (!$builder_id)
         exit_with_error('FailedToInsertBuilder', array('master' => $master, 'builderName' => $builder_name));
 

-    $build_id = add_build($db, $builder_id, $build_number);
+    $slave_id = add_slave($db, $_POST['build_slave']);
+    $build_id = add_build($db, $builder_id, $build_number, $slave_id);

     if (!$build_id)
         exit_with_error('FailedToInsertBuild', array('builderId' => $builder_id, 'buildNumber' => $build_number));
 
     if (!$build_id)
         exit_with_error('FailedToInsertBuild', array('builderId' => $builder_id, 'buildNumber' => $build_number));
 


@@ -30,11 +31,8 @@ function store_results($db, $master, $builder_name, $build_number, $start_time,
             or exit_with_error('FailedToInsertRevision', array('name' => $repository_name, 'data' => $revision_data));
     }
 
             or exit_with_error('FailedToInsertRevision', array('name' => $repository_name, 'data' => $revision_data));
     }
 

-    $slave_id = add_slave($db, $_POST['build_slave']);
-    if (!store_test_results($db, $test_results, $build_id, $start_time, $end_time, $slave_id))
+    if (!store_test_results($db, $test_results, $build_id, $start_time, $end_time))

         exit_with_error('FailedToStoreResults', array('buildId' => $build_id));
         exit_with_error('FailedToStoreResults', array('buildId' => $build_id));

-
-    return array('build_id' => $build_id, 'builder_id' => $builder_id);

 }
 
 function main() {
 }
 
 function main() {


@@ -66,26 +64,8 @@ function main() {
     $json_path = $_FILES['file']['tmp_name'];
 
     $db = connect();
     $json_path = $_FILES['file']['tmp_name'];
 
     $db = connect();

-    $builder_and_build = store_results($db, $master, $builder_name, $build_number, $start_time, $end_time, $revisions, $json_path);
-    @ob_end_clean();
-    ignore_user_abort();
-    ob_start();
-
+    store_results($db, $master, $builder_name, $build_number, $start_time, $end_time, $revisions, $json_path);

     echo_success();
     echo_success();

-
-    header('Connection: close');
-    header('Content-Length: ' . ob_get_length());
-
-    @ob_end_flush();
-    flush();
-    if (function_exists('fastcgi_finish_request'))
-        fastcgi_finish_request();
-
-    update_flakiness_after_inserting_build($db, $builder_and_build['build_id']);
-
-    $generator = new ResultsJSONGenerator($db, $builder_and_build['builder_id']);
-    $generator->generate('wrongexpectations');
-    $generator->generate('flaky');

 }
 
 main();
 }
 
 main();

diff --git a/Websites/test-results/public/include/test-results.php b/Websites/test-results/public/include/test-results.php
index 4f04173..5100603 100644 (file)
--- a/Websites/test-results/public/include/test-results.php
+++ b/Websites/test-results/public/include/test-results.php
@@ -15,8 +15,8 @@ function add_builder($db, $master, $builder_name) {
     return $db->select_or_insert_row('builders', NULL, array('master' => $master, 'name' => $builder_name));
 }
 
     return $db->select_or_insert_row('builders', NULL, array('master' => $master, 'name' => $builder_name));
 }
 

-function add_build($db, $builder_id, $build_number) {
-    return $db->select_or_insert_row('builds', NULL, array('builder' => $builder_id, 'number' => $build_number));
+function add_build($db, $builder_id, $build_number, $slave_id) {
+    return $db->select_or_insert_row('builds', NULL, array('builder' => $builder_id, 'number' => $build_number, 'slave' => $slave_id));

 }
 
 function add_slave($db, $name) {
 }
 
 function add_slave($db, $name) {


@@ -34,15 +34,15 @@ function fetch_and_parse_test_results_json($url, $jsonp = FALSE) {
     return json_decode($json_contents, true);
 }
 
     return json_decode($json_contents, true);
 }
 

-function store_test_results($db, $test_results, $build_id, $start_time, $end_time, $slave_id) {
+function store_test_results($db, $test_results, $build_id, $start_time, $end_time) {

     $db->begin_transaction();
 
     try {
         recursively_add_test_results($db, $build_id, $test_results['tests'], '');
 
         $db->query_and_get_affected_rows(
     $db->begin_transaction();
 
     try {
         recursively_add_test_results($db, $build_id, $test_results['tests'], '');
 
         $db->query_and_get_affected_rows(

-            'UPDATE builds SET (start_time, end_time, slave) = (least($1, start_time), greatest($2, end_time), $3) WHERE id = $4',
-            array($start_time->format('Y-m-d H:i:s.u'), $end_time->format('Y-m-d H:i:s.u'), $slave_id, $build_id));
+            'UPDATE builds SET (start_time, end_time, is_processed) = (least($1, start_time), greatest($2, end_time), FALSE) WHERE id = $3',
+            array($start_time->format('Y-m-d H:i:s.u'), $end_time->format('Y-m-d H:i:s.u'), $build_id));

         $db->commit_transaction();
     } catch (Exception $e) {
         $db->rollback_transaction();
         $db->commit_transaction();
     } catch (Exception $e) {
         $db->rollback_transaction();