Add retry for test groups with failed build requests.
authordewei_zhu@apple.com <dewei_zhu@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 5 Oct 2018 00:17:49 +0000 (00:17 +0000)
committerdewei_zhu@apple.com <dewei_zhu@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 5 Oct 2018 00:17:49 +0000 (00:17 +0000)
https://bugs.webkit.org/show_bug.cgi?id=190188

Reviewed by Ryosuke Niwa.

Added retry logic in run-analysis script.
Current retry logic will only be triggered when there is at least one successful build request for each commit set
in a test group.

* init-database.sql: Added 'testgroup_initial_repetition_count' and 'testgroup_may_need_more_requests'.
SQL to update existing database:
'''
BEGIN;
    ALTER TABLE analysis_test_groups
        ADD COLUMN testgroup_initial_repetition_count integer DEFAULT NULL,
        ADD COLUMN testgroup_may_need_more_requests boolean DEFAULT FALSE;
    UPDATE analysis_test_groups SET testgroup_initial_repetition_count = (
        SELECT DISTINCT(COUNT(*)) FROM build_requests WHERE request_group = testgroup_id AND request_order >= 0 GROUP BY request_commit_set
    );
    ALTER TABLE analysis_test_groups ALTER COLUMN testgroup_initial_repetition_count DROP DEFAULT, ALTER COLUMN testgroup_may_need_more_requests SET NOT NULL;
END;
'''
'testgroup_initial_repetition_count' represents the number of successful build request for each commit set when
test group is created.
'testgroup_may_need_more_requests' will be set when any build request in test group is set to 'failed'.
* public/api/build-requests.php: Added the logic to set 'testgroup_may_need_more_requests'.
* public/api/test-groups.php: Updated 'ready-for-notification' to 'ready-for-further-processing' so that it returns finished test
groups those either have 'needs_notification' or  'may_need_more_requests' set.
* public/include/commit-sets-helpers.php: Set 'initial_repetition_count' to repetition count.
* public/privileged-api/update-test-group.php: Added APIs to add build request for a test group and
update 'may_need_more_requests' flag.
* public/v3/models/test-group.js:
(TestGroup): Added '_mayNeedMoreRequests' and '_initialRepetitionCount' field.
Refactored code that interacts with '/api/update-test-group'.
(TestGroup.prototype.updateSingleton):
(TestGroup.prototype.mayNeedMoreRequests):
(TestGroup.prototype.initialRepetitionCount):
(TestGroup.prototype.async._updateBuildRequest):
(TestGroup.prototype.updateName):
(TestGroup.prototype.updateHiddenFlag):
(TestGroup.prototype.async.didSendNotification):
(TestGroup.prototype.async.addMoreBuildRequests):
(TestGroup.prototype.async.clearMayNeedMoreBuildRequests): Added API to clear 'may_need_more_requests' flag.
(TestGroup.fetchAllReadyForFurtherProcessing): Refactored 'TestGroup.fetchAllWithNotificationReady' to return test groups either
have 'needs_notification' or  'may_need_more_requests' set.
(TestGroup.fetchAllThatMayNeedMoreRequests): Fetches test groups those may need more build requests.
* server-tests/api-test-groups.js: Added unit tests.
* server-tests/privileged-api-add-build-requests-tests.js: Added unit tests for 'add-build-requests' API.
* server-tests/privileged-api-update-test-group-tests.js: Added unit tests.
* server-tests/resources/mock-data.js:
(MockData.addMockData):
* server-tests/resources/test-server.js:
(TestServer.prototype._determinePgsqlDirectory): Fixed a bug that 'childProcess.execFileSync' may return a buffer.
* tools/run-analysis.js: Added logic to add extra build request before sennding notification.
* tools/js/retry-failed-build-requests.js:
(async.createAdditionalBuildRequestsForTestGroupsWithFailedRequests): Module that add extra build requests.
* unit-tests/retry-failed-build-requests-tests.js: Added.
* unit-tests/test-groups-tests.js: Added unit tests.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@236861 268f45cc-cd09-0410-ab3c-d52691b4dbfc

18 files changed:
Websites/perf.webkit.org/ChangeLog
Websites/perf.webkit.org/init-database.sql
Websites/perf.webkit.org/public/api/build-requests.php
Websites/perf.webkit.org/public/api/test-groups.php
Websites/perf.webkit.org/public/include/commit-sets-helpers.php
Websites/perf.webkit.org/public/privileged-api/add-build-requests.php [new file with mode: 0644]
Websites/perf.webkit.org/public/privileged-api/update-test-group.php
Websites/perf.webkit.org/public/v3/models/build-request.js
Websites/perf.webkit.org/public/v3/models/test-group.js
Websites/perf.webkit.org/server-tests/api-test-groups.js
Websites/perf.webkit.org/server-tests/privileged-api-add-build-requests-tests.js [new file with mode: 0644]
Websites/perf.webkit.org/server-tests/privileged-api-update-test-group-tests.js
Websites/perf.webkit.org/server-tests/resources/mock-data.js
Websites/perf.webkit.org/server-tests/resources/test-server.js
Websites/perf.webkit.org/tools/js/retry-failed-build-requests.js [new file with mode: 0644]
Websites/perf.webkit.org/tools/run-analysis.js
Websites/perf.webkit.org/unit-tests/retry-failed-build-requests-tests.js [new file with mode: 0644]
Websites/perf.webkit.org/unit-tests/test-groups-tests.js

index b76ddc7..d2af938 100644 (file)
@@ -1,3 +1,64 @@
+2018-10-01  Dewei Zhu  <dewei_zhu@apple.com>
+
+        Add retry for test groups with failed build requests.
+        https://bugs.webkit.org/show_bug.cgi?id=190188
+
+        Reviewed by Ryosuke Niwa.
+
+        Added retry logic in run-analysis script.
+        Current retry logic will only be triggered when there is at least one successful build request for each commit set
+        in a test group.
+
+        * init-database.sql: Added 'testgroup_initial_repetition_count' and 'testgroup_may_need_more_requests'.
+        SQL to update existing database:
+        '''
+        BEGIN;
+            ALTER TABLE analysis_test_groups
+                ADD COLUMN testgroup_initial_repetition_count integer DEFAULT NULL,
+                ADD COLUMN testgroup_may_need_more_requests boolean DEFAULT FALSE;
+            UPDATE analysis_test_groups SET testgroup_initial_repetition_count = (
+                SELECT DISTINCT(COUNT(*)) FROM build_requests WHERE request_group = testgroup_id AND request_order >= 0 GROUP BY request_commit_set
+            );
+            ALTER TABLE analysis_test_groups ALTER COLUMN testgroup_initial_repetition_count DROP DEFAULT, ALTER COLUMN testgroup_may_need_more_requests SET NOT NULL;
+        END;
+        '''
+        'testgroup_initial_repetition_count' represents the number of successful build request for each commit set when
+        test group is created.
+        'testgroup_may_need_more_requests' will be set when any build request in test group is set to 'failed'.
+        * public/api/build-requests.php: Added the logic to set 'testgroup_may_need_more_requests'.
+        * public/api/test-groups.php: Updated 'ready-for-notification' to 'ready-for-further-processing' so that it returns finished test
+        groups those either have 'needs_notification' or  'may_need_more_requests' set.
+        * public/include/commit-sets-helpers.php: Set 'initial_repetition_count' to repetition count.
+        * public/privileged-api/update-test-group.php: Added APIs to add build request for a test group and
+        update 'may_need_more_requests' flag.
+        * public/v3/models/test-group.js:
+        (TestGroup): Added '_mayNeedMoreRequests' and '_initialRepetitionCount' field.
+        Refactored code that interacts with '/api/update-test-group'.
+        (TestGroup.prototype.updateSingleton):
+        (TestGroup.prototype.mayNeedMoreRequests):
+        (TestGroup.prototype.initialRepetitionCount):
+        (TestGroup.prototype.async._updateBuildRequest):
+        (TestGroup.prototype.updateName):
+        (TestGroup.prototype.updateHiddenFlag):
+        (TestGroup.prototype.async.didSendNotification):
+        (TestGroup.prototype.async.addMoreBuildRequests):
+        (TestGroup.prototype.async.clearMayNeedMoreBuildRequests): Added API to clear 'may_need_more_requests' flag.
+        (TestGroup.fetchAllReadyForFurtherProcessing): Refactored 'TestGroup.fetchAllWithNotificationReady' to return test groups either
+        have 'needs_notification' or  'may_need_more_requests' set.
+        (TestGroup.fetchAllThatMayNeedMoreRequests): Fetches test groups those may need more build requests.
+        * server-tests/api-test-groups.js: Added unit tests.
+        * server-tests/privileged-api-add-build-requests-tests.js: Added unit tests for 'add-build-requests' API.
+        * server-tests/privileged-api-update-test-group-tests.js: Added unit tests.
+        * server-tests/resources/mock-data.js:
+        (MockData.addMockData):
+        * server-tests/resources/test-server.js:
+        (TestServer.prototype._determinePgsqlDirectory): Fixed a bug that 'childProcess.execFileSync' may return a buffer.
+        * tools/run-analysis.js: Added logic to add extra build request before sennding notification.
+        * tools/js/retry-failed-build-requests.js:
+        (async.createAdditionalBuildRequestsForTestGroupsWithFailedRequests): Module that add extra build requests.
+        * unit-tests/retry-failed-build-requests-tests.js: Added.
+        * unit-tests/test-groups-tests.js: Added unit tests.
+
 2018-09-21  Dewei Zhu  <dewei_zhu@apple.com>
 
         Apache can return a corrupt manifest file while ManifestGenerator::store is running
index 913240a..8161efe 100644 (file)
@@ -281,6 +281,8 @@ CREATE TABLE analysis_test_groups (
     testgroup_hidden boolean NOT NULL DEFAULT FALSE,
     testgroup_needs_notification boolean NOT NULL DEFAULT FALSE,
     testgroup_notification_sent_at timestamp DEFAULT NULL,
+    testgroup_initial_repetition_count integer NOT NULL,
+    testgroup_may_need_more_requests boolean DEFAULT FALSE,
     CONSTRAINT testgroup_name_must_be_unique_for_each_task UNIQUE(testgroup_task, testgroup_name));
 CREATE INDEX testgroup_task_index ON analysis_test_groups(testgroup_task);
 
index 5caeee6..497376d 100644 (file)
@@ -41,12 +41,13 @@ function main($id, $path, $post_data) {
 
 function update_builds($db, $updates) {
     $db->begin_transaction();
+    $test_groups_may_need_more_requests = array();
     foreach ($updates as $id => $info) {
         $id = intval($id);
         $status = $info['status'];
         $url = array_get($info, 'url');
+        $request_row = $db->select_first_row('build_requests', 'request', array('id' => $id));
         if ($status == 'failedIfNotCompleted') {
-            $request_row = $db->select_first_row('build_requests', 'request', array('id' => $id));
             if (!$request_row) {
                 $db->rollback_transaction();
                 exit_with_error('FailedToFindBuildRequest', array('buildRequest' => $id));
@@ -66,7 +67,17 @@ function update_builds($db, $updates) {
                 exit_with_error('UnknownBuildRequestStatus', array('buildRequest' => $id, 'status' => $status));
             }
             $db->update_row('build_requests', 'request', array('id' => $id), array('status' => $status, 'url' => $url));
+            $test_group_id = $request_row['request_group'];
+            if ($status != 'failed')
+                continue;
         }
+
+        $test_group_id = $request_row['request_group'];
+        if (array_key_exists($test_group_id, $test_groups_may_need_more_requests))
+            continue;
+
+        $db->update_row('analysis_test_groups', 'testgroup', array('id' => $test_group_id), array('may_need_more_requests' => TRUE));
+        $test_groups_may_need_more_requests[$test_group_id] = TRUE;
     }
     $db->commit_transaction();
 }
index e76a24a..031a5c4 100644 (file)
@@ -37,6 +37,16 @@ function main($path) {
                 'uploadedFiles' => array()));
         }
         $build_requests_fetcher->fetch_requests_for_groups($test_groups);
+    } elseif ($path[0] == 'need-more-requests') {
+        $test_groups = $db->select_rows('analysis_test_groups', 'testgroup', array("hidden" => FALSE, "may_need_more_requests" => TRUE));
+        if (!count($test_groups)) {
+            exit_with_success(array('testGroups' => array(),
+                'buildRequests' => array(),
+                'commitSets' => array(),
+                'commits' => array(),
+                'uploadedFiles' => array()));
+        }
+        $build_requests_fetcher->fetch_requests_for_groups($test_groups);
     } else {
         $group_id = intval($path[0]);
         $group = $db->select_first_row('analysis_test_groups', 'testgroup', array('id' => $group_id));
@@ -82,6 +92,8 @@ function format_test_group($group_row) {
         'notificationSentAt' => Database::to_js_time($group_row['testgroup_notification_sent_at']),
         'hidden' => Database::is_true($group_row['testgroup_hidden']),
         'needsNotification' => Database::is_true($group_row['testgroup_needs_notification']),
+        'mayNeedMoreRequests' => Database::is_true($group_row['testgroup_may_need_more_requests']),
+        'initialRepetitionCount' => $group_row['testgroup_initial_repetition_count'],
         'buildRequests' => array(),
         'commitSets' => array(),
     );
index bec7d48..e8530a6 100644 (file)
@@ -9,7 +9,7 @@ function create_test_group_and_build_requests($db, $commit_sets, $task_id, $name
     list ($build_configuration_list, $test_configuration_list) = insert_commit_sets_and_construct_configuration_list($db, $commit_sets);
 
     $group_id = $db->insert_row('analysis_test_groups', 'testgroup',
-        array('task' => $task_id, 'name' => $name, 'author' => $author, 'needs_notification' => $needs_notification));
+        array('task' => $task_id, 'name' => $name, 'author' => $author, 'needs_notification' => $needs_notification, 'initial_repetition_count' => $repetition_count));
 
     $build_count = count($build_configuration_list);
     $order = -$build_count;
diff --git a/Websites/perf.webkit.org/public/privileged-api/add-build-requests.php b/Websites/perf.webkit.org/public/privileged-api/add-build-requests.php
new file mode 100644 (file)
index 0000000..d1c205e
--- /dev/null
@@ -0,0 +1,64 @@
+<?php
+
+require_once('../include/json-header.php');
+require_once('../include/commit-sets-helpers.php');
+
+function main() {
+    $db = connect();
+    $data = ensure_privileged_api_data_and_token_or_slave($db);
+
+    $additional_build_request_count = array_get($data, 'addCount');
+    if (!$additional_build_request_count)
+        exit_with_error('RequestCountNotSpecified');
+
+    $test_group_id = array_get($data, 'group');
+    if (!$test_group_id)
+        exit_with_error('TestGroupNotSpecified');
+
+    $test_group = $db->select_first_row('analysis_test_groups', 'testgroup', array('id' => $test_group_id));
+    if (!$test_group)
+        exit_with_error('InvalidTestGroup');
+
+    if (Database::is_true($test_group['testgroup_hidden']))
+        exit_with_error('CannotAddToHiddenTestGroup');
+
+    $existing_build_requests = $db->select_rows('build_requests', 'request', array('group' => $test_group_id), 'order');
+
+    $current_order = $existing_build_requests[count($existing_build_requests) - 1]['request_order'];
+    if ($current_order < 0)
+        exit_with_error('NoTestingBuildRequests');
+
+    $commit_sets = array();
+    $build_request_by_commit_set = array();
+    foreach ($existing_build_requests as $build_request) {
+        if ($build_request['request_order'] < 0)
+            continue;
+        $requested_commit_set = $build_request['request_commit_set'];
+        if (array_key_exists($requested_commit_set, $build_request_by_commit_set))
+            continue;
+        $build_request_by_commit_set[$requested_commit_set] = $build_request;
+        array_push($commit_sets, $requested_commit_set);
+    }
+
+    $db->begin_transaction();
+    for ($i = 0; $i < $additional_build_request_count; $i++) {
+        foreach ($commit_sets as $commit_set) {
+            $build_request = $build_request_by_commit_set[$commit_set];
+            $db->insert_row('build_requests', 'request', array(
+                'triggerable' => $build_request['request_triggerable'],
+                'repository_group' => $build_request['request_repository_group'],
+                'platform' => $build_request['request_platform'],
+                'test' => $build_request['request_test'],
+                'group' => $build_request['request_group'],
+                'order' => ++$current_order,
+                'commit_set' => $build_request['request_commit_set']));
+        }
+    }
+    $db->commit_transaction();
+
+    exit_with_success();
+}
+
+main();
+
+?>
\ No newline at end of file
index ad1755f..5915a30 100644 (file)
@@ -15,8 +15,13 @@ function main() {
     if (array_key_exists('name', $data))
         $values['name'] = $data['name'];
 
-    if (array_key_exists('hidden', $data))
+    if (array_key_exists('hidden', $data)) {
         $values['hidden'] = Database::to_database_boolean($data['hidden']);
+        $values['may_need_more_requests'] = FALSE;
+    }
+
+    if (array_key_exists('mayNeedMoreRequests', $data))
+         $values['may_need_more_requests'] = $data['mayNeedMoreRequests'];
 
     $has_needs_notification_field = array_key_exists('needsNotification', $data);
     $has_notification_sent_at_field = array_key_exists('notificationSentAt', $data);
index cc8609b..07054ae 100644 (file)
@@ -58,6 +58,7 @@ class BuildRequest extends DataModelObject {
 
     status() { return this._status; }
     hasFinished() { return this._status == 'failed' || this._status == 'completed' || this._status == 'canceled'; }
+    hasFailed() { return this._status == 'failed'; }
     hasCompleted() { return this._status == 'completed'; }
     hasStarted() { return this._status != 'pending'; }
     isScheduled() { return this._status == 'scheduled'; }
index 4ac43b4..97e8bc8 100644 (file)
@@ -10,6 +10,8 @@ class TestGroup extends LabeledObject {
         this._createdAt = new Date(object.createdAt);
         this._isHidden = object.hidden;
         this._needsNotification = object.needsNotification;
+        this._mayNeedMoreRequests = object.mayNeedMoreRequests;
+        this._initialRepetitionCount = object.initialRepetitionCount;
         this._buildRequests = [];
         this._orderBuildRequestsLazily = new LazilyEvaluatedFunction((...buildRequests) => {
             return buildRequests.sort((a, b) => a.order() - b.order());
@@ -33,6 +35,8 @@ class TestGroup extends LabeledObject {
         this._isHidden = object.hidden;
         this._needsNotification = object.needsNotification;
         this._notificationSentAt = object.notificationSentAt ? new Date(object.notificationSentAt) : null;
+        this._mayNeedMoreRequests = object.mayNeedMoreRequests;
+        this._initialRepetitionCount = object.initialRepetitionCount;
     }
 
     task() { return AnalysisTask.findById(this._taskId); }
@@ -40,6 +44,8 @@ class TestGroup extends LabeledObject {
     isHidden() { return this._isHidden; }
     buildRequests() { return this._buildRequests; }
     needsNotification() { return this._needsNotification; }
+    mayNeedMoreRequests() { return this._mayNeedMoreRequests; }
+    initialRepetitionCount() { return this._initialRepetitionCount; }
     notificationSentAt() { return this._notificationSentAt; }
     author() { return this._authorName; }
     addBuildRequest(request)
@@ -190,42 +196,52 @@ class TestGroup extends LabeledObject {
         return result;
     }
 
+    async _updateBuildRequest(content, endPoint='update-test-group')
+    {
+        await PrivilegedAPI.sendRequest(endPoint, content);
+        const data = await TestGroup.cachedFetch(`/api/test-groups/${this.id()}`, {}, true);
+        return TestGroup._createModelsFromFetchedTestGroups(data);
+    }
+
     updateName(newName)
     {
-        var self = this;
-        var id = this.id();
-        return PrivilegedAPI.sendRequest('update-test-group', {
-            group: id,
+        return this._updateBuildRequest({
+            group: this.id(),
             name: newName,
-        }).then(function (data) {
-            return TestGroup.cachedFetch(`/api/test-groups/${id}`, {}, true)
-                .then(TestGroup._createModelsFromFetchedTestGroups.bind(TestGroup));
         });
     }
 
     updateHiddenFlag(hidden)
     {
-        var self = this;
-        var id = this.id();
-        return PrivilegedAPI.sendRequest('update-test-group', {
-            group: id,
+        return this._updateBuildRequest({
+            group: this.id(),
             hidden: !!hidden,
-        }).then(function (data) {
-            return TestGroup.cachedFetch(`/api/test-groups/${id}`, {}, true)
-                .then(TestGroup._createModelsFromFetchedTestGroups.bind(TestGroup));
         });
     }
 
     async didSendNotification()
     {
-        const id = this.id();
-        await PrivilegedAPI.sendRequest('update-test-group', {
-            group: id,
+        return await this._updateBuildRequest({
+            group: this.id(),
             needsNotification: false,
             notificationSentAt: (new Date).toISOString()
         });
-        const data = await TestGroup.cachedFetch(`/api/test-groups/${id}`, {}, true);
-        return TestGroup._createModelsFromFetchedTestGroups(data);
+    }
+
+    async addMoreBuildRequests(addCount)
+    {
+        return await this._updateBuildRequest({
+            group: this.id(),
+            addCount,
+        }, 'add-build-requests');
+    }
+
+    async clearMayNeedMoreBuildRequests()
+    {
+        return await this._updateBuildRequest({
+            group: this.id(),
+            mayNeedMoreRequests: false
+        });
     }
 
     static createWithTask(taskName, platform, test, groupName, repetitionCount, commitSets, notifyOnCompletion)
@@ -278,6 +294,11 @@ class TestGroup extends LabeledObject {
         return this.cachedFetch('/api/test-groups/ready-for-notification', null, true).then(this._createModelsFromFetchedTestGroups.bind(this));
     }
 
+    static fetchAllThatMayNeedMoreRequests()
+    {
+        return this.cachedFetch('/api/test-groups/need-more-requests', null, true).then(this._createModelsFromFetchedTestGroups.bind(this));
+    }
+
     static _createModelsFromFetchedTestGroups(data)
     {
         var testGroups = data['testGroups'].map(function (row) {
index a6c676a..c019a42 100644 (file)
@@ -6,10 +6,9 @@ const TestServer = require('./resources/test-server.js');
 const prepareServerTest = require('./resources/common-operations.js').prepareServerTest;
 
 describe('/api/test-groups', function () {
-    prepareServerTest(this);
+    prepareServerTest(this, 'node');
 
     describe('/api/test-groups/ready-for-notification', () => {
-
         it('should give an empty list if there is not existing test group at all', async () => {
             const content = await TestServer.remoteAPI().getJSON('/api/test-groups/ready-for-notification');
             assert.equal(content.status, 'OK');
@@ -85,4 +84,105 @@ describe('/api/test-groups', function () {
             assert.deepEqual(content.uploadedFiles, []);
         });
     });
+
+    describe('/api/test-groups/<triggerables>', () => {
+        it('should set "testgroup_may_need_more_requests" when a build request is set to "failed" due to "failedIfNotCompleted"', async () => {
+            const database = TestServer.database();
+            await MockData.addMockData(database, ['completed', 'completed', 'completed', 'running']);
+
+            let data =  await TestServer.remoteAPI().getJSON('/api/test-groups/600');
+            assert.ok(!data['testGroups'][0].mayNeedMoreRequests);
+            await TestServer.remoteAPI().postJSON('/api/build-requests/build-webkit', {slaveName: 'test', slavePassword: 'password',
+                buildRequestUpdates: {703: {status: 'failedIfNotCompleted', url: 'http://webkit.org/build/1'}}});
+            data =  await TestServer.remoteAPI().getJSON('/api/test-groups/600');
+            assert.ok(data['testGroups'][0].mayNeedMoreRequests);
+        });
+
+        it('should set "testgroup_may_need_more_requests" when a build request is set to "failed"', async () => {
+            const database = TestServer.database();
+            await MockData.addMockData(database, ['completed', 'completed', 'completed', 'running']);
+
+            let data =  await TestServer.remoteAPI().getJSON('/api/test-groups/600');
+            assert.ok(!data['testGroups'][0].mayNeedMoreRequests);
+            await TestServer.remoteAPI().postJSON('/api/build-requests/build-webkit', {slaveName: 'test', slavePassword: 'password',
+                buildRequestUpdates: {703: {status: 'failed', url: 'http://webkit.org/build/1'}}});
+            data =  await TestServer.remoteAPI().getJSON('/api/test-groups/600');
+            assert.ok(data['testGroups'][0].mayNeedMoreRequests);
+        });
+
+        it('should set "testgroup_may_need_more_requests" to all test groups those have failed build request', async () => {
+            const database = TestServer.database();
+            await MockData.addMockData(database, ['completed', 'completed', 'completed', 'running']);
+            await MockData.addAnotherMockTestGroup(database, ['completed', 'completed', 'completed', 'running'], 'webkit');
+
+            let data =  await TestServer.remoteAPI().getJSON('/api/test-groups/600');
+            assert.ok(!data['testGroups'][0].mayNeedMoreRequests);
+            data =  await TestServer.remoteAPI().getJSON('/api/test-groups/601');
+            assert.ok(!data['testGroups'][0].mayNeedMoreRequests);
+
+            await TestServer.remoteAPI().postJSON('/api/build-requests/build-webkit', {slaveName: 'test', slavePassword: 'password',
+                buildRequestUpdates: {
+                    703: {status: 'failed', url: 'http://webkit.org/build/1'},
+                    713: {status: 'failedIfNotCompleted', url: 'http://webkit.org/build/11'},
+                }});
+
+            data =  await TestServer.remoteAPI().getJSON('/api/test-groups/600');
+            assert.ok(data['testGroups'][0].mayNeedMoreRequests);
+            data =  await TestServer.remoteAPI().getJSON('/api/test-groups/601');
+            assert.ok(data['testGroups'][0].mayNeedMoreRequests);
+        });
+    });
+
+    describe('/api/test-groups/need-more-requests', () => {
+        it('should list all test groups potentially need additional build requests', async () => {
+            await MockData.addMockData(TestServer.database(), ['completed', 'completed', 'failed', 'completed'], false);
+            await TestServer.database().query('UPDATE analysis_test_groups SET testgroup_may_need_more_requests = TRUE WHERE testgroup_id = 600');
+            const content = await TestServer.remoteAPI().getJSON('/api/test-groups/need-more-requests');
+            assert.equal(content.testGroups.length, 1);
+            const testGroup = content.testGroups[0];
+            assert.equal(testGroup.id, 600);
+            assert.equal(testGroup.task, 500);
+            assert.equal(testGroup.name, 'some test group');
+            assert.equal(testGroup.author, null);
+            assert.ok(!testGroup.hidden);
+            assert.ok(!testGroup.needsNotification);
+            assert.ok(testGroup.mayNeedMoreRequests);
+            assert.equal(testGroup.platform, 65);
+            assert.deepEqual(testGroup.buildRequests, ['700','701', '702', '703']);
+            assert.deepEqual(testGroup.commitSets, ['401', '402', '401', '402']);
+        });
+
+        it('should list all test groups may need additional build requests', async () => {
+            await MockData.addMockData(TestServer.database(), ['completed', 'completed', 'completed', 'completed']);
+            await MockData.addAnotherMockTestGroup(TestServer.database(), ['completed', 'completed', 'failed', 'completed'], 'webkit');
+            await TestServer.database().query('UPDATE analysis_test_groups SET testgroup_may_need_more_requests = TRUE WHERE testgroup_id = 600');
+            await TestServer.database().query('UPDATE analysis_test_groups SET testgroup_may_need_more_requests = TRUE WHERE testgroup_id = 601');
+            const content = await TestServer.remoteAPI().getJSON('/api/test-groups/need-more-requests');
+            assert.equal(content.testGroups.length, 2);
+
+            const oneTestGroup = content.testGroups[0];
+            assert.equal(oneTestGroup.id, 600);
+            assert.equal(oneTestGroup.task, 500);
+            assert.equal(oneTestGroup.name, 'some test group');
+            assert.equal(oneTestGroup.author, null);
+            assert.ok(!oneTestGroup.hidden);
+            assert.ok(oneTestGroup.needsNotification);
+            assert.ok(oneTestGroup.mayNeedMoreRequests);
+            assert.equal(oneTestGroup.platform, 65);
+            assert.deepEqual(oneTestGroup.buildRequests, ['700','701', '702', '703']);
+            assert.deepEqual(oneTestGroup.commitSets, ['401', '402', '401', '402']);
+
+            const anotherTestGroup = content.testGroups[1];
+            assert.equal(anotherTestGroup.id, 601);
+            assert.equal(anotherTestGroup.task, 500);
+            assert.equal(anotherTestGroup.name, 'another test group');
+            assert.equal(anotherTestGroup.author, 'webkit');
+            assert.ok(!anotherTestGroup.hidden);
+            assert.ok(!anotherTestGroup.needsNotification);
+            assert.ok(anotherTestGroup.mayNeedMoreRequests);
+            assert.equal(anotherTestGroup.platform, 65);
+            assert.deepEqual(anotherTestGroup.buildRequests, ['710','711', '712', '713']);
+            assert.deepEqual(anotherTestGroup.commitSets, ['401', '402', '401', '402']);
+        });
+    });
 });
\ No newline at end of file
diff --git a/Websites/perf.webkit.org/server-tests/privileged-api-add-build-requests-tests.js b/Websites/perf.webkit.org/server-tests/privileged-api-add-build-requests-tests.js
new file mode 100644 (file)
index 0000000..9440477
--- /dev/null
@@ -0,0 +1,183 @@
+'use strict';
+
+const assert = require('assert');
+
+const MockData = require('./resources/mock-data.js');
+const TestServer = require('./resources/test-server.js');
+const addSlaveForReport = require('./resources/common-operations.js').addSlaveForReport;
+const prepareServerTest = require('./resources/common-operations.js').prepareServerTest;
+const assertThrows = require('../server-tests/resources/common-operations.js').assertThrows;
+
+async function createAnalysisTask(name, webkitRevisions = ["191622", "191623"])
+{
+    const reportWithRevision = [{
+        "buildNumber": "124",
+        "buildTime": "2015-10-27T15:34:51",
+        "revisions": {
+            "WebKit": {
+                "revision": webkitRevisions[0],
+                "timestamp": '2015-10-27T11:36:56.878473Z',
+            },
+            "macOS": {
+                "revision": "15A284",
+            }
+        },
+        "builderName": "someBuilder",
+        "slaveName": "someSlave",
+        "slavePassword": "somePassword",
+        "platform": "some platform",
+        "tests": {
+            "some test": {
+                "metrics": {
+                    "Time": ["Arithmetic"],
+                },
+                "tests": {
+                    "test1": {
+                        "metrics": {"Time": { "current": [11] }},
+                    }
+                }
+            },
+        }}];
+
+    const anotherReportWithRevision = [{
+        "buildNumber": "125",
+        "buildTime": "2015-10-27T17:27:41",
+        "revisions": {
+            "WebKit": {
+                "revision": webkitRevisions[1],
+                "timestamp": '2015-10-27T16:38:10.768995Z',
+            },
+            "macOS": {
+                "revision": "15A284",
+            }
+        },
+        "builderName": "someBuilder",
+        "slaveName": "someSlave",
+        "slavePassword": "somePassword",
+        "platform": "some platform",
+        "tests": {
+            "some test": {
+                "metrics": {
+                    "Time": ["Arithmetic"],
+                },
+                "tests": {
+                    "test1": {
+                        "metrics": {"Time": { "current": [12] }},
+                    }
+                }
+            },
+        }}];
+
+    const db = TestServer.database();
+    const remote = TestServer.remoteAPI();
+    await addSlaveForReport(reportWithRevision[0]);
+    await remote.postJSON('/api/report/', reportWithRevision);
+    await remote.postJSON('/api/report/', anotherReportWithRevision);
+    await Manifest.fetch();
+    const test = Test.findByPath(['some test', 'test1']);
+    const platform = Platform.findByName('some platform');
+    const configRow = await db.selectFirstRow('test_configurations', {metric: test.metrics()[0].id(), platform: platform.id()});
+    const testRuns = await db.selectRows('test_runs', {config: configRow['id']});
+
+    assert.equal(testRuns.length, 2);
+    const content = await PrivilegedAPI.sendRequest('create-analysis-task', {
+        name: name,
+        startRun: testRuns[0]['id'],
+        endRun: testRuns[1]['id'],
+        needsNotification: true,
+    });
+    return content['taskId'];
+}
+
+async function addTriggerableAndCreateTask(name, webkitRevisions)
+{
+    const report = {
+        'slaveName': 'anotherSlave',
+        'slavePassword': 'anotherPassword',
+        'triggerable': 'build-webkit',
+        'configurations': [
+            {test: MockData.someTestId(), platform: MockData.somePlatformId()},
+            {test: MockData.someTestId(), platform: MockData.otherPlatformId()},
+        ],
+        'repositoryGroups': [
+            {name: 'os-only', acceptsRoot: true, repositories: [
+                    {repository: MockData.macosRepositoryId(), acceptsPatch: false},
+                ]},
+            {name: 'webkit-only', acceptsRoot: true, repositories: [
+                    {repository: MockData.webkitRepositoryId(), acceptsPatch: true},
+                ]},
+            {name: 'system-and-webkit', acceptsRoot: true, repositories: [
+                    {repository: MockData.macosRepositoryId(), acceptsPatch: false},
+                    {repository: MockData.webkitRepositoryId(), acceptsPatch: true}
+                ]},
+            {name: 'system-webkit-sjc', acceptsRoot: true, repositories: [
+                    {repository: MockData.macosRepositoryId(), acceptsPatch: false},
+                    {repository: MockData.jscRepositoryId(), acceptsPatch: false},
+                    {repository: MockData.webkitRepositoryId(), acceptsPatch: true}
+                ]},
+        ]
+    };
+    await MockData.addMockData(TestServer.database());
+    await addSlaveForReport(report);
+    await TestServer.remoteAPI().postJSON('/api/update-triggerable/', report);
+    await createAnalysisTask(name, webkitRevisions);
+}
+
+describe('/privileged-api/add-build-requests', function() {
+    prepareServerTest(this, 'node');
+    beforeEach(() => {
+        PrivilegedAPI.configure('test', 'password');
+    });
+
+    it('should be able to add build requests to test group', async () => {
+        await addTriggerableAndCreateTask('some task');
+        const webkit = Repository.all().filter((repository) => repository.name() == 'WebKit')[0];
+        const revisionSets = [{[webkit.id()]: {revision: '191622'}}, {[webkit.id()]: {revision: '191623'}}];
+        let result = await PrivilegedAPI.sendRequest('create-test-group',
+            {name: 'test', taskName: 'other task', platform: MockData.somePlatformId(), test: MockData.someTestId(), needsNotification: false, repetitionCount: 2, revisionSets});
+        const insertedGroupId = result['testGroupId'];
+
+        await PrivilegedAPI.sendRequest('update-test-group', {'group': insertedGroupId, mayNeedMoreRequests: true});
+
+        const testGroups = await TestGroup.fetchForTask(result['taskId'], true);
+        assert.equal(testGroups.length, 1);
+        const group = testGroups[0];
+        assert.equal(group.id(), insertedGroupId);
+        assert.equal(group.mayNeedMoreRequests(), true);
+        assert.equal(group.repetitionCount(), 2);
+        assert.equal(group.initialRepetitionCount(), 2);
+
+        await PrivilegedAPI.sendRequest('add-build-requests', {group: insertedGroupId, addCount: 2});
+
+        const updatedGroups = await TestGroup.fetchForTask(result['taskId'], true);
+        assert.equal(updatedGroups.length, 1);
+        assert.equal(updatedGroups[0].repetitionCount(), 4);
+        assert.equal(updatedGroups[0].initialRepetitionCount(), 2);
+        assert.equal(group.mayNeedMoreRequests(), true);
+        for (const commitSet of updatedGroups[0].requestedCommitSets()) {
+            const buildRequests = updatedGroups[0].requestsForCommitSet(commitSet);
+            assert.equal(buildRequests.length, 4);
+        }
+    });
+
+    it('should not be able to add build requests to a hidden test group', async () => {
+        await addTriggerableAndCreateTask('some task');
+        const webkit = Repository.all().filter((repository) => repository.name() == 'WebKit')[0];
+        const revisionSets = [{[webkit.id()]: {revision: '191622'}}, {[webkit.id()]: {revision: '191623'}}];
+        let result = await PrivilegedAPI.sendRequest('create-test-group',
+            {name: 'test', taskName: 'other task', platform: MockData.somePlatformId(), test: MockData.someTestId(),
+                hidden: true, needsNotification: false, repetitionCount: 2, revisionSets});
+        const insertedGroupId = result['testGroupId'];
+
+        const testGroups = await TestGroup.fetchForTask(result['taskId'], true);
+        assert.equal(testGroups.length, 1);
+        const group = testGroups[0];
+        assert.equal(group.id(), insertedGroupId);
+        assert.equal(group.mayNeedMoreRequests(), false);
+        assert.equal(group.repetitionCount(), 2);
+        assert.equal(group.initialRepetitionCount(), 2);
+        await group.updateHiddenFlag(true);
+
+        await assertThrows('CannotAddToHiddenTestGroup', async () => await PrivilegedAPI.sendRequest('add-build-requests', {group: insertedGroupId, addCount: 2}))
+    });
+});
\ No newline at end of file
index c7be48a..86d0e79 100644 (file)
@@ -204,4 +204,76 @@ describe('/privileged-api/update-test-group', function(){
         assert.equal(updatedGroups[0].needsNotification(), true);
         assert.ok(!group.notificationSentAt());
     });
+
+    it('should create a test group with "may_need_more_requests" field defaults to false', async () => {
+        await addTriggerableAndCreateTask('some task');
+        const webkit = Repository.all().filter((repository) => repository.name() == 'WebKit')[0];
+        const revisionSets = [{[webkit.id()]: {revision: '191622'}}, {[webkit.id()]: {revision: '191623'}}];
+        const result = await PrivilegedAPI.sendRequest('create-test-group',
+            {name: 'test', taskName: 'other task', platform: MockData.somePlatformId(), test: MockData.someTestId(), needsNotification: false, revisionSets});
+        const insertedGroupId = result['testGroupId'];
+
+        const testGroups = await TestGroup.fetchForTask(result['taskId'], true);
+        assert.equal(testGroups.length, 1);
+        const group = testGroups[0];
+        assert.equal(group.id(), insertedGroupId);
+        assert.equal(group.mayNeedMoreRequests(), false);
+        assert.ok(!group.notificationSentAt());
+
+    });
+
+    it('should be able to update "may_need_more_requests" field to true and false', async () => {
+        await addTriggerableAndCreateTask('some task');
+        const webkit = Repository.all().filter((repository) => repository.name() == 'WebKit')[0];
+        const revisionSets = [{[webkit.id()]: {revision: '191622'}}, {[webkit.id()]: {revision: '191623'}}];
+        const result = await PrivilegedAPI.sendRequest('create-test-group',
+            {name: 'test', taskName: 'other task', platform: MockData.somePlatformId(), test: MockData.someTestId(), needsNotification: false, revisionSets});
+        const insertedGroupId = result['testGroupId'];
+
+        const testGroups = await TestGroup.fetchForTask(result['taskId'], true);
+        assert.equal(testGroups.length, 1);
+        const group = testGroups[0];
+        assert.equal(group.id(), insertedGroupId);
+        assert.equal(group.mayNeedMoreRequests(), false);
+
+        await PrivilegedAPI.sendRequest('update-test-group', {group: insertedGroupId, mayNeedMoreRequests: true});
+
+        let updatedGroups = await TestGroup.fetchForTask(result['taskId'], true);
+        assert.equal(updatedGroups.length, 1);
+        assert.equal(updatedGroups[0].mayNeedMoreRequests(), true);
+
+        await PrivilegedAPI.sendRequest('update-test-group', {group: insertedGroupId, mayNeedMoreRequests: false});
+
+        updatedGroups = await TestGroup.fetchForTask(result['taskId'], true);
+        assert.equal(updatedGroups.length, 1);
+        assert.equal(updatedGroups[0].mayNeedMoreRequests(), false);
+    });
+
+    it('should clear "may_need_more_requests" when hiding a test group', async () => {
+        await addTriggerableAndCreateTask('some task');
+        const webkit = Repository.all().filter((repository) => repository.name() == 'WebKit')[0];
+        const revisionSets = [{[webkit.id()]: {revision: '191622'}}, {[webkit.id()]: {revision: '191623'}}];
+        const result = await PrivilegedAPI.sendRequest('create-test-group',
+            {name: 'test', taskName: 'other task', platform: MockData.somePlatformId(), test: MockData.someTestId(), needsNotification: false, revisionSets});
+        const insertedGroupId = result['testGroupId'];
+
+        const testGroups = await TestGroup.fetchForTask(result['taskId'], true);
+        assert.equal(testGroups.length, 1);
+        const group = testGroups[0];
+        assert.equal(group.id(), insertedGroupId);
+        assert.equal(group.mayNeedMoreRequests(), false);
+
+        await PrivilegedAPI.sendRequest('update-test-group', {group: insertedGroupId, mayNeedMoreRequests: true});
+
+        let updatedGroups = await TestGroup.fetchForTask(result['taskId'], true);
+        assert.equal(updatedGroups.length, 1);
+        assert.equal(updatedGroups[0].mayNeedMoreRequests(), true);
+
+        await PrivilegedAPI.sendRequest('update-test-group', {group: insertedGroupId, hidden: true});
+
+        updatedGroups = await TestGroup.fetchForTask(result['taskId'], true);
+        assert.equal(updatedGroups.length, 1);
+        assert.equal(updatedGroups[0].mayNeedMoreRequests(), false);
+        assert.equal(updatedGroups[0].isHidden(), true);
+    });
 });
\ No newline at end of file
index 64520a5..6091459 100644 (file)
@@ -64,7 +64,7 @@ MockData = {
             db.insert('test_runs', {id: 801, config: 301, build: 901, mean_cache: 100}),
         ]);
     },
-    addMockData: function (db, statusList)
+    addMockData: function (db, statusList, needsNotification=true)
     {
         if (!statusList)
             statusList = ['pending', 'pending', 'pending', 'pending'];
@@ -79,7 +79,7 @@ MockData = {
             db.insert('analysis_tasks', {id: 500, platform: 65, metric: 300, name: 'some task',
                 start_run: 801, start_run_time: '2015-10-27T12:05:27.1Z',
                 end_run: 801, end_run_time: '2015-10-27T12:05:27.1Z'}),
-            db.insert('analysis_test_groups', {id: 600, task: 500, name: 'some test group', needs_notification: true}),
+            db.insert('analysis_test_groups', {id: 600, task: 500, name: 'some test group', initial_repetition_count: 4, needs_notification: needsNotification}),
             db.insert('build_requests', {id: 700, status: statusList[0], triggerable: 1000, repository_group: 2001, platform: 65, test: 200, group: 600, order: 0, commit_set: 401}),
             db.insert('build_requests', {id: 701, status: statusList[1], triggerable: 1000, repository_group: 2001, platform: 65, test: 200, group: 600, order: 1, commit_set: 402}),
             db.insert('build_requests', {id: 702, status: statusList[2], triggerable: 1000, repository_group: 2001, platform: 65, test: 200, group: 600, order: 2, commit_set: 401}),
@@ -123,7 +123,7 @@ MockData = {
             db.insert('commit_sets', {id: 1402}),
             db.insert('commit_set_items', {set: 1402, commit: 87832}),
             db.insert('commit_set_items', {set: 1402, commit: 196336}),
-            db.insert('analysis_test_groups', {id: 1600, task: 500, name: 'test group with git'}),
+            db.insert('analysis_test_groups', {id: 1600, task: 500, name: 'test group with git', initial_repetition_count: 4}),
             db.insert('build_requests', {id: 1700, status: 'pending', triggerable: 1000, repository_group: 2002, platform: 65, test: 200, group: 1600, order: 0, commit_set: 1401}),
             db.insert('build_requests', {id: 1701, status: 'pending', triggerable: 1000, repository_group: 2002, platform: 65, test: 200, group: 1600, order: 1, commit_set: 1402}),
             db.insert('build_requests', {id: 1702, status: 'pending', triggerable: 1000, repository_group: 2002, platform: 65, test: 200, group: 1600, order: 2, commit_set: 1401}),
@@ -139,7 +139,7 @@ MockData = {
         const platform = 65;
         const repository_group = 2001;
         return Promise.all([
-            db.insert('analysis_test_groups', {id: 601, task: 500, name: 'another test group', author}),
+            db.insert('analysis_test_groups', {id: 601, task: 500, name: 'another test group', author, initial_repetition_count: 4}),
             db.insert('build_requests', {id: 713, status: statusList[3], triggerable, repository_group, platform, test, group: 601, order: 3, commit_set: 402}),
             db.insert('build_requests', {id: 710, status: statusList[0], triggerable, repository_group, platform, test, group: 601, order: 0, commit_set: 401}),
             db.insert('build_requests', {id: 712, status: statusList[2], triggerable, repository_group, platform, test, group: 601, order: 2, commit_set: 401}),
@@ -157,7 +157,7 @@ MockData = {
             db.insert('analysis_tasks', {id: 1080, platform: 65, metric: 300, name: 'some task with component test',
                 start_run: 801, start_run_time: '2015-10-27T12:05:27.1Z',
                 end_run: 801, end_run_time: '2015-10-27T12:05:27.1Z'}),
-            db.insert('analysis_test_groups', {id: 900, task: 1080, name: 'some test group with component test'}),
+            db.insert('analysis_test_groups', {id: 900, task: 1080, name: 'some test group with component test', initial_repetition_count: 4}),
             db.insert('commit_sets', {id: 403}),
             db.insert('commit_set_items', {set: 403, commit: 87832}),
             db.insert('commit_set_items', {set: 403, commit: 93116}),
@@ -180,7 +180,7 @@ MockData = {
             db.insert('analysis_tasks', {id: 1080, platform: 65, metric: 300, name: 'some task with component test',
                 start_run: 801, start_run_time: '2015-10-27T12:05:27.1Z',
                 end_run: 801, end_run_time: '2015-10-27T12:05:27.1Z'}),
-            db.insert('analysis_test_groups', {id: 900, task: 1080, name: 'some test group with component test'}),
+            db.insert('analysis_test_groups', {id: 900, task: 1080, name: 'some test group with component test', initial_repetition_count: 4}),
             db.insert('commit_sets', {id: 404}),
             db.insert('commit_set_items', {set: 404, commit: 87832}),
             db.insert('commit_set_items', {set: 404, commit: 96336}),
index a8ec0ed..ad60c46 100644 (file)
@@ -156,7 +156,7 @@ class TestServer {
     _determinePgsqlDirectory()
     {
         try {
-            let initdbLocation = childProcess.execFileSync('which', ['initdb']);
+            let initdbLocation = childProcess.execFileSync('which', ['initdb']).toString();
             return path.dirname(initdbLocation);
         } catch (error) {
             let serverPgsqlLocation = '/Applications/Server.app/Contents/ServerRoot/usr/bin/';
diff --git a/Websites/perf.webkit.org/tools/js/retry-failed-build-requests.js b/Websites/perf.webkit.org/tools/js/retry-failed-build-requests.js
new file mode 100644 (file)
index 0000000..905ba22
--- /dev/null
@@ -0,0 +1,47 @@
+async function createAdditionalBuildRequestsForTestGroupsWithFailedRequests(testGroups, maximumRetryFactor)
+{
+    for (const testGroup of testGroups) {
+        if (!testGroup.mayNeedMoreRequests())
+            continue;
+
+        if (testGroup.isHidden()) {
+            await testGroup.clearMayNeedMoreBuildRequests();
+            continue;
+        }
+
+        let maxMissingBuildRequestCount = 0;
+        let hasCompletedBuildRequestForEachCommitSet = true;
+        let hasUnfinishedBuildRequest = false;
+        for (const commitSet of testGroup.requestedCommitSets()) {
+            const buildRequests = testGroup.requestsForCommitSet(commitSet).filter((buildRequest) => buildRequest.isTest());
+
+            const completedBuildRequestCount = buildRequests.filter((buildRequest) => buildRequest.hasCompleted()).length;
+            if (!completedBuildRequestCount)
+                hasCompletedBuildRequestForEachCommitSet = false;
+
+            hasCompletedBuildRequestForEachCommitSet &= (completedBuildRequestCount > 0);
+            const unfinishedBuildRequestCount = buildRequests.filter((buildRequest) => !buildRequest.hasFinished()).length;
+            // "potentiallySuccessfulCount" might be larger than testGroup.initialRepetitionCount() as user may add build requests manually.
+            const potentiallySuccessfulCount = completedBuildRequestCount + unfinishedBuildRequestCount;
+            maxMissingBuildRequestCount = Math.max(maxMissingBuildRequestCount, testGroup.initialRepetitionCount() - potentiallySuccessfulCount);
+
+            if (unfinishedBuildRequestCount)
+                hasUnfinishedBuildRequest = true;
+        }
+        const willExceedMaximumRetry = testGroup.repetitionCount() + maxMissingBuildRequestCount > maximumRetryFactor * testGroup.initialRepetitionCount();
+
+        console.assert(maxMissingBuildRequestCount <= testGroup.initialRepetitionCount());
+        if (maxMissingBuildRequestCount && !willExceedMaximumRetry && hasUnfinishedBuildRequest && !hasCompletedBuildRequestForEachCommitSet)
+            continue;
+
+        if (maxMissingBuildRequestCount && !willExceedMaximumRetry && hasCompletedBuildRequestForEachCommitSet) {
+            await testGroup.addMoreBuildRequests(maxMissingBuildRequestCount);
+            const analysisTask = await testGroup.fetchTask();
+            console.log(`Added ${maxMissingBuildRequestCount} build request(s) to "${testGroup.name()}" of analysis task: ${analysisTask.id()} - "${analysisTask.name()}"`);
+        }
+        await testGroup.clearMayNeedMoreBuildRequests();
+    }
+}
+
+if (typeof module !== 'undefined')
+    module.exports.createAdditionalBuildRequestsForTestGroupsWithFailedRequests = createAdditionalBuildRequestsForTestGroupsWithFailedRequests;
\ No newline at end of file
index 0f7d3f0..85b5e2a 100644 (file)
@@ -6,6 +6,7 @@ const RemoteAPI = require('./js/remote.js').RemoteAPI;
 const MeasurementSetAnalyzer = require('./js/measurement-set-analyzer.js').MeasurementSetAnalyzer;
 const AnalysisResultsNotifier = require('./js/analysis-results-notifier.js').AnalysisResultsNotifier;
 const Subprocess = require('./js/subprocess.js').Subprocess;
+const createAdditionalBuildRequestsForTestGroupsWithFailedRequests = require('./js/retry-failed-build-requests').createAdditionalBuildRequestsForTestGroupsWithFailedRequests;
 require('./js/v3-models.js');
 global.PrivilegedAPI = require('./js/privileged-api.js').PrivilegedAPI;
 
@@ -16,6 +17,7 @@ function main(argv)
         {name: '--notification-config-json', required: true},
         {name: '--analysis-range-in-days', type: parseFloat, default: 10},
         {name: '--seconds-to-sleep', type: parseFloat, default: 1200},
+        {name: '--max-retry-factor', type: parseFloat, default: 3},
     ]);
 
     if (!options)
@@ -31,6 +33,7 @@ async function analysisLoop(options)
         const serverConfig = JSON.parse(fs.readFileSync(options['--server-config-json'], 'utf-8'));
         const notificationConfig = JSON.parse(fs.readFileSync(options['--notification-config-json'], 'utf-8'));
         const analysisRangeInDays = options['--analysis-range-in-days'];
+        const maximumRetryFactor = options['--max-retry-factor'];
         secondsToSleep = options['--seconds-to-sleep'];
         global.RemoteAPI = new RemoteAPI(serverConfig.server);
         PrivilegedAPI.configure(serverConfig.slave.name, serverConfig.slave.password);
@@ -45,14 +48,16 @@ async function analysisLoop(options)
         console.log(`Start analyzing last ${analysisRangeInDays} days measurement sets.`);
         await analyzer.analyzeOnce();
 
-        const testGroups = await TestGroup.fetchAllWithNotificationReady();
+        const testGroupsMayNeedMoreRequests = await TestGroup.fetchAllThatMayNeedMoreRequests();
+        await createAdditionalBuildRequestsForTestGroupsWithFailedRequests(testGroupsMayNeedMoreRequests, maximumRetryFactor);
 
+        const testGroupsNeedNotification = await TestGroup.fetchAllWithNotificationReady();
         const notificationRemoveAPI = new RemoteAPI(notificationConfig.notificationServerConfig);
         const notificationMessageConfig = notificationConfig.notificationMessageConfig;
         const notifier = new AnalysisResultsNotifier(notificationMessageConfig.messageTemplate, notificationMessageConfig.finalizeScript,
             notificationMessageConfig.messageConstructionRules, notificationRemoveAPI, notificationConfig.notificationServerConfig.path, new Subprocess);
 
-        await notifier.sendNotificationsForTestGroups(testGroups);
+        await notifier.sendNotificationsForTestGroups(testGroupsNeedNotification);
     } catch(error) {
         console.error(`Failed analyze measurement sets due to ${error}`);
     }
diff --git a/Websites/perf.webkit.org/unit-tests/retry-failed-build-requests-tests.js b/Websites/perf.webkit.org/unit-tests/retry-failed-build-requests-tests.js
new file mode 100644 (file)
index 0000000..044f59e
--- /dev/null
@@ -0,0 +1,252 @@
+'use strict';
+
+const assert = require('assert');
+require('../tools/js/v3-models.js');
+const MockModels = require('./resources/mock-v3-models.js').MockModels;
+const MockRemoteAPI = require('./resources/mock-remote-api.js').MockRemoteAPI;
+const NodePrivilegedAPI = require('../tools/js/privileged-api').PrivilegedAPI;
+const createAdditionalBuildRequestsForTestGroupsWithFailedRequests = require('../tools/js/retry-failed-build-requests').createAdditionalBuildRequestsForTestGroupsWithFailedRequests;
+
+
+function sampleTestGroup(config) {
+    const needsNotification = config.needsNotification;
+    const initialRepetitionCount = config.initialRepetitionCount;
+    const mayNeedMoreRequests = config.mayNeedMoreRequests;
+    const hidden = config.hidden;
+    const statusList = config.statusList;
+
+    return {
+        "testGroups": [{
+            "id": "2128",
+            "task": "1376",
+            "platform": "31",
+            "name": "Confirm",
+            "author": "rniwa",
+            "createdAt": 1458688514000,
+            hidden,
+            needsNotification,
+            "buildRequests": ["16985", "16986", "16987", "16988", "16989", "16990"],
+            "commitSets": ["4255", "4256"],
+            "notificationSentAt": null,
+            initialRepetitionCount,
+            mayNeedMoreRequests
+        }],
+        "buildRequests": [{
+            "id": "16985",
+            "triggerable": "3",
+            "test": "844",
+            "platform": "31",
+            "testGroup": "2128",
+            "order": "0",
+            "commitSet": "4255",
+            "status": statusList[0],
+            "url": null,
+            "build": null,
+            "createdAt": 1458688514000
+        }, {
+            "id": "16986",
+            "triggerable": "3",
+            "test": "844",
+            "platform": "31",
+            "testGroup": "2128",
+            "order": "1",
+            "commitSet": "4256",
+            "status": statusList[1],
+            "url": null,
+            "build": null,
+            "createdAt": 1458688514000
+        }, {
+            "id": "16987",
+            "triggerable": "3",
+            "test": "844",
+            "platform": "31",
+            "testGroup": "2128",
+            "order": "2",
+            "commitSet": "4255",
+            "status": statusList[2],
+            "url": null,
+            "build": null,
+            "createdAt": 1458688514000
+        }, {
+            "id": "16988",
+            "triggerable": "3",
+            "test": "844",
+            "platform": "31",
+            "testGroup": "2128",
+            "order": "3",
+            "commitSet": "4256",
+            "status": statusList[3],
+            "url": null,
+            "build": null,
+            "createdAt": 1458688514000
+        }, {
+            "id": "16989",
+            "triggerable": "3",
+            "test": "844",
+            "platform": "31",
+            "testGroup": "2128",
+            "order": "3",
+            "commitSet": "4255",
+            "status": statusList[4],
+            "url": null,
+            "build": null,
+            "createdAt": 1458688514000
+        }, {
+            "id": "16990",
+            "triggerable": "3",
+            "test": "844",
+            "platform": "31",
+            "testGroup": "2128",
+            "order": "3",
+            "commitSet": "4256",
+            "status": statusList[5],
+            "url": null,
+            "build": null,
+            "createdAt": 1458688514000
+        }],
+        "commitSets": [{
+            "id": "4255",
+            "revisionItems": [{"commit": "87832"}, {"commit": "93116"}],
+            "customRoots": [],
+        }, {
+            "id": "4256",
+            "revisionItems": [{"commit": "87832"}, {"commit": "96336"}],
+            "customRoots": [],
+        }],
+        "commits": [{
+            "id": "87832",
+            "repository": "9",
+            "revision": "10.11 15A284",
+            "time": 0
+        }, {
+            "id": "93116",
+            "repository": "11",
+            "revision": "191622",
+            "time": 1445945816878
+        }, {
+            "id": "87832",
+            "repository": "9",
+            "revision": "10.11 15A284",
+            "time": 0
+        }, {
+            "id": "96336",
+            "repository": "11",
+            "revision": "192736",
+            "time": 1448225325650
+        }],
+        "uploadedFiles": [],
+        "status": "OK"
+    };
+}
+
+describe('createAdditionalBuildRequestsForTestGroupsWithFailedRequests', () => {
+    let requests = MockRemoteAPI.inject(null, NodePrivilegedAPI);
+    MockModels.inject();
+    beforeEach(() => {
+        PrivilegedAPI.configure('slave_name', 'password');
+    });
+
+    it('should add one more build request when one of the existing requests failed', async () => {
+        const testGroupConfig = {needsNotification: false, initialRepetitionCount: 3, mayNeedMoreRequests: true, hidden: false,
+            statusList: ["completed", "completed", "completed", "completed", "completed", "failed"]};
+        const data = sampleTestGroup(testGroupConfig);
+        const testGroups = TestGroup._createModelsFromFetchedTestGroups(data);
+        createAdditionalBuildRequestsForTestGroupsWithFailedRequests(testGroups, 3);
+        assert.equal(requests.length, 1);
+
+        assert.equal(requests[0].url, '/privileged-api/add-build-requests');
+        assert.deepEqual(requests[0].data, {slaveName: 'slave_name', slavePassword: 'password', group: '2128', addCount: 1});
+        requests[0].resolve();
+
+        await MockRemoteAPI.waitForRequest();
+        assert.equal(requests.length, 2);
+        assert.equal(requests[1].url, '/api/test-groups/2128');
+    });
+
+    it('should add 2 more build requests when 2 failed build request found for a commit set', async () => {
+        const testGroupConfig = {needsNotification: false, initialRepetitionCount: 3, mayNeedMoreRequests: true, hidden: false,
+            statusList: ["completed", "failed", "completed", "completed", "completed", "failed"]};
+        const data = sampleTestGroup(testGroupConfig);
+        const testGroups = TestGroup._createModelsFromFetchedTestGroups(data);
+        createAdditionalBuildRequestsForTestGroupsWithFailedRequests(testGroups, 3);
+        assert.equal(requests.length, 1);
+
+        assert.equal(requests[0].url, '/privileged-api/add-build-requests');
+        assert.deepEqual(requests[0].data, {slaveName: 'slave_name', slavePassword: 'password', group: '2128', addCount: 2});
+        requests[0].resolve();
+
+        await MockRemoteAPI.waitForRequest();
+        assert.equal(requests.length, 2);
+        assert.equal(requests[1].url, '/api/test-groups/2128');
+    });
+
+    it('should not schedule more build requests when all requests for a commit set had failed', async () => {
+        const testGroupConfig = {needsNotification: false, initialRepetitionCount: 3, mayNeedMoreRequests: true, hidden: false,
+            statusList: ["completed", "failed", "completed", "failed", "completed", "failed"]};
+        const data = sampleTestGroup(testGroupConfig);
+        const testGroups = TestGroup._createModelsFromFetchedTestGroups(data);
+        createAdditionalBuildRequestsForTestGroupsWithFailedRequests(testGroups, 3);
+        assert.equal(requests.length, 1);
+
+        assert.equal(requests[0].url, '/privileged-api/update-test-group');
+        assert.deepEqual(requests[0].data, {slaveName: 'slave_name', slavePassword: 'password', group: '2128', mayNeedMoreRequests: false});
+        requests[0].resolve();
+    });
+
+    it('should not schedule more build requests when "may_need_more_requests" is not set', async () => {
+        const testGroupConfig = {needsNotification: false, initialRepetitionCount: 3, mayNeedMoreRequests: false, hidden: false,
+            statusList: ["completed", "failed", "completed", "completed", "completed", "failed"]};
+        const data = sampleTestGroup(testGroupConfig);
+        const testGroups = TestGroup._createModelsFromFetchedTestGroups(data);
+        createAdditionalBuildRequestsForTestGroupsWithFailedRequests(testGroups, 3);
+        assert.equal(requests.length, 0);
+    });
+
+    it('should not schedule more build requests when build request is hidden', async () => {
+        const testGroupConfig = {needsNotification: false, initialRepetitionCount: 3, mayNeedMoreRequests: true, hidden: true,
+            statusList: ["completed", "failed", "completed", "completed", "completed", "failed"]};
+        const data = sampleTestGroup(testGroupConfig);
+        const testGroups = TestGroup._createModelsFromFetchedTestGroups(data);
+        createAdditionalBuildRequestsForTestGroupsWithFailedRequests(testGroups, 3);
+        assert.equal(requests.length, 1);
+
+        assert.equal(requests[0].url, '/privileged-api/update-test-group');
+        assert.deepEqual(requests[0].data, {slaveName: 'slave_name', slavePassword: 'password', group: '2128', mayNeedMoreRequests: false});
+        requests[0].resolve();
+    });
+
+    it('should not schedule more build request when we\'ve already hit the maximum retry count', async () => {
+        const testGroupConfig = {needsNotification: false, initialRepetitionCount: 3, mayNeedMoreRequests: true, hidden: false,
+            statusList: ["completed", "completed", "failed", "failed", "failed", "failed"]};
+        const data = sampleTestGroup(testGroupConfig);
+        const testGroups = TestGroup._createModelsFromFetchedTestGroups(data);
+        createAdditionalBuildRequestsForTestGroupsWithFailedRequests(testGroups, 1.5);
+        assert.equal(requests.length, 1);
+
+        assert.equal(requests[0].url, '/privileged-api/update-test-group');
+        assert.deepEqual(requests[0].data, {slaveName: 'slave_name', slavePassword: 'password', group: '2128', mayNeedMoreRequests: false});
+        requests[0].resolve();
+    });
+
+    it('should not schedule more when additional build requests are still pending', async () => {
+        const testGroupConfig = {needsNotification: false, initialRepetitionCount: 2, mayNeedMoreRequests: true, hidden: false,
+            statusList: ["completed", "completed", "failed", "failed", "pending", "pending"]};
+        const data = sampleTestGroup(testGroupConfig);
+        const testGroups = TestGroup._createModelsFromFetchedTestGroups(data);
+        createAdditionalBuildRequestsForTestGroupsWithFailedRequests(testGroups, 3);
+        assert.equal(requests.length, 1);
+
+        assert.equal(requests[0].url, '/privileged-api/update-test-group');
+        assert.deepEqual(requests[0].data, {slaveName: 'slave_name', slavePassword: 'password', group: '2128', mayNeedMoreRequests: false});
+        requests[0].resolve();
+    });
+
+    it('should not clear mayNeedMoreRequest flag when one commit set has not got a successful run but have pending builds', async () => {
+        const testGroupConfig = {needsNotification: false, initialRepetitionCount: 3, mayNeedMoreRequests: true, hidden: false,
+            statusList: ["completed", "failed", "completed", "failed", "pending", "pending"]};
+        const data = sampleTestGroup(testGroupConfig);
+        const testGroups = TestGroup._createModelsFromFetchedTestGroups(data);
+        createAdditionalBuildRequestsForTestGroupsWithFailedRequests(testGroups, 3);
+        assert.equal(requests.length, 0);
+    });
+});
index fb89162..aa43554 100644 (file)
@@ -7,7 +7,7 @@ const NodePrivilegedAPI = require('../tools/js/privileged-api.js').PrivilegedAPI
 const MockModels = require('./resources/mock-v3-models.js').MockModels;
 const MockRemoteAPI = require('./resources/mock-remote-api.js').MockRemoteAPI;
 
-function sampleTestGroup(needsNotification=true) {
+function sampleTestGroup(needsNotification=true, initialRepetitionCount=2, mayNeedMoreRequests=true) {
     return {
         "testGroups": [{
             "id": "2128",
@@ -20,6 +20,9 @@ function sampleTestGroup(needsNotification=true) {
             "needsNotification": needsNotification,
             "buildRequests": ["16985", "16986", "16987", "16988", "16989", "16990", "16991", "16992"],
             "commitSets": ["4255", "4256"],
+            "notificationSentAt": null,
+            initialRepetitionCount,
+            mayNeedMoreRequests
         }],
         "buildRequests": [{
             "id": "16985",
@@ -174,6 +177,66 @@ describe('TestGroup', function () {
         });
     });
 
+    describe('initialRepetitionCount', () => {
+        const requests = MockRemoteAPI.inject('https://perf.webkit.org', NodePrivilegedAPI);
+        beforeEach(() => {
+            PrivilegedAPI.configure('test', 'password');
+        });
+
+        it('should construct initialRepetitionCount from data', async () => {
+            const fetchPromise = TestGroup.fetchForTask(1376);
+            requests[0].resolve(sampleTestGroup());
+            let testGroups = await fetchPromise;
+            assert(testGroups.length, 1);
+            let testGroup = testGroups[0];
+            assert.equal(testGroup.initialRepetitionCount(), 2);
+        });
+    });
+
+    describe('mayNeedMoreRequests', () => {
+        const requests = MockRemoteAPI.inject('https://perf.webkit.org', NodePrivilegedAPI);
+        beforeEach(() => {
+            PrivilegedAPI.configure('test', 'password');
+        });
+
+        it('should construct mayNeedMoreRequests from data', async () => {
+            const fetchPromise = TestGroup.fetchForTask(1376);
+            requests[0].resolve(sampleTestGroup());
+            let testGroups = await fetchPromise;
+            assert(testGroups.length, 1);
+            let testGroup = testGroups[0];
+            assert.ok(testGroup.mayNeedMoreRequests());
+        });
+
+        it('should be able to clear mayNeedMoreRequests flag', async () => {
+            const fetchPromise = TestGroup.fetchForTask(1376);
+            requests[0].resolve(sampleTestGroup());
+            let testGroups = await fetchPromise;
+            assert(testGroups.length, 1);
+            let testGroup = testGroups[0];
+            assert.ok(testGroup.mayNeedMoreRequests());
+
+            const updatePromise = testGroup.clearMayNeedMoreBuildRequests();
+            assert.equal(requests.length, 2);
+            assert.equal(requests.length, 2);
+            assert.equal(requests[1].method, 'POST');
+            assert.equal(requests[1].url, '/privileged-api/update-test-group');
+            assert.deepEqual(requests[1].data, {group: '2128', mayNeedMoreRequests: false, slaveName: 'test', slavePassword: 'password'});
+            requests[1].resolve();
+
+            await MockRemoteAPI.waitForRequest();
+            assert.equal(requests.length, 3);
+            assert.equal(requests[2].method, 'GET');
+            assert.equal(requests[2].url, '/api/test-groups/2128');
+            const updatedTestGroup = sampleTestGroup(true, 4, false);
+            requests[2].resolve(updatedTestGroup);
+
+            testGroups = await updatePromise;
+            testGroup = testGroups[0];
+            assert.equal(testGroup.mayNeedMoreRequests(), false);
+        });
+    });
+
     describe('_createModelsFromFetchedTestGroups', function () {
         it('should create test groups', function () {
             var groups = TestGroup._createModelsFromFetchedTestGroups(sampleTestGroup());