New syncing script sometimes schedules a build request on a wrong builder
[WebKit.git] / Websites / perf.webkit.org / tools / js / buildbot-triggerable.js
1 'use strict';
2
3 let assert = require('assert');
4
5 require('./v3-models.js');
6
7 let BuildbotSyncer = require('./buildbot-syncer').BuildbotSyncer;
8
9 class BuildbotTriggerable {
10     constructor(config, remote, buildbotRemote, slaveInfo, logger)
11     {
12         this._name = config.triggerableName;
13         assert(typeof(this._name) == 'string', 'triggerableName must be specified');
14
15         this._lookbackCount = config.lookbackCount;
16         assert(typeof(this._lookbackCount) == 'number' && this._lookbackCount > 0, 'lookbackCount must be a number greater than 0');
17
18         this._remote = remote;
19
20         this._slaveInfo = slaveInfo;
21         assert(typeof(slaveInfo.name) == 'string', 'slave name must be specified');
22         assert(typeof(slaveInfo.password) == 'string', 'slave password must be specified');
23
24         this._syncers = BuildbotSyncer._loadConfig(buildbotRemote, config);
25         this._logger = logger || {log: function () { }, error: function () { }};
26     }
27
28     name() { return this._name; }
29
30     syncOnce()
31     {
32         let syncerList = this._syncers;
33         let buildReqeustsByGroup = new Map;
34
35         let self = this;
36         this._logger.log(`Fetching build requests for ${this._name}...`);
37         return BuildRequest.fetchForTriggerable(this._name).then(function (buildRequests) {
38             self._validateRequests(buildRequests);
39             buildReqeustsByGroup = BuildbotTriggerable._testGroupMapForBuildRequests(buildRequests);
40             return self._pullBuildbotOnAllSyncers(buildReqeustsByGroup);
41         }).then(function (updates) {
42             self._logger.log('Scheduling builds');
43             let promistList = [];
44             let testGroupList = Array.from(buildReqeustsByGroup.values()).sort(function (a, b) { return a.groupOrder - b.groupOrder; });
45             for (let group of testGroupList) {
46                 let promise = self._scheduleNextRequestInGroupIfSlaveIsAvailable(group, updates);
47                 if (promise)
48                     promistList.push(promise);
49             }
50             return Promise.all(promistList);
51         }).then(function () {
52             // Pull all buildbots for the second time since the previous step may have scheduled more builds.
53             return self._pullBuildbotOnAllSyncers(buildReqeustsByGroup);
54         }).then(function (updates) {
55             // FIXME: Add a new API that just updates the requests.
56             return self._remote.postJSON(`/api/build-requests/${self._name}`, {
57                 'slaveName': self._slaveInfo.name,
58                 'slavePassword': self._slaveInfo.password,
59                 'buildRequestUpdates': updates});
60         }).then(function (response) {
61             if (response['status'] != 'OK')
62                 self._logger.log('Failed to update the build requests status: ' + response['status']);
63         })
64     }
65
66     _validateRequests(buildRequests)
67     {
68         let testPlatformPairs = {};
69         for (let request of buildRequests) {
70             if (!this._syncers.some(function (syncer) { return syncer.matchesConfiguration(request); })) {
71                 let key = request.platform().id + '-' + request.test().id();
72                 if (!(key in testPlatformPairs))
73                     this._logger.error(`No matching configuration for "${request.test().fullName()}" on "${request.platform().name()}".`);                
74                 testPlatformPairs[key] = true;
75             }
76         }
77     }
78
79     _pullBuildbotOnAllSyncers(buildReqeustsByGroup)
80     {
81         let updates = {};
82         let associatedRequests = new Set;
83         let self = this;
84         return Promise.all(this._syncers.map(function (syncer) {
85             return syncer.pullBuildbot(self._lookbackCount).then(function (entryList) {
86                 for (let entry of entryList) {
87                     let request = BuildRequest.findById(entry.buildRequestId());
88                     if (!request)
89                         continue;
90                     associatedRequests.add(request);
91
92                     let info = buildReqeustsByGroup.get(request.testGroupId());
93                     assert(!info.syncer || info.syncer == syncer);
94                     info.syncer = syncer;
95                     if (entry.slaveName()) {
96                         assert(!info.slaveName || info.slaveName == entry.slaveName());
97                         info.slaveName = entry.slaveName();
98                     }
99
100                     let newStatus = entry.buildRequestStatusIfUpdateIsNeeded(request);
101                     if (newStatus) {
102                         self._logger.log(`Updating the status of build request ${request.id()} from ${request.status()} to ${newStatus}`);
103                         updates[entry.buildRequestId()] = {status: newStatus, url: entry.url()};
104                     } else if (!request.statusUrl()) {
105                         self._logger.log(`Setting the status URL of build request ${request.id()} to ${entry.url()}`);
106                         updates[entry.buildRequestId()] = {status: request.status(), url: entry.url()};
107                     }
108                 }
109             });
110         })).then(function () {
111             for (let request of BuildRequest.all()) {
112                 if (request.hasStarted() && !request.hasFinished() && !associatedRequests.has(request)) {
113                     self._logger.log(`Updating the status of build request ${request.id()} from ${request.status()} to failedIfNotCompleted`);
114                     assert(!(request.id() in updates));
115                     updates[request.id()] = {status: 'failedIfNotCompleted'};
116                 }
117             }
118         }).then(function () { return updates; });
119     }
120
121     _scheduleNextRequestInGroupIfSlaveIsAvailable(groupInfo, pendingUpdates)
122     {
123         let nextRequest = null;
124         for (let request of groupInfo.requests) {
125             if (request.isScheduled() || (request.id() in pendingUpdates && pendingUpdates[request.id()]['status'] == 'scheduled'))
126                 break;
127             if (request.isPending() && !(request.id() in pendingUpdates)) {
128                 nextRequest = request;
129                 break;
130             }
131         }
132         if (!nextRequest)
133             return null;
134
135         if (!!nextRequest.order()) {
136             this._logger.log(`Scheduling build request ${nextRequest.id()} on ${groupInfo.slaveName} in ${groupInfo.syncer.builderName()}`);
137             return groupInfo.syncer.scheduleRequest(nextRequest, groupInfo.slaveName);
138         }
139
140         for (let syncer of this._syncers) {
141             let promise = syncer.scheduleFirstRequestInGroupIfAvailable(nextRequest);
142             if (promise) {
143                 let slaveName = groupInfo.slaveName ? ` on ${groupInfo.slaveName}` : '';
144                 this._logger.log(`Scheduling build request ${nextRequest.id()}${slaveName} in ${syncer.builderName()}`);
145                 return promise;
146             }
147         }
148         return null;
149     }
150
151     static _testGroupMapForBuildRequests(buildRequests)
152     {
153         let map = new Map;
154         let groupOrder = 0;
155         for (let request of buildRequests) {
156             let groupId = request.testGroupId();
157             if (!map.has(groupId)) // Don't use real TestGroup objects to avoid executing postgres query in the server
158                 map.set(groupId, {id: groupId, groupOrder: groupOrder++, requests: [request], syncer: null, slaveName: null});
159             else
160                 map.get(groupId).requests.push(request);
161         }
162         return map;
163     }
164 }
165
166 if (typeof module != 'undefined')
167     module.exports.BuildbotTriggerable = BuildbotTriggerable;