sync-builedbot.js fails to schedule the second request to test with a patch
[WebKit.git] / Websites / perf.webkit.org / tools / js / buildbot-triggerable.js
1 'use strict';
2
3 let assert = require('assert');
4
5 require('./v3-models.js');
6
7 let BuildbotSyncer = require('./buildbot-syncer').BuildbotSyncer;
8
9 class BuildbotTriggerable {
10     constructor(config, remote, buildbotRemote, slaveInfo, logger)
11     {
12         this._name = config.triggerableName;
13         assert(typeof(this._name) == 'string', 'triggerableName must be specified');
14
15         this._lookbackCount = config.lookbackCount;
16         assert(typeof(this._lookbackCount) == 'number' && this._lookbackCount > 0, 'lookbackCount must be a number greater than 0');
17
18         this._remote = remote;
19
20         this._slaveInfo = slaveInfo;
21         assert(typeof(slaveInfo.name) == 'string', 'slave name must be specified');
22         assert(typeof(slaveInfo.password) == 'string', 'slave password must be specified');
23
24         this._syncers = BuildbotSyncer._loadConfig(buildbotRemote, config);
25         this._logger = logger || {log: () => { }, error: () => { }};
26     }
27
28     name() { return this._name; }
29
30     updateTriggerable()
31     {
32         const map = new Map;
33         let repositoryGroups = [];
34         for (const syncer of this._syncers) {
35             for (const config of syncer.testConfigurations()) {
36                 const entry = {test: config.test.id(), platform: config.platform.id()};
37                 map.set(entry.test + '-' + entry.platform, entry);
38             }
39             // FIXME: Move BuildbotSyncer._loadConfig here and store repository groups directly.
40             repositoryGroups = syncer.repositoryGroups();
41         }
42         return this._remote.postJSONWithStatus(`/api/update-triggerable/`, {
43             'slaveName': this._slaveInfo.name,
44             'slavePassword': this._slaveInfo.password,
45             'triggerable': this._name,
46             'configurations': Array.from(map.values()),
47             'repositoryGroups': Object.keys(repositoryGroups).map((groupName) => {
48                 const group = repositoryGroups[groupName];
49                 return {
50                     name: groupName,
51                     description: group.description,
52                     acceptsRoots: group.acceptsRoots,
53                     repositories: group.repositoryList,
54                 };
55             })});
56     }
57
58     syncOnce()
59     {
60         let syncerList = this._syncers;
61         let buildReqeustsByGroup = new Map;
62
63         this._logger.log(`Fetching build requests for ${this._name}...`);
64         let validRequests;
65         return BuildRequest.fetchForTriggerable(this._name).then((buildRequests) => {
66             validRequests = this._validateRequests(buildRequests);
67             buildReqeustsByGroup = BuildbotTriggerable._testGroupMapForBuildRequests(buildRequests);
68             return this._pullBuildbotOnAllSyncers(buildReqeustsByGroup);
69         }).then((updates) => {
70             this._logger.log('Scheduling builds');
71             const promistList = [];
72             const testGroupList = Array.from(buildReqeustsByGroup.values()).sort(function (a, b) { return a.groupOrder - b.groupOrder; });
73             for (const group of testGroupList) {
74                 const nextRequest = this._nextRequestInGroup(group, updates);
75                 if (!validRequests.has(nextRequest))
76                     continue;
77                 const promise = this._scheduleRequestIfSlaveIsAvailable(nextRequest, group.requests,
78                     nextRequest.isBuild() ? group.buildSyncer : group.testSyncer,
79                     nextRequest.isBuild() ? group.buildSlaveName : group.testSlaveName);
80                 if (promise)
81                     promistList.push(promise);
82             }
83             return Promise.all(promistList);
84         }).then(() => {
85             // Pull all buildbots for the second time since the previous step may have scheduled more builds.
86             return this._pullBuildbotOnAllSyncers(buildReqeustsByGroup);
87         }).then((updates) => {
88             // FIXME: Add a new API that just updates the requests.
89             return this._remote.postJSONWithStatus(`/api/build-requests/${this._name}`, {
90                 'slaveName': this._slaveInfo.name,
91                 'slavePassword': this._slaveInfo.password,
92                 'buildRequestUpdates': updates});
93         });
94     }
95
96     _validateRequests(buildRequests)
97     {
98         const testPlatformPairs = {};
99         const validatedRequests = new Set;
100         for (let request of buildRequests) {
101             if (!this._syncers.some((syncer) => syncer.matchesConfiguration(request))) {
102                 const key = request.platform().id + '-' + (request.isBuild() ? 'build' : request.test().id());
103                 const kind = request.isBuild() ? 'Building' : `"${request.test().fullName()}"`;
104                 if (!(key in testPlatformPairs))
105                     this._logger.error(`Build request ${request.id()} has no matching configuration: ${kind} on "${request.platform().name()}".`);
106                 testPlatformPairs[key] = true;
107                 continue;
108             }
109             const triggerable = request.triggerable();
110             if (!triggerable) {
111                 this._logger.error(`Build request ${request.id()} does not specify a valid triggerable`);
112                 continue;
113             }
114             assert(triggerable instanceof Triggerable, 'Must specify a valid triggerable');
115             assert.equal(triggerable.name(), this._name, 'Must specify the triggerable of this syncer');
116             const repositoryGroup = request.repositoryGroup();
117             if (!repositoryGroup) {
118                 this._logger.error(`Build request ${request.id()} does not specify a repository group. Such a build request is no longer supported.`);
119                 continue;
120             }
121             const acceptedGroups = triggerable.repositoryGroups();
122             if (!acceptedGroups.includes(repositoryGroup)) {
123                 const acceptedNames = acceptedGroups.map((group) => group.name()).join(', ');
124                 this._logger.error(`Build request ${request.id()} specifies ${repositoryGroup.name()} but triggerable ${this._name} only accepts ${acceptedNames}`);
125                 continue;
126             }
127             validatedRequests.add(request);
128         }
129
130         return validatedRequests;
131     }
132
133     _pullBuildbotOnAllSyncers(buildReqeustsByGroup)
134     {
135         let updates = {};
136         let associatedRequests = new Set;
137         return Promise.all(this._syncers.map((syncer) => {
138             return syncer.pullBuildbot(this._lookbackCount).then((entryList) => {
139                 for (const entry of entryList) {
140                     const request = BuildRequest.findById(entry.buildRequestId());
141                     if (!request)
142                         continue;
143                     associatedRequests.add(request);
144
145                     const info = buildReqeustsByGroup.get(request.testGroupId());
146                     if (request.isBuild()) {
147                         assert(!info.buildSyncer || info.buildSyncer == buildSyncer);
148                         if (entry.slaveName()) {
149                             assert(!info.buildSlaveName || info.buildSlaveName == entry.slaveName());
150                             info.buildSlaveName = entry.slaveName();
151                         }
152                     } else {
153                         assert(!info.testSyncer || info.testSyncer == testSyncer);
154                         if (entry.slaveName()) {
155                             assert(!info.testSlaveName || info.testSlaveName == entry.slaveName());
156                             info.testSlaveName = entry.slaveName();
157                         }
158                     }
159
160                     const newStatus = entry.buildRequestStatusIfUpdateIsNeeded(request);
161                     if (newStatus) {
162                         this._logger.log(`Updating the status of build request ${request.id()} from ${request.status()} to ${newStatus}`);
163                         updates[entry.buildRequestId()] = {status: newStatus, url: entry.url()};
164                     } else if (!request.statusUrl()) {
165                         this._logger.log(`Setting the status URL of build request ${request.id()} to ${entry.url()}`);
166                         updates[entry.buildRequestId()] = {status: request.status(), url: entry.url()};
167                     }
168                 }
169             });
170         })).then(() => {
171             for (const request of BuildRequest.all()) {
172                 if (request.hasStarted() && !request.hasFinished() && !associatedRequests.has(request)) {
173                     this._logger.log(`Updating the status of build request ${request.id()} from ${request.status()} to failedIfNotCompleted`);
174                     assert(!(request.id() in updates));
175                     updates[request.id()] = {status: 'failedIfNotCompleted'};
176                 }
177             }
178         }).then(() => updates);
179     }
180
181     _nextRequestInGroup(groupInfo, pendingUpdates)
182     {
183         for (const request of groupInfo.requests) {
184             if (request.isScheduled() || (request.id() in pendingUpdates && pendingUpdates[request.id()]['status'] == 'scheduled'))
185                 return null;
186             if (request.isPending() && !(request.id() in pendingUpdates))
187                 return request;
188             if (request.isBuild() && !request.hasCompleted())
189                 return null; // A build request is still pending, scheduled, running, or failed.
190         }
191         return null;
192     }
193
194     _scheduleRequestIfSlaveIsAvailable(nextRequest, requestsInGroup, syncer, slaveName)
195     {
196         if (!nextRequest)
197             return null;
198
199         const isFirstRequest = nextRequest == requestsInGroup[0] || !nextRequest.order();
200         if (!isFirstRequest) {
201             if (syncer)
202                 return this._scheduleRequestWithLog(syncer, nextRequest, slaveName);
203             this._logger.error(`Could not identify the syncer for ${nextRequest.id()}.`);
204         }
205
206         // Pick a new syncer for the first test.
207         for (const syncer of this._syncers) {
208             const promise = this._scheduleRequestWithLog(syncer, nextRequest, null);
209             if (promise)
210                 return promise;
211         }
212         return null;
213     }
214
215     _scheduleRequestWithLog(syncer, request, slaveName)
216     {
217         const promise = syncer.scheduleRequestInGroupIfAvailable(request, slaveName);
218         if (!promise)
219             return promise;
220         this._logger.log(`Scheduling build request ${request.id()}${slaveName ? ' on ' + slaveName : ''} in ${syncer.builderName()}`);
221         return promise;
222     }
223
224     static _testGroupMapForBuildRequests(buildRequests)
225     {
226         const map = new Map;
227         let groupOrder = 0;
228         for (let request of buildRequests) {
229             let groupId = request.testGroupId();
230             if (!map.has(groupId)) // Don't use real TestGroup objects to avoid executing postgres query in the server
231                 map.set(groupId, {id: groupId, groupOrder: groupOrder++, requests: [request], syncer: null, slaveName: null});
232             else
233                 map.get(groupId).requests.push(request);
234         }
235         return map;
236     }
237 }
238
239 if (typeof module != 'undefined')
240     module.exports.BuildbotTriggerable = BuildbotTriggerable;