15d38997f1a2fa78501fdb548599e88cdc932de8
[WebKit-https.git] / Websites / perf.webkit.org / tools / js / buildbot-triggerable.js
1 'use strict';
2
3 let assert = require('assert');
4
5 require('./v3-models.js');
6
7 let BuildbotSyncer = require('./buildbot-syncer').BuildbotSyncer;
8
9 class BuildbotTriggerable {
10     constructor(config, remote, buildbotRemote, slaveInfo, logger)
11     {
12         this._name = config.triggerableName;
13         assert(typeof(this._name) == 'string', 'triggerableName must be specified');
14
15         this._lookbackCount = config.lookbackCount;
16         assert(typeof(this._lookbackCount) == 'number' && this._lookbackCount > 0, 'lookbackCount must be a number greater than 0');
17
18         this._remote = remote;
19
20         this._slaveInfo = slaveInfo;
21         assert(typeof(slaveInfo.name) == 'string', 'slave name must be specified');
22         assert(typeof(slaveInfo.password) == 'string', 'slave password must be specified');
23
24         this._syncers = BuildbotSyncer._loadConfig(buildbotRemote, config);
25         this._logger = logger || {log: () => { }, error: () => { }};
26     }
27
28     name() { return this._name; }
29
30     updateTriggerable()
31     {
32         const map = new Map;
33         let repositoryGroups = [];
34         for (const syncer of this._syncers) {
35             for (const config of syncer.testConfigurations()) {
36                 const entry = {test: config.test.id(), platform: config.platform.id()};
37                 map.set(entry.test + '-' + entry.platform, entry);
38             }
39             // FIXME: Move BuildbotSyncer._loadConfig here and store repository groups directly.
40             repositoryGroups = syncer.repositoryGroups();
41         }
42         return this._remote.postJSONWithStatus(`/api/update-triggerable/`, {
43             'slaveName': this._slaveInfo.name,
44             'slavePassword': this._slaveInfo.password,
45             'triggerable': this._name,
46             'configurations': Array.from(map.values()),
47             'repositoryGroups': Object.keys(repositoryGroups).map((groupName) => repositoryGroups[groupName])});
48     }
49
50     syncOnce()
51     {
52         let syncerList = this._syncers;
53         let buildReqeustsByGroup = new Map;
54
55         this._logger.log(`Fetching build requests for ${this._name}...`);
56         let validRequests;
57         return BuildRequest.fetchForTriggerable(this._name).then((buildRequests) => {
58             validRequests = this._validateRequests(buildRequests);
59             buildReqeustsByGroup = BuildbotTriggerable._testGroupMapForBuildRequests(buildRequests);
60             return this._pullBuildbotOnAllSyncers(buildReqeustsByGroup);
61         }).then((updates) => {
62             this._logger.log('Scheduling builds');
63             const promistList = [];
64             const testGroupList = Array.from(buildReqeustsByGroup.values()).sort(function (a, b) { return a.groupOrder - b.groupOrder; });
65             for (const group of testGroupList) {
66                 const nextRequest = this._nextRequestInGroup(group, updates);
67                 if (!validRequests.has(nextRequest))
68                     continue;
69                 const promise = this._scheduleRequestIfSlaveIsAvailable(nextRequest, group.syncer, group.slaveName);
70                 if (promise)
71                     promistList.push(promise);
72             }
73             return Promise.all(promistList);
74         }).then(() => {
75             // Pull all buildbots for the second time since the previous step may have scheduled more builds.
76             return this._pullBuildbotOnAllSyncers(buildReqeustsByGroup);
77         }).then((updates) => {
78             // FIXME: Add a new API that just updates the requests.
79             return this._remote.postJSONWithStatus(`/api/build-requests/${this._name}`, {
80                 'slaveName': this._slaveInfo.name,
81                 'slavePassword': this._slaveInfo.password,
82                 'buildRequestUpdates': updates});
83         });
84     }
85
86     _validateRequests(buildRequests)
87     {
88         const testPlatformPairs = {};
89         const validatedRequests = new Set;
90         for (let request of buildRequests) {
91             if (!this._syncers.some((syncer) => syncer.matchesConfiguration(request))) {
92                 const key = request.platform().id + '-' + request.test().id();
93                 if (!(key in testPlatformPairs))
94                     this._logger.error(`Build request ${request.id()} has no matching configuration: "${request.test().fullName()}" on "${request.platform().name()}".`);
95                 testPlatformPairs[key] = true;
96                 continue;
97             }
98             const triggerable = request.triggerable();
99             if (!triggerable) {
100                 this._logger.error(`Build request ${request.id()} does not specify a valid triggerable`);
101                 continue;
102             }
103             assert(triggerable instanceof Triggerable, 'Must specify a valid triggerable');
104             assert.equal(triggerable.name(), this._name, 'Must specify the triggerable of this syncer');
105             const repositoryGroup = request.repositoryGroup();
106             if (!repositoryGroup) {
107                 this._logger.error(`Build request ${request.id()} does not specify a repository group. Such a build request is no longer supported.`);
108                 continue;
109             }
110             const acceptedGroups = triggerable.repositoryGroups();
111             if (!acceptedGroups.includes(repositoryGroup)) {
112                 const acceptedNames = acceptedGroups.map((group) => group.name()).join(', ');
113                 this._logger.error(`Build request ${request.id()} specifies ${repositoryGroup.name()} but triggerable ${this._name} only accepts ${acceptedNames}`);
114                 continue;
115             }
116             validatedRequests.add(request);
117         }
118
119         return validatedRequests;
120     }
121
122     _pullBuildbotOnAllSyncers(buildReqeustsByGroup)
123     {
124         let updates = {};
125         let associatedRequests = new Set;
126         return Promise.all(this._syncers.map((syncer) => {
127             return syncer.pullBuildbot(this._lookbackCount).then((entryList) => {
128                 for (const entry of entryList) {
129                     const request = BuildRequest.findById(entry.buildRequestId());
130                     if (!request)
131                         continue;
132                     associatedRequests.add(request);
133
134                     const info = buildReqeustsByGroup.get(request.testGroupId());
135                     assert(!info.syncer || info.syncer == syncer);
136                     info.syncer = syncer;
137                     if (entry.slaveName()) {
138                         assert(!info.slaveName || info.slaveName == entry.slaveName());
139                         info.slaveName = entry.slaveName();
140                     }
141
142                     const newStatus = entry.buildRequestStatusIfUpdateIsNeeded(request);
143                     if (newStatus) {
144                         this._logger.log(`Updating the status of build request ${request.id()} from ${request.status()} to ${newStatus}`);
145                         updates[entry.buildRequestId()] = {status: newStatus, url: entry.url()};
146                     } else if (!request.statusUrl()) {
147                         this._logger.log(`Setting the status URL of build request ${request.id()} to ${entry.url()}`);
148                         updates[entry.buildRequestId()] = {status: request.status(), url: entry.url()};
149                     }
150                 }
151             });
152         })).then(() => {
153             for (const request of BuildRequest.all()) {
154                 if (request.hasStarted() && !request.hasFinished() && !associatedRequests.has(request)) {
155                     this._logger.log(`Updating the status of build request ${request.id()} from ${request.status()} to failedIfNotCompleted`);
156                     assert(!(request.id() in updates));
157                     updates[request.id()] = {status: 'failedIfNotCompleted'};
158                 }
159             }
160         }).then(() => updates);
161     }
162
163     _nextRequestInGroup(groupInfo, pendingUpdates)
164     {
165         for (const request of groupInfo.requests) {
166             if (request.isScheduled() || (request.id() in pendingUpdates && pendingUpdates[request.id()]['status'] == 'scheduled'))
167                 return null;
168             if (request.isPending() && !(request.id() in pendingUpdates))
169                 return request;
170         }
171         return null;
172     }
173
174     _scheduleRequestIfSlaveIsAvailable(nextRequest, syncer, slaveName)
175     {
176         if (!nextRequest)
177             return null;
178
179         if (!!nextRequest.order()) {
180             if (syncer)
181                 return this._scheduleRequestWithLog(syncer, nextRequest, slaveName);
182             this._logger.error(`Could not identify the syncer for ${nextRequest.id()}.`);
183         }
184
185         for (const syncer of this._syncers) {
186             const promise = this._scheduleRequestWithLog(syncer, nextRequest, null);
187             if (promise)
188                 return promise;
189         }
190         return null;
191     }
192
193     _scheduleRequestWithLog(syncer, request, slaveName)
194     {
195         const promise = syncer.scheduleRequestInGroupIfAvailable(request, slaveName);
196         if (!promise)
197             return promise;
198         this._logger.log(`Scheduling build request ${request.id()}${slaveName ? ' on ' + slaveName : ''} in ${syncer.builderName()}`);
199         return promise;
200     }
201
202     static _testGroupMapForBuildRequests(buildRequests)
203     {
204         const map = new Map;
205         let groupOrder = 0;
206         for (let request of buildRequests) {
207             let groupId = request.testGroupId();
208             if (!map.has(groupId)) // Don't use real TestGroup objects to avoid executing postgres query in the server
209                 map.set(groupId, {id: groupId, groupOrder: groupOrder++, requests: [request], syncer: null, slaveName: null});
210             else
211                 map.get(groupId).requests.push(request);
212         }
213         return map;
214     }
215 }
216
217 if (typeof module != 'undefined')
218     module.exports.BuildbotTriggerable = BuildbotTriggerable;