429a41e8075a3cdc5f1b3ea91474abab841d1e87
[WebKit-https.git] / Websites / perf.webkit.org / tools / js / buildbot-triggerable.js
1 'use strict';
2
3 let assert = require('assert');
4
5 require('./v3-models.js');
6
7 let BuildbotSyncer = require('./buildbot-syncer').BuildbotSyncer;
8
9 class BuildbotTriggerable {
10     constructor(config, remote, buildbotRemote, slaveInfo, logger)
11     {
12         this._name = config.triggerableName;
13         assert(typeof(this._name) == 'string', 'triggerableName must be specified');
14
15         this._lookbackCount = config.lookbackCount;
16         assert(typeof(this._lookbackCount) == 'number' && this._lookbackCount > 0, 'lookbackCount must be a number greater than 0');
17
18         this._remote = remote;
19
20         this._slaveInfo = slaveInfo;
21         assert(typeof(slaveInfo.name) == 'string', 'slave name must be specified');
22         assert(typeof(slaveInfo.password) == 'string', 'slave password must be specified');
23
24         this._syncers = BuildbotSyncer._loadConfig(buildbotRemote, config);
25         this._logger = logger || {log: () => { }, error: () => { }};
26     }
27
28     name() { return this._name; }
29
30     updateTriggerable()
31     {
32         const map = new Map;
33         let repositoryGroups = [];
34         for (const syncer of this._syncers) {
35             for (const config of syncer.testConfigurations()) {
36                 const entry = {test: config.test.id(), platform: config.platform.id()};
37                 map.set(entry.test + '-' + entry.platform, entry);
38             }
39             // FIXME: Move BuildbotSyncer._loadConfig here and store repository groups directly.
40             repositoryGroups = syncer.repositoryGroups();
41         }
42         return this._remote.postJSONWithStatus(`/api/update-triggerable/`, {
43             'slaveName': this._slaveInfo.name,
44             'slavePassword': this._slaveInfo.password,
45             'triggerable': this._name,
46             'configurations': Array.from(map.values()),
47             'repositoryGroups': Object.keys(repositoryGroups).map((groupName) => {
48                 return {name: groupName, repositories: repositoryGroups[groupName].repositories};
49             })});
50     }
51
52     syncOnce()
53     {
54         let syncerList = this._syncers;
55         let buildReqeustsByGroup = new Map;
56
57         this._logger.log(`Fetching build requests for ${this._name}...`);
58         let validRequests;
59         return BuildRequest.fetchForTriggerable(this._name).then((buildRequests) => {
60             validRequests = this._validateRequests(buildRequests);
61             buildReqeustsByGroup = BuildbotTriggerable._testGroupMapForBuildRequests(buildRequests);
62             return this._pullBuildbotOnAllSyncers(buildReqeustsByGroup);
63         }).then((updates) => {
64             this._logger.log('Scheduling builds');
65             const promistList = [];
66             const testGroupList = Array.from(buildReqeustsByGroup.values()).sort(function (a, b) { return a.groupOrder - b.groupOrder; });
67             for (const group of testGroupList) {
68                 const nextRequest = this._nextRequestInGroup(group, updates);
69                 if (!validRequests.has(nextRequest))
70                     continue;
71                 const promise = this._scheduleRequestIfSlaveIsAvailable(nextRequest, group.syncer, group.slaveName);
72                 if (promise)
73                     promistList.push(promise);
74             }
75             return Promise.all(promistList);
76         }).then(() => {
77             // Pull all buildbots for the second time since the previous step may have scheduled more builds.
78             return this._pullBuildbotOnAllSyncers(buildReqeustsByGroup);
79         }).then((updates) => {
80             // FIXME: Add a new API that just updates the requests.
81             return this._remote.postJSONWithStatus(`/api/build-requests/${this._name}`, {
82                 'slaveName': this._slaveInfo.name,
83                 'slavePassword': this._slaveInfo.password,
84                 'buildRequestUpdates': updates});
85         });
86     }
87
88     _validateRequests(buildRequests)
89     {
90         const testPlatformPairs = {};
91         const validatedRequests = new Set;
92         for (let request of buildRequests) {
93             if (!this._syncers.some((syncer) => syncer.matchesConfiguration(request))) {
94                 const key = request.platform().id + '-' + request.test().id();
95                 if (!(key in testPlatformPairs))
96                     this._logger.error(`Build request ${request.id()} has no matching configuration: "${request.test().fullName()}" on "${request.platform().name()}".`);
97                 testPlatformPairs[key] = true;
98                 continue;
99             }
100             const triggerable = request.triggerable();
101             if (!triggerable) {
102                 this._logger.error(`Build request ${request.id()} does not specify a valid triggerable`);
103                 continue;
104             }
105             assert(triggerable instanceof Triggerable, 'Must specify a valid triggerable');
106             assert.equal(triggerable.name(), this._name, 'Must specify the triggerable of this syncer');
107             const repositoryGroup = request.repositoryGroup();
108             if (!repositoryGroup) {
109                 this._logger.error(`Build request ${request.id()} does not specify a repository group. Such a build request is no longer supported.`);
110                 continue;
111             }
112             const acceptedGroups = triggerable.repositoryGroups();
113             if (!acceptedGroups.includes(repositoryGroup)) {
114                 const acceptedNames = acceptedGroups.map((group) => group.name()).join(', ');
115                 this._logger.error(`Build request ${request.id()} specifies ${repositoryGroup.name()} but triggerable ${this._name} only accepts ${acceptedNames}`);
116                 continue;
117             }
118             validatedRequests.add(request);
119         }
120
121         return validatedRequests;
122     }
123
124     _pullBuildbotOnAllSyncers(buildReqeustsByGroup)
125     {
126         let updates = {};
127         let associatedRequests = new Set;
128         return Promise.all(this._syncers.map((syncer) => {
129             return syncer.pullBuildbot(this._lookbackCount).then((entryList) => {
130                 for (const entry of entryList) {
131                     const request = BuildRequest.findById(entry.buildRequestId());
132                     if (!request)
133                         continue;
134                     associatedRequests.add(request);
135
136                     const info = buildReqeustsByGroup.get(request.testGroupId());
137                     assert(!info.syncer || info.syncer == syncer);
138                     info.syncer = syncer;
139                     if (entry.slaveName()) {
140                         assert(!info.slaveName || info.slaveName == entry.slaveName());
141                         info.slaveName = entry.slaveName();
142                     }
143
144                     const newStatus = entry.buildRequestStatusIfUpdateIsNeeded(request);
145                     if (newStatus) {
146                         this._logger.log(`Updating the status of build request ${request.id()} from ${request.status()} to ${newStatus}`);
147                         updates[entry.buildRequestId()] = {status: newStatus, url: entry.url()};
148                     } else if (!request.statusUrl()) {
149                         this._logger.log(`Setting the status URL of build request ${request.id()} to ${entry.url()}`);
150                         updates[entry.buildRequestId()] = {status: request.status(), url: entry.url()};
151                     }
152                 }
153             });
154         })).then(() => {
155             for (const request of BuildRequest.all()) {
156                 if (request.hasStarted() && !request.hasFinished() && !associatedRequests.has(request)) {
157                     this._logger.log(`Updating the status of build request ${request.id()} from ${request.status()} to failedIfNotCompleted`);
158                     assert(!(request.id() in updates));
159                     updates[request.id()] = {status: 'failedIfNotCompleted'};
160                 }
161             }
162         }).then(() => updates);
163     }
164
165     _nextRequestInGroup(groupInfo, pendingUpdates)
166     {
167         for (const request of groupInfo.requests) {
168             if (request.isScheduled() || (request.id() in pendingUpdates && pendingUpdates[request.id()]['status'] == 'scheduled'))
169                 return null;
170             if (request.isPending() && !(request.id() in pendingUpdates))
171                 return request;
172         }
173         return null;
174     }
175
176     _scheduleRequestIfSlaveIsAvailable(nextRequest, syncer, slaveName)
177     {
178         if (!nextRequest)
179             return null;
180
181         if (!!nextRequest.order()) {
182             if (syncer)
183                 return this._scheduleRequestWithLog(syncer, nextRequest, slaveName);
184             this._logger.error(`Could not identify the syncer for ${nextRequest.id()}.`);
185         }
186
187         for (const syncer of this._syncers) {
188             const promise = this._scheduleRequestWithLog(syncer, nextRequest, null);
189             if (promise)
190                 return promise;
191         }
192         return null;
193     }
194
195     _scheduleRequestWithLog(syncer, request, slaveName)
196     {
197         const promise = syncer.scheduleRequestInGroupIfAvailable(request, slaveName);
198         if (!promise)
199             return promise;
200         this._logger.log(`Scheduling build request ${request.id()}${slaveName ? ' on ' + slaveName : ''} in ${syncer.builderName()}`);
201         return promise;
202     }
203
204     static _testGroupMapForBuildRequests(buildRequests)
205     {
206         const map = new Map;
207         let groupOrder = 0;
208         for (let request of buildRequests) {
209             let groupId = request.testGroupId();
210             if (!map.has(groupId)) // Don't use real TestGroup objects to avoid executing postgres query in the server
211                 map.set(groupId, {id: groupId, groupOrder: groupOrder++, requests: [request], syncer: null, slaveName: null});
212             else
213                 map.get(groupId).requests.push(request);
214         }
215         return map;
216     }
217 }
218
219 if (typeof module != 'undefined')
220     module.exports.BuildbotTriggerable = BuildbotTriggerable;