88aed05ab885a11a987a97c42e1de6fa6dbb0da3
[WebKit-https.git] / Websites / perf.webkit.org / tools / js / buildbot-syncer.js
1 'use strict';
2
3 let assert = require('assert');
4
5 require('./v3-models.js');
6
7 class BuildbotBuildEntry {
8     constructor(syncer, rawData)
9     {
10         assert.equal(syncer.builderName(), rawData['builderName']);
11
12         this._syncer = syncer;
13         this._slaveName = null;
14         this._buildRequestId = null;
15         this._isInProgress = rawData['currentStep'] || (rawData['times'] && !rawData['times'][1]);
16         this._buildNumber = rawData['number'];
17
18         for (let propertyTuple of (rawData['properties'] || [])) {
19             // e.g. ['build_request_id', '16733', 'Force Build Form']
20             let name = propertyTuple[0];
21             let value = propertyTuple[1];
22             if (name == syncer._slavePropertyName)
23                 this._slaveName = value;
24             else if (name == syncer._buildRequestPropertyName)
25                 this._buildRequestId = value;
26         }
27     }
28
29     syncer() { return this._syncer; }
30     buildNumber() { return this._buildNumber; }
31     slaveName() { return this._slaveName; }
32     buildRequestId() { return this._buildRequestId; }
33     isPending() { return typeof(this._buildNumber) != 'number'; }
34     isInProgress() { return this._isInProgress; }
35     hasFinished() { return !this.isPending() && !this.isInProgress(); }
36     url() { return this.isPending() ? this._syncer.url() : this._syncer.urlForBuildNumber(this._buildNumber); }
37
38     buildRequestStatusIfUpdateIsNeeded(request)
39     {
40         assert.equal(request.id(), this._buildRequestId);
41         if (!request)
42             return null;
43         if (this.isPending()) {
44             if (request.isPending())
45                 return 'scheduled';
46         } else if (this.isInProgress()) {
47             if (!request.hasStarted() || request.isScheduled())
48                 return 'running';
49         } else if (this.hasFinished()) {
50             if (!request.hasFinished())
51                 return 'failedIfNotCompleted';
52         }
53         return null;
54     }
55 }
56
57
58 class BuildbotSyncer {
59
60     constructor(remote, object)
61     {
62         this._remote = remote;
63         this._testConfigurations = [];
64         this._builderName = object.builder;
65         this._slavePropertyName = object.slaveArgument;
66         this._slaveList = object.slaveList;
67         this._buildRequestPropertyName = object.buildRequestArgument;
68         this._entryList = null;
69         this._slavesWithNewRequests = new Set;
70     }
71
72     builderName() { return this._builderName; }
73
74     addTestConfiguration(test, platform, propertiesTemplate)
75     {
76         assert(test instanceof Test);
77         assert(platform instanceof Platform);
78         this._testConfigurations.push({test: test, platform: platform, propertiesTemplate: propertiesTemplate});
79     }
80     testConfigurations() { return this._testConfigurations; }
81
82     matchesConfiguration(request)
83     {
84         for (let config of this._testConfigurations) {
85             if (config.platform == request.platform() && config.test == request.test())
86                 return true;
87         }
88         return false;
89     }
90
91     scheduleRequest(newRequest, slaveName)
92     {
93         assert(!this._slavesWithNewRequests.has(slaveName));
94         let properties = this._propertiesForBuildRequest(newRequest);
95
96         assert.equal(!this._slavePropertyName, !slaveName);
97         if (this._slavePropertyName)
98             properties[this._slavePropertyName] = slaveName;
99
100         this._slavesWithNewRequests.add(slaveName);
101         return this._remote.postFormUrlencodedData(this.pathForForceBuild(), properties);
102     }
103
104     scheduleRequestInGroupIfAvailable(newRequest, slaveName)
105     {
106         assert(newRequest instanceof BuildRequest);
107
108         if (!this.matchesConfiguration(newRequest))
109             return null;
110
111         let hasPendingBuildsWithoutSlaveNameSpecified = false;
112         let usedSlaves = new Set;
113         for (let entry of this._entryList) {
114             if (entry.isPending()) {
115                 if (!entry.slaveName())
116                     hasPendingBuildsWithoutSlaveNameSpecified = true;
117                 usedSlaves.add(entry.slaveName());
118             }
119         }
120
121         if (!this._slaveList || hasPendingBuildsWithoutSlaveNameSpecified) {
122             if (usedSlaves.size || this._slavesWithNewRequests.size)
123                 return null;
124             return this.scheduleRequest(newRequest, null);
125         }
126
127         if (slaveName) {
128             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
129                 return this.scheduleRequest(newRequest, slaveName);
130             return null;
131         }
132
133         for (let slaveName of this._slaveList) {
134             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
135                 return this.scheduleRequest(newRequest, slaveName);
136         }
137
138         return null;
139     }
140
141     pullBuildbot(count)
142     {
143         let self = this;
144         return this._remote.getJSON(this.pathForPendingBuildsJSON()).then(function (content) {
145             let pendingEntries = content.map(function (entry) { return new BuildbotBuildEntry(self, entry); });
146             return self._pullRecentBuilds(count).then(function (entries) {
147                 let entryByRequest = {};
148
149                 for (let entry of pendingEntries)
150                     entryByRequest[entry.buildRequestId()] = entry;
151
152                 for (let entry of entries)
153                     entryByRequest[entry.buildRequestId()] = entry;
154
155                 let entryList = [];
156                 for (let id in entryByRequest)
157                     entryList.push(entryByRequest[id]);
158
159                 self._entryList = entryList;
160                 self._slavesWithNewRequests.clear();
161
162                 return entryList;
163             });
164         });
165     }
166
167     _pullRecentBuilds(count)
168     {
169         if (!count)
170             return Promise.resolve([]);
171
172         let selectedBuilds = new Array(count);
173         for (let i = 0; i < count; i++)
174             selectedBuilds[i] = -i - 1;
175
176         let self = this;
177         return this._remote.getJSON(this.pathForBuildJSON(selectedBuilds)).then(function (content) {
178             var entries = [];
179             for (let index of selectedBuilds) {
180                 let entry = content[index];
181                 if (entry && !entry['error'])
182                     entries.push(new BuildbotBuildEntry(self, entry));
183             }
184             return entries;
185         });
186     }
187
188     pathForPendingBuildsJSON() { return `/json/builders/${escape(this._builderName)}/pendingBuilds`; }
189     pathForBuildJSON(selectedBuilds)
190     {
191         return `/json/builders/${escape(this._builderName)}/builds/?`
192             + selectedBuilds.map(function (number) { return 'select=' + number; }).join('&');
193     }
194     pathForForceBuild() { return `/builders/${escape(this._builderName)}/force`; }
195
196     url() { return this._remote.url(`/builders/${escape(this._builderName)}/`); }
197     urlForBuildNumber(number) { return this._remote.url(`/builders/${escape(this._builderName)}/builds/${number}`); }
198
199     _propertiesForBuildRequest(buildRequest)
200     {
201         assert(buildRequest instanceof BuildRequest);
202
203         let rootSet = buildRequest.rootSet();
204         assert(rootSet instanceof RootSet);
205
206         let repositoryByName = {};
207         for (let repository of rootSet.repositories())
208             repositoryByName[repository.name()] = repository;
209
210         let propertiesTemplate = null;
211         for (let config of this._testConfigurations) {
212             if (config.platform == buildRequest.platform() && config.test == buildRequest.test())
213                 propertiesTemplate = config.propertiesTemplate;
214         }
215         assert(propertiesTemplate);
216
217         let properties = {};
218         for (let key in propertiesTemplate) {
219             let value = propertiesTemplate[key];
220             if (typeof(value) != 'object')
221                 properties[key] = value;
222             else if ('root' in value) {
223                 let repositoryName = value['root'];
224                 let repository = repositoryByName[repositoryName];
225                 assert(repository, `"${repositoryName}" must be specified`);
226                 properties[key] = rootSet.revisionForRepository(repository);
227             } else if ('rootOptions' in value) {
228                 const filteredOptions = value['rootOptions'].filter((option) => option in repositoryByName);
229                 assert.equal(filteredOptions.length, 1, `There should be exactly one valid root among "${value['rootOptions']}".`);
230                 properties[key] = rootSet.revisionForRepository(repositoryByName[filteredOptions[0]]);
231             }
232             else if ('rootsExcluding' in value) {
233                 let revisionSet = this._revisionSetFromRootSetWithExclusionList(rootSet, value['rootsExcluding']);
234                 properties[key] = JSON.stringify(revisionSet);
235             }
236         }
237
238         properties[this._buildRequestPropertyName] = buildRequest.id();
239
240         return properties;
241     }
242
243     _revisionSetFromRootSetWithExclusionList(rootSet, exclusionList)
244     {
245         let revisionSet = {};
246         for (let repository of rootSet.repositories()) {
247             if (exclusionList.indexOf(repository.name()) >= 0)
248                 continue;
249             let commit = rootSet.commitForRepository(repository);
250             revisionSet[repository.name()] = {
251                 id: commit.id(),
252                 time: +commit.time(),
253                 repository: repository.name(),
254                 revision: commit.revision(),
255             };
256         }
257         return revisionSet;
258     }
259
260     static _loadConfig(remote, config)
261     {
262         let shared = config['shared'] || {};
263         let types = config['types'] || {};
264         let builders = config['builders'] || {};
265
266         let syncerByBuilder = new Map;
267         for (let entry of config['configurations']) {
268             let newConfig = {};
269             this._validateAndMergeConfig(newConfig, shared);
270             this._validateAndMergeConfig(newConfig, entry);
271
272             let expandedConfigurations = this._expandTypesAndPlatforms(newConfig);
273             for (let config of expandedConfigurations) {
274                 if ('type' in config) {
275                     let type = config['type'];
276                     assert(type, `${type} is not a valid type in the configuration`);
277                     this._validateAndMergeConfig(config, types[type]);
278                 }
279
280                 let builder = entry['builder'];
281                 if (builders[builder])
282                     this._validateAndMergeConfig(config, builders[builder]);
283
284                 this._createTestConfiguration(remote, syncerByBuilder, config);
285             }
286         }
287
288         return Array.from(syncerByBuilder.values());
289     }
290
291     static _expandTypesAndPlatforms(unresolvedConfig)
292     {
293         let typeExpanded = [];
294         if ('types' in unresolvedConfig) {
295             for (let type of unresolvedConfig['types'])
296                 typeExpanded.push(this._validateAndMergeConfig({'type': type}, unresolvedConfig, 'types'));
297         } else
298             typeExpanded.push(unresolvedConfig);
299
300         let configurations = [];
301         for (let config of typeExpanded) {
302             if ('platforms' in config) {
303                 for (let platform of config['platforms'])
304                     configurations.push(this._validateAndMergeConfig({'platform': platform}, config, 'platforms'));
305             } else
306                 configurations.push(config);
307         }
308
309         return configurations;
310     }
311
312     static _createTestConfiguration(remote, syncerByBuilder, newConfig)
313     {
314         assert('platform' in newConfig, 'configuration must specify a platform');
315         assert('test' in newConfig, 'configuration must specify a test');
316         assert('builder' in newConfig, 'configuration must specify a builder');
317         assert('properties' in newConfig, 'configuration must specify arguments to post on a builder');
318         assert('buildRequestArgument' in newConfig, 'configuration must specify buildRequestArgument');
319
320         let test = Test.findByPath(newConfig.test);
321         assert(test, `${newConfig.test} is not a valid test path`);
322
323         let platform = Platform.findByName(newConfig.platform);
324         assert(platform, `${newConfig.platform} is not a valid platform name`);
325
326         let syncer = syncerByBuilder.get(newConfig.builder);
327         if (!syncer) {
328             syncer = new BuildbotSyncer(remote, newConfig);
329             syncerByBuilder.set(newConfig.builder, syncer);
330         }
331         syncer.addTestConfiguration(test, platform, newConfig.properties);
332     }
333
334     static _validateAndMergeConfig(config, valuesToMerge, excludedProperty)
335     {
336         for (let name in valuesToMerge) {
337             let value = valuesToMerge[name];
338             if (name == excludedProperty)
339                 continue;
340
341             switch (name) {
342             case 'properties': // Fallthrough
343             case 'arguments':
344                 assert.equal(typeof(value), 'object', 'arguments should be a dictionary');
345                 if (!config['properties'])
346                     config['properties'] = {};
347                 this._validateAndMergeProperties(config['properties'], value);
348                 break;
349             case 'test': // Fallthrough
350             case 'slaveList': // Fallthrough
351             case 'platforms':
352             case 'types':
353                 assert(value instanceof Array, `${name} should be an array`);
354                 assert(value.every(function (part) { return typeof part == 'string'; }), `${name} should be an array of strings`);
355                 config[name] = value.slice();
356                 break;
357             case 'type': // Fallthrough
358             case 'builder': // Fallthrough
359             case 'platform': // Fallthrough
360             case 'slaveArgument': // Fallthrough
361             case 'buildRequestArgument':
362                 assert.equal(typeof(value), 'string', `${name} should be of string type`);
363                 config[name] = value;
364                 break;
365             default:
366                 assert(false, `Unrecognized parameter ${name}`);
367             }
368         }
369         return config;
370     }
371
372     static _validateAndMergeProperties(properties, configArguments)
373     {
374         for (let name in configArguments) {
375             let value = configArguments[name];
376             if (typeof(value) == 'string') {
377                 properties[name] = value;
378                 continue;
379             }
380             assert.equal(typeof(value), 'object', 'A argument value must be either a string or a dictionary');
381
382             let keys = Object.keys(value);
383             assert.equal(keys.length, 1, 'arguments value cannot contain more than one key');
384             let namedValue = value[keys[0]];
385             switch (keys[0]) {
386             case 'root':
387                 assert.equal(typeof(namedValue), 'string', 'root name must be a string');
388                 break;
389             case 'rootOptions': // Fallthrough
390             case 'rootsExcluding':
391                 assert(namedValue instanceof Array, `${keys[0]} must specify an array`);
392                 for (let excludedRootName of namedValue)
393                     assert.equal(typeof(excludedRootName), 'string', `${keys[0]} must specify an array of strings`);
394                 namedValue = namedValue.slice();
395                 break;
396             default:
397                 assert(false, `Unrecognized named argument ${keys[0]}`);
398             }
399             properties[name] = {[keys[0]]: namedValue};
400         }
401     }
402
403 }
404
405 if (typeof module != 'undefined') {
406     module.exports.BuildbotSyncer = BuildbotSyncer;
407     module.exports.BuildbotBuildEntry = BuildbotBuildEntry;
408 }