af60dea214663c0c05b5bfea8d8ce7c3c67da6ea
[WebKit-https.git] / Websites / perf.webkit.org / tools / js / buildbot-syncer.js
1 'use strict';
2
3 let assert = require('assert');
4
5 require('./v3-models.js');
6
7 class BuildbotBuildEntry {
8     constructor(syncer, rawData)
9     {
10         assert.equal(syncer.builderName(), rawData['builderName']);
11
12         this._syncer = syncer;
13         this._slaveName = null;
14         this._buildRequestId = null;
15         this._isInProgress = rawData['currentStep'] || (rawData['times'] && !rawData['times'][1]);
16         this._buildNumber = rawData['number'];
17
18         for (let propertyTuple of (rawData['properties'] || [])) {
19             // e.g. ['build_request_id', '16733', 'Force Build Form']
20             let name = propertyTuple[0];
21             let value = propertyTuple[1];
22             if (name == syncer._slavePropertyName)
23                 this._slaveName = value;
24             else if (name == syncer._buildRequestPropertyName)
25                 this._buildRequestId = value;
26         }
27     }
28
29     syncer() { return this._syncer; }
30     buildNumber() { return this._buildNumber; }
31     slaveName() { return this._slaveName; }
32     buildRequestId() { return this._buildRequestId; }
33     isPending() { return typeof(this._buildNumber) != 'number'; }
34     isInProgress() { return this._isInProgress; }
35     hasFinished() { return !this.isPending() && !this.isInProgress(); }
36     url() { return this.isPending() ? this._syncer.url() : this._syncer.urlForBuildNumber(this._buildNumber); }
37
38     buildRequestStatusIfUpdateIsNeeded(request)
39     {
40         assert.equal(request.id(), this._buildRequestId);
41         if (!request)
42             return null;
43         if (this.isPending()) {
44             if (request.isPending())
45                 return 'scheduled';
46         } else if (this.isInProgress()) {
47             if (!request.hasStarted() || request.isScheduled())
48                 return 'running';
49         } else if (this.hasFinished()) {
50             if (!request.hasFinished())
51                 return 'failedIfNotCompleted';
52         }
53         return null;
54     }
55 }
56
57
58 class BuildbotSyncer {
59
60     constructor(remote, object)
61     {
62         this._remote = remote;
63         this._testConfigurations = [];
64         this._builderName = object.builder;
65         this._slavePropertyName = object.slaveArgument;
66         this._slaveList = object.slaveList;
67         this._buildRequestPropertyName = object.buildRequestArgument;
68         this._entryList = null;
69         this._slavesWithNewRequests = new Set;
70     }
71
72     builderName() { return this._builderName; }
73
74     addTestConfiguration(test, platform, propertiesTemplate)
75     {
76         assert(test instanceof Test);
77         assert(platform instanceof Platform);
78         this._testConfigurations.push({test: test, platform: platform, propertiesTemplate: propertiesTemplate});
79     }
80     testConfigurations() { return this._testConfigurations; }
81
82     matchesConfiguration(request)
83     {
84         for (let config of this._testConfigurations) {
85             if (config.platform == request.platform() && config.test == request.test())
86                 return true;
87         }
88         return false;
89     }
90
91     scheduleRequest(newRequest, slaveName)
92     {
93         assert(!this._slavesWithNewRequests.has(slaveName));
94         let properties = this._propertiesForBuildRequest(newRequest);
95
96         assert.equal(!this._slavePropertyName, !slaveName);
97         if (this._slavePropertyName)
98             properties[this._slavePropertyName] = slaveName;
99
100         this._slavesWithNewRequests.add(slaveName);
101         return this._remote.postFormUrlencodedData(this.pathForForceBuild(), properties);
102     }
103
104     scheduleRequestInGroupIfAvailable(newRequest, slaveName)
105     {
106         assert(newRequest instanceof BuildRequest);
107
108         if (!this.matchesConfiguration(newRequest))
109             return null;
110
111         let hasPendingBuildsWithoutSlaveNameSpecified = false;
112         let usedSlaves = new Set;
113         for (let entry of this._entryList) {
114             if (entry.isPending()) {
115                 if (!entry.slaveName())
116                     hasPendingBuildsWithoutSlaveNameSpecified = true;
117                 usedSlaves.add(entry.slaveName());
118             }
119         }
120
121         if (!this._slaveList || hasPendingBuildsWithoutSlaveNameSpecified) {
122             if (usedSlaves.size || this._slavesWithNewRequests.size)
123                 return null;
124             return this.scheduleRequest(newRequest, null);
125         }
126
127         if (slaveName) {
128             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
129                 return this.scheduleRequest(newRequest, slaveName);
130             return null;
131         }
132
133         for (let slaveName of this._slaveList) {
134             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
135                 return this.scheduleRequest(newRequest, slaveName);
136         }
137
138         return null;
139     }
140
141     pullBuildbot(count)
142     {
143         let self = this;
144         return this._remote.getJSON(this.pathForPendingBuildsJSON()).then(function (content) {
145             let pendingEntries = content.map(function (entry) { return new BuildbotBuildEntry(self, entry); });
146             return self._pullRecentBuilds(count).then(function (entries) {
147                 let entryByRequest = {};
148
149                 for (let entry of pendingEntries)
150                     entryByRequest[entry.buildRequestId()] = entry;
151
152                 for (let entry of entries)
153                     entryByRequest[entry.buildRequestId()] = entry;
154
155                 let entryList = [];
156                 for (let id in entryByRequest)
157                     entryList.push(entryByRequest[id]);
158
159                 self._entryList = entryList;
160                 self._slavesWithNewRequests.clear();
161
162                 return entryList;
163             });
164         });
165     }
166
167     _pullRecentBuilds(count)
168     {
169         if (!count)
170             return Promise.resolve([]);
171
172         let selectedBuilds = new Array(count);
173         for (let i = 0; i < count; i++)
174             selectedBuilds[i] = -i - 1;
175
176         let self = this;
177         return this._remote.getJSON(this.pathForBuildJSON(selectedBuilds)).then(function (content) {
178             var entries = [];
179             for (let index of selectedBuilds) {
180                 let entry = content[index];
181                 if (entry && !entry['error'])
182                     entries.push(new BuildbotBuildEntry(self, entry));
183             }
184             return entries;
185         });
186     }
187
188     pathForPendingBuildsJSON() { return `/json/builders/${escape(this._builderName)}/pendingBuilds`; }
189     pathForBuildJSON(selectedBuilds)
190     {
191         return `/json/builders/${escape(this._builderName)}/builds/?`
192             + selectedBuilds.map(function (number) { return 'select=' + number; }).join('&');
193     }
194     pathForForceBuild() { return `/builders/${escape(this._builderName)}/force`; }
195
196     url() { return this._remote.url(`/builders/${escape(this._builderName)}/`); }
197     urlForBuildNumber(number) { return this._remote.url(`/builders/${escape(this._builderName)}/builds/${number}`); }
198
199     _propertiesForBuildRequest(buildRequest)
200     {
201         assert(buildRequest instanceof BuildRequest);
202
203         let rootSet = buildRequest.rootSet();
204         assert(rootSet instanceof RootSet);
205
206         let repositoryByName = {};
207         for (let repository of rootSet.repositories())
208             repositoryByName[repository.name()] = repository;
209
210         let propertiesTemplate = null;
211         for (let config of this._testConfigurations) {
212             if (config.platform == buildRequest.platform() && config.test == buildRequest.test())
213                 propertiesTemplate = config.propertiesTemplate;
214         }
215         assert(propertiesTemplate);
216
217         let properties = {};
218         for (let key in propertiesTemplate) {
219             let value = propertiesTemplate[key];
220             if (typeof(value) != 'object')
221                 properties[key] = value;
222             else if ('root' in value) {
223                 let repositoryName = value['root'];
224                 let repository = repositoryByName[repositoryName];
225                 assert(repository, '"${repositoryName}" must be specified');
226                 properties[key] = rootSet.revisionForRepository(repository);
227             } else if ('rootsExcluding' in value) {
228                 let revisionSet = this._revisionSetFromRootSetWithExclusionList(rootSet, value['rootsExcluding']);
229                 properties[key] = JSON.stringify(revisionSet);
230             }
231         }
232
233         properties[this._buildRequestPropertyName] = buildRequest.id();
234
235         return properties;
236     }
237
238     _revisionSetFromRootSetWithExclusionList(rootSet, exclusionList)
239     {
240         let revisionSet = {};
241         for (let repository of rootSet.repositories()) {
242             if (exclusionList.indexOf(repository.name()) >= 0)
243                 continue;
244             let commit = rootSet.commitForRepository(repository);
245             revisionSet[repository.name()] = {
246                 id: commit.id(),
247                 time: +commit.time(),
248                 repository: repository.name(),
249                 revision: commit.revision(),
250             };
251         }
252         return revisionSet;
253     }
254
255     static _loadConfig(remote, config)
256     {
257         let shared = config['shared'] || {};
258         let types = config['types'] || {};
259         let builders = config['builders'] || {};
260
261         let syncerByBuilder = new Map;
262         for (let entry of config['configurations']) {
263             let newConfig = {};
264             this._validateAndMergeConfig(newConfig, shared);
265             this._validateAndMergeConfig(newConfig, entry);
266
267             let expandedConfigurations = this._expandTypesAndPlatforms(newConfig);
268             for (let config of expandedConfigurations) {
269                 if ('type' in config) {
270                     let type = config['type'];
271                     assert(type, `${type} is not a valid type in the configuration`);
272                     this._validateAndMergeConfig(config, types[type]);
273                 }
274
275                 let builder = entry['builder'];
276                 if (builders[builder])
277                     this._validateAndMergeConfig(config, builders[builder]);
278
279                 this._createTestConfiguration(remote, syncerByBuilder, config);
280             }
281         }
282
283         return Array.from(syncerByBuilder.values());
284     }
285
286     static _expandTypesAndPlatforms(unresolvedConfig)
287     {
288         let typeExpanded = [];
289         if ('types' in unresolvedConfig) {
290             for (let type of unresolvedConfig['types'])
291                 typeExpanded.push(this._validateAndMergeConfig({'type': type}, unresolvedConfig, 'types'));
292         } else
293             typeExpanded.push(unresolvedConfig);
294
295         let configurations = [];
296         for (let config of typeExpanded) {
297             if ('platforms' in config) {
298                 for (let platform of config['platforms'])
299                     configurations.push(this._validateAndMergeConfig({'platform': platform}, config, 'platforms'));
300             } else
301                 configurations.push(config);
302         }
303
304         return configurations;
305     }
306
307     static _createTestConfiguration(remote, syncerByBuilder, newConfig)
308     {
309         assert('platform' in newConfig, 'configuration must specify a platform');
310         assert('test' in newConfig, 'configuration must specify a test');
311         assert('builder' in newConfig, 'configuration must specify a builder');
312         assert('properties' in newConfig, 'configuration must specify arguments to post on a builder');
313         assert('buildRequestArgument' in newConfig, 'configuration must specify buildRequestArgument');
314
315         let test = Test.findByPath(newConfig.test);
316         assert(test, `${newConfig.test} is not a valid test path`);
317
318         let platform = Platform.findByName(newConfig.platform);
319         assert(platform, `${newConfig.platform} is not a valid platform name`);
320
321         let syncer = syncerByBuilder.get(newConfig.builder);
322         if (!syncer) {
323             syncer = new BuildbotSyncer(remote, newConfig);
324             syncerByBuilder.set(newConfig.builder, syncer);
325         }
326         syncer.addTestConfiguration(test, platform, newConfig.properties);
327     }
328
329     static _validateAndMergeConfig(config, valuesToMerge, excludedProperty)
330     {
331         for (let name in valuesToMerge) {
332             let value = valuesToMerge[name];
333             if (name == excludedProperty)
334                 continue;
335
336             switch (name) {
337             case 'properties': // fallthrough
338             case 'arguments':
339                 assert.equal(typeof(value), 'object', 'arguments should be a dictionary');
340                 if (!config['properties'])
341                     config['properties'] = {};
342                 this._validateAndMergeProperties(config['properties'], value);
343                 break;
344             case 'test': // fallthrough
345             case 'slaveList': // fallthrough
346             case 'platforms':
347             case 'types':
348                 assert(value instanceof Array, `${name} should be an array`);
349                 assert(value.every(function (part) { return typeof part == 'string'; }), `${name} should be an array of strings`);
350                 config[name] = value.slice();
351                 break;
352             case 'type': // fallthrough
353             case 'builder': // fallthrough
354             case 'platform': // fallthrough
355             case 'slaveArgument': // fallthrough
356             case 'buildRequestArgument':
357                 assert.equal(typeof(value), 'string', `${name} should be of string type`);
358                 config[name] = value;
359                 break;
360             default:
361                 assert(false, `Unrecognized parameter ${name}`);
362             }
363         }
364         return config;
365     }
366
367     static _validateAndMergeProperties(properties, configArguments)
368     {
369         for (let name in configArguments) {
370             let value = configArguments[name];
371             if (typeof(value) == 'string') {
372                 properties[name] = value;
373                 continue;
374             }
375             assert.equal(typeof(value), 'object', 'A argument value must be either a string or a dictionary');
376                 
377             let keys = Object.keys(value);
378             assert.equal(keys.length, 1, 'arguments value cannot contain more than one key');
379             let namedValue = value[keys[0]];
380             switch (keys[0]) {
381             case 'root':
382                 assert.equal(typeof(namedValue), 'string', 'root name must be a string');
383                 break;
384             case 'rootsExcluding':
385                 assert(namedValue instanceof Array, 'rootsExcluding must specify an array');
386                 for (let excludedRootName of namedValue)
387                     assert.equal(typeof(excludedRootName), 'string', 'rootsExcluding must specify an array of strings');
388                 namedValue = namedValue.slice();
389                 break;
390             default:
391                 assert(false, `Unrecognized named argument ${keys[0]}`);
392             }
393             properties[name] = {[keys[0]]: namedValue};
394         }
395     }
396
397 }
398
399 if (typeof module != 'undefined') {
400     module.exports.BuildbotSyncer = BuildbotSyncer;
401     module.exports.BuildbotBuildEntry = BuildbotBuildEntry;
402 }