94b5a207e36aa0aa3bcae7bb82021bfa12198183
[WebKit.git] / Websites / perf.webkit.org / tools / js / buildbot-syncer.js
1 'use strict';
2
3 let assert = require('assert');
4
5 require('./v3-models.js');
6
7 class BuildbotBuildEntry {
8     constructor(syncer, rawData)
9     {
10         assert.equal(syncer.builderName(), rawData['builderName']);
11
12         this._syncer = syncer;
13         this._slaveName = null;
14         this._buildRequestId = null;
15         this._isInProgress = rawData['currentStep'] || (rawData['times'] && !rawData['times'][1]);
16         this._buildNumber = rawData['number'];
17
18         for (let propertyTuple of (rawData['properties'] || [])) {
19             // e.g. ['build_request_id', '16733', 'Force Build Form']
20             const name = propertyTuple[0];
21             const value = propertyTuple[1];
22             if (name == syncer._slavePropertyName)
23                 this._slaveName = value;
24             else if (name == syncer._buildRequestPropertyName)
25                 this._buildRequestId = value;
26         }
27     }
28
29     syncer() { return this._syncer; }
30     buildNumber() { return this._buildNumber; }
31     slaveName() { return this._slaveName; }
32     buildRequestId() { return this._buildRequestId; }
33     isPending() { return typeof(this._buildNumber) != 'number'; }
34     isInProgress() { return this._isInProgress; }
35     hasFinished() { return !this.isPending() && !this.isInProgress(); }
36     url() { return this.isPending() ? this._syncer.url() : this._syncer.urlForBuildNumber(this._buildNumber); }
37
38     buildRequestStatusIfUpdateIsNeeded(request)
39     {
40         assert.equal(request.id(), this._buildRequestId);
41         if (!request)
42             return null;
43         if (this.isPending()) {
44             if (request.isPending())
45                 return 'scheduled';
46         } else if (this.isInProgress()) {
47             if (!request.hasStarted() || request.isScheduled())
48                 return 'running';
49         } else if (this.hasFinished()) {
50             if (!request.hasFinished())
51                 return 'failedIfNotCompleted';
52         }
53         return null;
54     }
55 }
56
57
58 class BuildbotSyncer {
59
60     constructor(remote, object)
61     {
62         this._remote = remote;
63         this._testConfigurations = [];
64         this._builderName = object.builder;
65         this._slavePropertyName = object.slaveArgument;
66         this._slaveList = object.slaveList;
67         this._buildRequestPropertyName = object.buildRequestArgument;
68         this._entryList = null;
69         this._slavesWithNewRequests = new Set;
70     }
71
72     builderName() { return this._builderName; }
73
74     addTestConfiguration(test, platform, propertiesTemplate)
75     {
76         assert(test instanceof Test);
77         assert(platform instanceof Platform);
78         this._testConfigurations.push({test, platform, propertiesTemplate});
79     }
80     testConfigurations() { return this._testConfigurations; }
81
82     matchesConfiguration(request)
83     {
84         return this._testConfigurations.some((config) => config.platform == request.platform() && config.test == request.test());
85     }
86
87     scheduleRequest(newRequest, slaveName)
88     {
89         assert(!this._slavesWithNewRequests.has(slaveName));
90         let properties = this._propertiesForBuildRequest(newRequest);
91
92         assert.equal(!this._slavePropertyName, !slaveName);
93         if (this._slavePropertyName)
94             properties[this._slavePropertyName] = slaveName;
95
96         this._slavesWithNewRequests.add(slaveName);
97         return this._remote.postFormUrlencodedData(this.pathForForceBuild(), properties);
98     }
99
100     scheduleRequestInGroupIfAvailable(newRequest, slaveName)
101     {
102         assert(newRequest instanceof BuildRequest);
103
104         if (!this.matchesConfiguration(newRequest))
105             return null;
106
107         let hasPendingBuildsWithoutSlaveNameSpecified = false;
108         let usedSlaves = new Set;
109         for (let entry of this._entryList) {
110             if (entry.isPending()) {
111                 if (!entry.slaveName())
112                     hasPendingBuildsWithoutSlaveNameSpecified = true;
113                 usedSlaves.add(entry.slaveName());
114             }
115         }
116
117         if (!this._slaveList || hasPendingBuildsWithoutSlaveNameSpecified) {
118             if (usedSlaves.size || this._slavesWithNewRequests.size)
119                 return null;
120             return this.scheduleRequest(newRequest, null);
121         }
122
123         if (slaveName) {
124             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
125                 return this.scheduleRequest(newRequest, slaveName);
126             return null;
127         }
128
129         for (let slaveName of this._slaveList) {
130             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
131                 return this.scheduleRequest(newRequest, slaveName);
132         }
133
134         return null;
135     }
136
137     pullBuildbot(count)
138     {
139         return this._remote.getJSON(this.pathForPendingBuildsJSON()).then((content) => {
140             let pendingEntries = content.map((entry) => new BuildbotBuildEntry(this, entry));
141             return this._pullRecentBuilds(count).then((entries) => {
142                 let entryByRequest = {};
143
144                 for (let entry of pendingEntries)
145                     entryByRequest[entry.buildRequestId()] = entry;
146
147                 for (let entry of entries)
148                     entryByRequest[entry.buildRequestId()] = entry;
149
150                 let entryList = [];
151                 for (let id in entryByRequest)
152                     entryList.push(entryByRequest[id]);
153
154                 this._entryList = entryList;
155                 this._slavesWithNewRequests.clear();
156
157                 return entryList;
158             });
159         });
160     }
161
162     _pullRecentBuilds(count)
163     {
164         if (!count)
165             return Promise.resolve([]);
166
167         let selectedBuilds = new Array(count);
168         for (let i = 0; i < count; i++)
169             selectedBuilds[i] = -i - 1;
170
171         return this._remote.getJSON(this.pathForBuildJSON(selectedBuilds)).then((content) => {
172             const entries = [];
173             for (let index of selectedBuilds) {
174                 const entry = content[index];
175                 if (entry && !entry['error'])
176                     entries.push(new BuildbotBuildEntry(this, entry));
177             }
178             return entries;
179         });
180     }
181
182     pathForPendingBuildsJSON() { return `/json/builders/${escape(this._builderName)}/pendingBuilds`; }
183     pathForBuildJSON(selectedBuilds)
184     {
185         return `/json/builders/${escape(this._builderName)}/builds/?` + selectedBuilds.map((number) => 'select=' + number).join('&');
186     }
187     pathForForceBuild() { return `/builders/${escape(this._builderName)}/force`; }
188
189     url() { return this._remote.url(`/builders/${escape(this._builderName)}/`); }
190     urlForBuildNumber(number) { return this._remote.url(`/builders/${escape(this._builderName)}/builds/${number}`); }
191
192     _propertiesForBuildRequest(buildRequest)
193     {
194         assert(buildRequest instanceof BuildRequest);
195
196         const commitSet = buildRequest.commitSet();
197         assert(commitSet instanceof CommitSet);
198
199         const repositoryByName = {};
200         for (let repository of commitSet.repositories())
201             repositoryByName[repository.name()] = repository;
202
203         let propertiesTemplate = null;
204         for (let config of this._testConfigurations) {
205             if (config.platform == buildRequest.platform() && config.test == buildRequest.test())
206                 propertiesTemplate = config.propertiesTemplate;
207         }
208         assert(propertiesTemplate);
209
210         const properties = {};
211         for (let key in propertiesTemplate) {
212             const value = propertiesTemplate[key];
213             if (typeof(value) != 'object')
214                 properties[key] = value;
215             else if ('root' in value) {
216                 const repositoryName = value['root'];
217                 const repository = repositoryByName[repositoryName];
218                 assert(repository, `"${repositoryName}" must be specified`);
219                 properties[key] = commitSet.revisionForRepository(repository);
220             } else if ('rootOptions' in value) {
221                 const filteredOptions = value['rootOptions'].filter((option) => option in repositoryByName);
222                 assert.equal(filteredOptions.length, 1, `There should be exactly one valid root among "${value['rootOptions']}".`);
223                 properties[key] = commitSet.revisionForRepository(repositoryByName[filteredOptions[0]]);
224             }
225             else if ('rootsExcluding' in value) {
226                 const revisionSet = this._revisionSetFromCommitSetWithExclusionList(commitSet, value['rootsExcluding']);
227                 properties[key] = JSON.stringify(revisionSet);
228             }
229         }
230
231         properties[this._buildRequestPropertyName] = buildRequest.id();
232
233         return properties;
234     }
235
236     _revisionSetFromCommitSetWithExclusionList(commitSet, exclusionList)
237     {
238         const revisionSet = {};
239         for (let repository of commitSet.repositories()) {
240             if (exclusionList.indexOf(repository.name()) >= 0)
241                 continue;
242             const commit = commitSet.commitForRepository(repository);
243             revisionSet[repository.name()] = {
244                 id: commit.id(),
245                 time: +commit.time(),
246                 repository: repository.name(),
247                 revision: commit.revision(),
248             };
249         }
250         return revisionSet;
251     }
252
253     static _loadConfig(remote, config)
254     {
255         const shared = config['shared'] || {};
256         const types = config['types'] || {};
257         const builders = config['builders'] || {};
258
259         let syncerByBuilder = new Map;
260         for (let entry of config['configurations']) {
261             const newConfig = {};
262             this._validateAndMergeConfig(newConfig, shared);
263             this._validateAndMergeConfig(newConfig, entry);
264
265             const expandedConfigurations = this._expandTypesAndPlatforms(newConfig);
266             for (let config of expandedConfigurations) {
267                 if ('type' in config) {
268                     const type = config['type'];
269                     assert(type, `${type} is not a valid type in the configuration`);
270                     this._validateAndMergeConfig(config, types[type]);
271                 }
272
273                 const builder = entry['builder'];
274                 if (builders[builder])
275                     this._validateAndMergeConfig(config, builders[builder]);
276
277                 this._createTestConfiguration(remote, syncerByBuilder, config);
278             }
279         }
280
281         return Array.from(syncerByBuilder.values());
282     }
283
284     static _expandTypesAndPlatforms(unresolvedConfig)
285     {
286         const typeExpanded = [];
287         if ('types' in unresolvedConfig) {
288             for (let type of unresolvedConfig['types'])
289                 typeExpanded.push(this._validateAndMergeConfig({'type': type}, unresolvedConfig, 'types'));
290         } else
291             typeExpanded.push(unresolvedConfig);
292
293         const configurations = [];
294         for (let config of typeExpanded) {
295             if ('platforms' in config) {
296                 for (let platform of config['platforms'])
297                     configurations.push(this._validateAndMergeConfig({'platform': platform}, config, 'platforms'));
298             } else
299                 configurations.push(config);
300         }
301
302         return configurations;
303     }
304
305     static _createTestConfiguration(remote, syncerByBuilder, newConfig)
306     {
307         assert('platform' in newConfig, 'configuration must specify a platform');
308         assert('test' in newConfig, 'configuration must specify a test');
309         assert('builder' in newConfig, 'configuration must specify a builder');
310         assert('properties' in newConfig, 'configuration must specify arguments to post on a builder');
311         assert('buildRequestArgument' in newConfig, 'configuration must specify buildRequestArgument');
312
313         const test = Test.findByPath(newConfig.test);
314         assert(test, `${newConfig.test} is not a valid test path`);
315
316         const platform = Platform.findByName(newConfig.platform);
317         assert(platform, `${newConfig.platform} is not a valid platform name`);
318
319         let syncer = syncerByBuilder.get(newConfig.builder);
320         if (!syncer) {
321             syncer = new BuildbotSyncer(remote, newConfig);
322             syncerByBuilder.set(newConfig.builder, syncer);
323         }
324         syncer.addTestConfiguration(test, platform, newConfig.properties);
325     }
326
327     static _validateAndMergeConfig(config, valuesToMerge, excludedProperty)
328     {
329         for (const name in valuesToMerge) {
330             const value = valuesToMerge[name];
331             if (name == excludedProperty)
332                 continue;
333
334             switch (name) {
335             case 'properties': // Fallthrough
336             case 'arguments':
337                 assert.equal(typeof(value), 'object', 'arguments should be a dictionary');
338                 if (!config['properties'])
339                     config['properties'] = {};
340                 this._validateAndMergeProperties(config['properties'], value);
341                 break;
342             case 'test': // Fallthrough
343             case 'slaveList': // Fallthrough
344             case 'platforms':
345             case 'types':
346                 assert(value instanceof Array, `${name} should be an array`);
347                 assert(value.every(function (part) { return typeof part == 'string'; }), `${name} should be an array of strings`);
348                 config[name] = value.slice();
349                 break;
350             case 'type': // Fallthrough
351             case 'builder': // Fallthrough
352             case 'platform': // Fallthrough
353             case 'slaveArgument': // Fallthrough
354             case 'buildRequestArgument':
355                 assert.equal(typeof(value), 'string', `${name} should be of string type`);
356                 config[name] = value;
357                 break;
358             default:
359                 assert(false, `Unrecognized parameter ${name}`);
360             }
361         }
362         return config;
363     }
364
365     static _validateAndMergeProperties(properties, configArguments)
366     {
367         for (let name in configArguments) {
368             const value = configArguments[name];
369             if (typeof(value) == 'string') {
370                 properties[name] = value;
371                 continue;
372             }
373             assert.equal(typeof(value), 'object', 'A argument value must be either a string or a dictionary');
374
375             const keys = Object.keys(value);
376             assert.equal(keys.length, 1, 'arguments value cannot contain more than one key');
377             let namedValue = value[keys[0]];
378             switch (keys[0]) {
379             case 'root':
380                 assert.equal(typeof(namedValue), 'string', 'root name must be a string');
381                 break;
382             case 'rootOptions': // Fallthrough
383             case 'rootsExcluding':
384                 assert(namedValue instanceof Array, `${keys[0]} must specify an array`);
385                 for (let excludedRootName of namedValue)
386                     assert.equal(typeof(excludedRootName), 'string', `${keys[0]} must specify an array of strings`);
387                 namedValue = namedValue.slice();
388                 break;
389             default:
390                 assert(false, `Unrecognized named argument ${keys[0]}`);
391             }
392             properties[name] = {[keys[0]]: namedValue};
393         }
394     }
395
396 }
397
398 if (typeof module != 'undefined') {
399     module.exports.BuildbotSyncer = BuildbotSyncer;
400     module.exports.BuildbotBuildEntry = BuildbotBuildEntry;
401 }