Remove the code for old syncing script configuration in BuildbotSyncer
[WebKit-https.git] / Websites / perf.webkit.org / tools / js / buildbot-syncer.js
1 'use strict';
2
3 let assert = require('assert');
4
5 require('./v3-models.js');
6
7 class BuildbotBuildEntry {
8     constructor(syncer, rawData)
9     {
10         assert.equal(syncer.builderName(), rawData['builderName']);
11
12         this._syncer = syncer;
13         this._slaveName = null;
14         this._buildRequestId = null;
15         this._isInProgress = rawData['currentStep'] || (rawData['times'] && !rawData['times'][1]);
16         this._buildNumber = rawData['number'];
17
18         for (let propertyTuple of (rawData['properties'] || [])) {
19             // e.g. ['build_request_id', '16733', 'Force Build Form']
20             const name = propertyTuple[0];
21             const value = propertyTuple[1];
22             if (name == syncer._slavePropertyName)
23                 this._slaveName = value;
24             else if (name == syncer._buildRequestPropertyName)
25                 this._buildRequestId = value;
26         }
27     }
28
29     syncer() { return this._syncer; }
30     buildNumber() { return this._buildNumber; }
31     slaveName() { return this._slaveName; }
32     buildRequestId() { return this._buildRequestId; }
33     isPending() { return typeof(this._buildNumber) != 'number'; }
34     isInProgress() { return this._isInProgress; }
35     hasFinished() { return !this.isPending() && !this.isInProgress(); }
36     url() { return this.isPending() ? this._syncer.url() : this._syncer.urlForBuildNumber(this._buildNumber); }
37
38     buildRequestStatusIfUpdateIsNeeded(request)
39     {
40         assert.equal(request.id(), this._buildRequestId);
41         if (!request)
42             return null;
43         if (this.isPending()) {
44             if (request.isPending())
45                 return 'scheduled';
46         } else if (this.isInProgress()) {
47             if (!request.hasStarted() || request.isScheduled())
48                 return 'running';
49         } else if (this.hasFinished()) {
50             if (!request.hasFinished())
51                 return 'failedIfNotCompleted';
52         }
53         return null;
54     }
55 }
56
57
58 class BuildbotSyncer {
59
60     constructor(remote, object, commonConfigurations)
61     {
62         this._remote = remote;
63         this._testConfigurations = [];
64         this._repositoryGroups = commonConfigurations.repositoryGroups;
65         this._slavePropertyName = commonConfigurations.slaveArgument;
66         this._buildRequestPropertyName = commonConfigurations.buildRequestArgument;
67         this._builderName = object.builder;
68         this._slaveList = object.slaveList;
69         this._entryList = null;
70         this._slavesWithNewRequests = new Set;
71     }
72
73     builderName() { return this._builderName; }
74
75     addTestConfiguration(test, platform, propertiesTemplate)
76     {
77         assert(test instanceof Test);
78         assert(platform instanceof Platform);
79         this._testConfigurations.push({test, platform, propertiesTemplate});
80     }
81     testConfigurations() { return this._testConfigurations; }
82     repositoryGroups() { return this._repositoryGroups; }
83
84     matchesConfiguration(request)
85     {
86         return this._testConfigurations.some((config) => config.platform == request.platform() && config.test == request.test());
87     }
88
89     scheduleRequest(newRequest, slaveName)
90     {
91         assert(!this._slavesWithNewRequests.has(slaveName));
92         let properties = this._propertiesForBuildRequest(newRequest);
93
94         assert.equal(!this._slavePropertyName, !slaveName);
95         if (this._slavePropertyName)
96             properties[this._slavePropertyName] = slaveName;
97
98         this._slavesWithNewRequests.add(slaveName);
99         return this._remote.postFormUrlencodedData(this.pathForForceBuild(), properties);
100     }
101
102     scheduleRequestInGroupIfAvailable(newRequest, slaveName)
103     {
104         assert(newRequest instanceof BuildRequest);
105
106         if (!this.matchesConfiguration(newRequest))
107             return null;
108
109         let hasPendingBuildsWithoutSlaveNameSpecified = false;
110         let usedSlaves = new Set;
111         for (let entry of this._entryList) {
112             if (entry.isPending()) {
113                 if (!entry.slaveName())
114                     hasPendingBuildsWithoutSlaveNameSpecified = true;
115                 usedSlaves.add(entry.slaveName());
116             }
117         }
118
119         if (!this._slaveList || hasPendingBuildsWithoutSlaveNameSpecified) {
120             if (usedSlaves.size || this._slavesWithNewRequests.size)
121                 return null;
122             return this.scheduleRequest(newRequest, null);
123         }
124
125         if (slaveName) {
126             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
127                 return this.scheduleRequest(newRequest, slaveName);
128             return null;
129         }
130
131         for (let slaveName of this._slaveList) {
132             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
133                 return this.scheduleRequest(newRequest, slaveName);
134         }
135
136         return null;
137     }
138
139     pullBuildbot(count)
140     {
141         return this._remote.getJSON(this.pathForPendingBuildsJSON()).then((content) => {
142             let pendingEntries = content.map((entry) => new BuildbotBuildEntry(this, entry));
143             return this._pullRecentBuilds(count).then((entries) => {
144                 let entryByRequest = {};
145
146                 for (let entry of pendingEntries)
147                     entryByRequest[entry.buildRequestId()] = entry;
148
149                 for (let entry of entries)
150                     entryByRequest[entry.buildRequestId()] = entry;
151
152                 let entryList = [];
153                 for (let id in entryByRequest)
154                     entryList.push(entryByRequest[id]);
155
156                 this._entryList = entryList;
157                 this._slavesWithNewRequests.clear();
158
159                 return entryList;
160             });
161         });
162     }
163
164     _pullRecentBuilds(count)
165     {
166         if (!count)
167             return Promise.resolve([]);
168
169         let selectedBuilds = new Array(count);
170         for (let i = 0; i < count; i++)
171             selectedBuilds[i] = -i - 1;
172
173         return this._remote.getJSON(this.pathForBuildJSON(selectedBuilds)).then((content) => {
174             const entries = [];
175             for (let index of selectedBuilds) {
176                 const entry = content[index];
177                 if (entry && !entry['error'])
178                     entries.push(new BuildbotBuildEntry(this, entry));
179             }
180             return entries;
181         });
182     }
183
184     pathForPendingBuildsJSON() { return `/json/builders/${escape(this._builderName)}/pendingBuilds`; }
185     pathForBuildJSON(selectedBuilds)
186     {
187         return `/json/builders/${escape(this._builderName)}/builds/?` + selectedBuilds.map((number) => 'select=' + number).join('&');
188     }
189     pathForForceBuild() { return `/builders/${escape(this._builderName)}/force`; }
190
191     url() { return this._remote.url(`/builders/${escape(this._builderName)}/`); }
192     urlForBuildNumber(number) { return this._remote.url(`/builders/${escape(this._builderName)}/builds/${number}`); }
193
194     _propertiesForBuildRequest(buildRequest)
195     {
196         assert(buildRequest instanceof BuildRequest);
197
198         const commitSet = buildRequest.commitSet();
199         assert(commitSet instanceof CommitSet);
200
201         const repositoryByName = {};
202         for (let repository of commitSet.repositories())
203             repositoryByName[repository.name()] = repository;
204
205         const matchingConfiguration = this._testConfigurations.find((config) => config.platform == buildRequest.platform() && config.test == buildRequest.test());
206         assert(matchingConfiguration, `Build request ${buildRequest.id()} does not match a configuration in the builder "${this._builderName}"`);
207         const propertiesTemplate = matchingConfiguration.propertiesTemplate;
208
209         const repositoryGroup = buildRequest.repositoryGroup();
210         assert(repositoryGroup.accepts(commitSet), `Build request ${buildRequest.id()} does not specify a commit set accepted by the repository group ${repositoryGroup.id()}`);
211
212         const repositoryGroupConfiguration = this._repositoryGroups[repositoryGroup.name()];
213         assert(repositoryGroupConfiguration, `Build request ${buildRequest.id()} uses an unsupported repository group "${repositoryGroup.name()}"`);
214
215         const properties = {};
216         for (let propertyName in propertiesTemplate)
217             properties[propertyName] = propertiesTemplate[propertyName];
218
219         const repositoryGroupTemplate = repositoryGroupConfiguration.propertiesTemplate;
220         for (let propertyName in repositoryGroupTemplate) {
221             const value = repositoryGroupTemplate[propertyName];
222             properties[propertyName] = value instanceof Repository ? commitSet.revisionForRepository(value) : value;
223         }
224         properties[this._buildRequestPropertyName] = buildRequest.id();
225
226         return properties;
227     }
228
229     _revisionSetFromCommitSetWithExclusionList(commitSet, exclusionList)
230     {
231         const revisionSet = {};
232         for (let repository of commitSet.repositories()) {
233             if (exclusionList.indexOf(repository.name()) >= 0)
234                 continue;
235             const commit = commitSet.commitForRepository(repository);
236             revisionSet[repository.name()] = {
237                 id: commit.id(),
238                 time: +commit.time(),
239                 repository: repository.name(),
240                 revision: commit.revision(),
241             };
242         }
243         return revisionSet;
244     }
245
246     static _loadConfig(remote, config)
247     {
248         const types = config['types'] || {};
249         const builders = config['builders'] || {};
250
251         assert(config.buildRequestArgument, 'buildRequestArgument must specify the name of the property used to store the build request ID');
252
253         assert.equal(typeof(config.repositoryGroups), 'object', 'repositoryGroups must specify a dictionary from the name to its definition');
254
255         const repositoryGroups = {};
256         for (const name in config.repositoryGroups)
257             repositoryGroups[name] = this._parseRepositoryGroup(name, config.repositoryGroups[name]);
258
259         const commonConfigurations = {
260             repositoryGroups,
261             slaveArgument: config.slaveArgument,
262             buildRequestArgument: config.buildRequestArgument,
263         };
264
265         let syncerByBuilder = new Map;
266         const expandedConfigurations = [];
267         for (let entry of config['configurations']) {
268             for (const expandedConfig of this._expandTypesAndPlatforms(entry))
269                 expandedConfigurations.push(expandedConfig);
270         }
271
272         for (let entry of expandedConfigurations) {
273             const mergedConfig = {};
274             this._validateAndMergeConfig(mergedConfig, entry);
275
276             if ('type' in mergedConfig) {
277                 const type = mergedConfig['type'];
278                 assert(type, `${type} is not a valid type in the configuration`);
279                 this._validateAndMergeConfig(mergedConfig, types[type]);
280             }
281
282             const builder = entry['builder'];
283             if (builders[builder])
284                 this._validateAndMergeConfig(mergedConfig, builders[builder]);
285
286             this._createTestConfiguration(remote, syncerByBuilder, mergedConfig, commonConfigurations);
287         }
288
289         return Array.from(syncerByBuilder.values());
290     }
291
292     static _parseRepositoryGroup(name, group)
293     {
294         assert.equal(typeof(group.repositories), 'object',
295             `Repository group "${name}" does not specify a dictionary of repositories`);
296         assert(!('description' in group) || typeof(group['description']) == 'string',
297             `Repository group "${name}" have an invalid description`);
298         assert.equal(typeof(group.properties), 'object', `Repository group "${name}" specifies an invalid dictionary of properties`);
299         assert([undefined, true, false].includes(group.acceptsRoots),
300             `Repository group "${name}" contains invalid acceptsRoots value: ${group.acceptsRoots}`);
301
302         const repositoryByName = {};
303         const parsedRepositoryList = [];
304         for (const repositoryName in group.repositories) {
305             const options = group.repositories[repositoryName];
306             const repository = Repository.findTopLevelByName(repositoryName);
307             assert(repository, `"${repositoryName}" is not a valid repository name`);
308             repositoryByName[repositoryName] = repository;
309             assert.equal(typeof(options), 'object', `"${repositoryName}" does not specify a valid option`);
310             assert([undefined, true, false].includes(options.acceptsPatch),
311                 `"${repositoryName}" contains invalid acceptsPatch value: ${options.acceptsPatch}`);
312             repositoryByName[repositoryName] = repository;
313             parsedRepositoryList.push({repository: repository.id(), acceptsPatch: options.acceptsPatch});
314         }
315
316         const propertiesTemplate = {};
317         const usedRepositories = [];
318         for (const propertyName in group.properties) {
319             let value = group.properties[propertyName];
320             const match = value.match(/^<(.+)>$/);
321             if (match) {
322                 const repositoryName = match[1];
323                 value = repositoryByName[repositoryName];
324                 assert(value, `Repository group "${name}" uses "${repositoryName}" in its property but does not list in the list of repositories`);
325                 usedRepositories.push(value);
326             }
327             propertiesTemplate[propertyName] = value;
328         }
329         assert(parsedRepositoryList.length, `Repository group "${name}" does not specify any repository`);
330         assert.equal(parsedRepositoryList.length, usedRepositories.length,
331             `Repository group "${name}" does not use some of the repositories listed`);
332         return {
333             name: group.name,
334             description: group.description,
335             acceptsRoots: group.acceptsRoots,
336             propertiesTemplate,
337             repositoryList: parsedRepositoryList,
338         };
339     }
340
341     static _expandTypesAndPlatforms(unresolvedConfig)
342     {
343         const typeExpanded = [];
344         if ('types' in unresolvedConfig) {
345             for (let type of unresolvedConfig['types'])
346                 typeExpanded.push(this._validateAndMergeConfig({'type': type}, unresolvedConfig, 'types'));
347         } else
348             typeExpanded.push(unresolvedConfig);
349
350         const configurations = [];
351         for (let config of typeExpanded) {
352             if ('platforms' in config) {
353                 for (let platform of config['platforms'])
354                     configurations.push(this._validateAndMergeConfig({'platform': platform}, config, 'platforms'));
355             } else
356                 configurations.push(config);
357         }
358
359         return configurations;
360     }
361
362     static _createTestConfiguration(remote, syncerByBuilder, newConfig, commonConfigurations)
363     {
364         assert('platform' in newConfig, 'configuration must specify a platform');
365         assert('test' in newConfig, 'configuration must specify a test');
366         assert('builder' in newConfig, 'configuration must specify a builder');
367
368         const test = Test.findByPath(newConfig.test);
369         assert(test, `${newConfig.test} is not a valid test path`);
370
371         const platform = Platform.findByName(newConfig.platform);
372         assert(platform, `${newConfig.platform} is not a valid platform name`);
373
374         let syncer = syncerByBuilder.get(newConfig.builder);
375         if (!syncer) {
376             syncer = new BuildbotSyncer(remote, newConfig, commonConfigurations);
377             syncerByBuilder.set(newConfig.builder, syncer);
378         }
379         syncer.addTestConfiguration(test, platform, newConfig.properties || {});
380     }
381
382     static _validateAndMergeConfig(config, valuesToMerge, excludedProperty)
383     {
384         for (const name in valuesToMerge) {
385             const value = valuesToMerge[name];
386             if (name == excludedProperty)
387                 continue;
388
389             switch (name) {
390             case 'properties': // Fallthrough
391                 assert.equal(typeof(value), 'object', 'properties should be a dictionary');
392                 if (!config['properties'])
393                     config['properties'] = {};
394                 const properties = config['properties'];
395                 for (const name in value) {
396                     assert.equal(typeof(value[name]), 'string', 'A argument value must be a string');
397                     properties[name] = value[name];
398                 }
399                 break;
400             case 'test': // Fallthrough
401             case 'slaveList': // Fallthrough
402             case 'platforms': // Fallthrough
403             case 'types':
404                 assert(value instanceof Array, `${name} should be an array`);
405                 assert(value.every(function (part) { return typeof part == 'string'; }), `${name} should be an array of strings`);
406                 config[name] = value.slice();
407                 break;
408             case 'type': // Fallthrough
409             case 'builder': // Fallthrough
410             case 'platform': // Fallthrough
411                 assert.equal(typeof(value), 'string', `${name} should be of string type`);
412                 config[name] = value;
413                 break;
414             default:
415                 assert(false, `Unrecognized parameter ${name}`);
416             }
417         }
418         return config;
419     }
420
421 }
422
423 if (typeof module != 'undefined') {
424     module.exports.BuildbotSyncer = BuildbotSyncer;
425     module.exports.BuildbotBuildEntry = BuildbotBuildEntry;
426 }