Add support for builderNameToIDMap in BuildbotSyncer
[WebKit-https.git] / Websites / perf.webkit.org / tools / js / buildbot-syncer.js
1 'use strict';
2
3 let assert = require('assert');
4
5 require('./v3-models.js');
6
7 class BuildbotBuildEntry {
8     constructor(syncer, rawData)
9     {
10         assert.equal(syncer.builderName(), rawData['builderName']);
11
12         this._syncer = syncer;
13         this._slaveName = null;
14         this._buildRequestId = null;
15         this._isInProgress = rawData['currentStep'] || (rawData['times'] && !rawData['times'][1]);
16         this._buildNumber = rawData['number'];
17
18         for (let propertyTuple of (rawData['properties'] || [])) {
19             // e.g. ['build_request_id', '16733', 'Force Build Form']
20             const name = propertyTuple[0];
21             const value = propertyTuple[1];
22             if (name == syncer._slavePropertyName)
23                 this._slaveName = value;
24             else if (name == syncer._buildRequestPropertyName)
25                 this._buildRequestId = value;
26         }
27     }
28
29     syncer() { return this._syncer; }
30     buildNumber() { return this._buildNumber; }
31     slaveName() { return this._slaveName; }
32     buildRequestId() { return this._buildRequestId; }
33     isPending() { return typeof(this._buildNumber) != 'number'; }
34     isInProgress() { return this._isInProgress; }
35     hasFinished() { return !this.isPending() && !this.isInProgress(); }
36     url() { return this.isPending() ? this._syncer.url() : this._syncer.urlForBuildNumber(this._buildNumber); }
37
38     buildRequestStatusIfUpdateIsNeeded(request)
39     {
40         assert.equal(request.id(), this._buildRequestId);
41         if (!request)
42             return null;
43         if (this.isPending()) {
44             if (request.isPending())
45                 return 'scheduled';
46         } else if (this.isInProgress()) {
47             if (!request.hasStarted() || request.isScheduled())
48                 return 'running';
49         } else if (this.hasFinished()) {
50             if (!request.hasFinished())
51                 return 'failedIfNotCompleted';
52         }
53         return null;
54     }
55 }
56
57
58 class BuildbotSyncer {
59
60     constructor(remote, object, commonConfigurations)
61     {
62         this._remote = remote;
63         this._type = null;
64         this._configurations = [];
65         this._repositoryGroups = commonConfigurations.repositoryGroups;
66         this._slavePropertyName = commonConfigurations.slaveArgument;
67         this._platformPropertyName = commonConfigurations.platformArgument;
68         this._buildRequestPropertyName = commonConfigurations.buildRequestArgument;
69         this._builderName = object.builder;
70         this._builderID = object.builderID;
71         this._slaveList = object.slaveList;
72         this._entryList = null;
73         this._slavesWithNewRequests = new Set;
74     }
75
76     builderName() { return this._builderName; }
77
78     addTestConfiguration(test, platform, propertiesTemplate)
79     {
80         assert(test instanceof Test);
81         assert(platform instanceof Platform);
82         assert(this._type == null || this._type == 'tester');
83         this._type = 'tester';
84         this._configurations.push({test, platform, propertiesTemplate});
85     }
86     testConfigurations() { return this._type == 'tester' ? this._configurations : []; }
87
88     addBuildConfiguration(platform, propertiesTemplate)
89     {
90         assert(platform instanceof Platform);
91         assert(this._type == null || this._type == 'builder');
92         this._type = 'builder';
93         this._configurations.push({test: null, platform, propertiesTemplate});
94     }
95     buildConfigurations() { return this._type == 'builder' ? this._configurations : []; }
96
97     isTester() { return this._type == 'tester'; }
98
99     repositoryGroups() { return this._repositoryGroups; }
100
101     matchesConfiguration(request)
102     {
103         return this._configurations.some((config) => config.platform == request.platform() && config.test == request.test());
104     }
105
106     scheduleRequest(newRequest, requestsInGroup, slaveName)
107     {
108         assert(!this._slavesWithNewRequests.has(slaveName));
109         let properties = this._propertiesForBuildRequest(newRequest, requestsInGroup);
110
111         assert.equal(!this._slavePropertyName, !slaveName);
112         if (this._slavePropertyName)
113             properties[this._slavePropertyName] = slaveName;
114
115         if (this._platformPropertyName)
116             properties[this._platformPropertyName] = newRequest.platform().name();
117
118         this._slavesWithNewRequests.add(slaveName);
119         return this._remote.postFormUrlencodedData(this.pathForForceBuild(), properties);
120     }
121
122     scheduleRequestInGroupIfAvailable(newRequest, requestsInGroup, slaveName)
123     {
124         assert(newRequest instanceof BuildRequest);
125
126         if (!this.matchesConfiguration(newRequest))
127             return null;
128
129         let hasPendingBuildsWithoutSlaveNameSpecified = false;
130         let usedSlaves = new Set;
131         for (let entry of this._entryList) {
132             let entryPreventsNewRequest = entry.isPending();
133             if (entry.isInProgress()) {
134                 const requestInProgress = BuildRequest.findById(entry.buildRequestId());
135                 if (!requestInProgress || requestInProgress.testGroupId() != newRequest.testGroupId())
136                     entryPreventsNewRequest = true;
137             }
138             if (entryPreventsNewRequest) {
139                 if (!entry.slaveName())
140                     hasPendingBuildsWithoutSlaveNameSpecified = true;
141                 usedSlaves.add(entry.slaveName());
142             }
143         }
144
145         if (!this._slaveList || hasPendingBuildsWithoutSlaveNameSpecified) {
146             if (usedSlaves.size || this._slavesWithNewRequests.size)
147                 return null;
148             return this.scheduleRequest(newRequest, requestsInGroup, null);
149         }
150
151         if (slaveName) {
152             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
153                 return this.scheduleRequest(newRequest, requestsInGroup, slaveName);
154             return null;
155         }
156
157         for (let slaveName of this._slaveList) {
158             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
159                 return this.scheduleRequest(newRequest, requestsInGroup, slaveName);
160         }
161
162         return null;
163     }
164
165     pullBuildbot(count)
166     {
167         return this._remote.getJSON(this.pathForPendingBuildsJSON()).then((content) => {
168             let pendingEntries = content.map((entry) => new BuildbotBuildEntry(this, entry));
169             return this._pullRecentBuilds(count).then((entries) => {
170                 let entryByRequest = {};
171
172                 for (let entry of pendingEntries)
173                     entryByRequest[entry.buildRequestId()] = entry;
174
175                 for (let entry of entries)
176                     entryByRequest[entry.buildRequestId()] = entry;
177
178                 let entryList = [];
179                 for (let id in entryByRequest)
180                     entryList.push(entryByRequest[id]);
181
182                 this._entryList = entryList;
183                 this._slavesWithNewRequests.clear();
184
185                 return entryList;
186             });
187         });
188     }
189
190     _pullRecentBuilds(count)
191     {
192         if (!count)
193             return Promise.resolve([]);
194
195         let selectedBuilds = new Array(count);
196         for (let i = 0; i < count; i++)
197             selectedBuilds[i] = -i - 1;
198
199         return this._remote.getJSON(this.pathForBuildJSON(selectedBuilds)).then((content) => {
200             const entries = [];
201             for (let index of selectedBuilds) {
202                 const entry = content[index];
203                 if (entry && !entry['error'])
204                     entries.push(new BuildbotBuildEntry(this, entry));
205             }
206             return entries;
207         });
208     }
209
210     pathForPendingBuildsJSON() { return `/json/builders/${escape(this._builderName)}/pendingBuilds`; }
211     pathForBuildJSON(selectedBuilds)
212     {
213         return `/json/builders/${escape(this._builderName)}/builds/?` + selectedBuilds.map((number) => 'select=' + number).join('&');
214     }
215     pathForForceBuild() { return `/builders/${escape(this._builderName)}/force`; }
216
217     url() { return this._remote.url(`/builders/${escape(this._builderName)}/`); }
218     urlForBuildNumber(number) { return this._remote.url(`/builders/${escape(this._builderName)}/builds/${number}`); }
219
220     _propertiesForBuildRequest(buildRequest, requestsInGroup)
221     {
222         assert(buildRequest instanceof BuildRequest);
223         assert(requestsInGroup[0] instanceof BuildRequest);
224
225         const commitSet = buildRequest.commitSet();
226         assert(commitSet instanceof CommitSet);
227
228         const repositoryByName = {};
229         for (let repository of commitSet.repositories())
230             repositoryByName[repository.name()] = repository;
231
232         const matchingConfiguration = this._configurations.find((config) => config.platform == buildRequest.platform() && config.test == buildRequest.test());
233         assert(matchingConfiguration, `Build request ${buildRequest.id()} does not match a configuration in the builder "${this._builderName}"`);
234         const propertiesTemplate = matchingConfiguration.propertiesTemplate;
235
236         const repositoryGroup = buildRequest.repositoryGroup();
237         assert(repositoryGroup.accepts(commitSet), `Build request ${buildRequest.id()} does not specify a commit set accepted by the repository group ${repositoryGroup.id()}`);
238
239         const repositoryGroupConfiguration = this._repositoryGroups[repositoryGroup.name()];
240         assert(repositoryGroupConfiguration, `Build request ${buildRequest.id()} uses an unsupported repository group "${repositoryGroup.name()}"`);
241
242         const properties = {};
243         for (let propertyName in propertiesTemplate)
244             properties[propertyName] = propertiesTemplate[propertyName];
245
246         const repositoryGroupTemplate = buildRequest.isBuild() ? repositoryGroupConfiguration.buildPropertiesTemplate : repositoryGroupConfiguration.testPropertiesTemplate;
247         for (let propertyName in repositoryGroupTemplate) {
248             let value = repositoryGroupTemplate[propertyName];
249             const type = typeof(value) == 'object' ? value.type : 'string';
250             switch (type) {
251             case 'string':
252                 break;
253             case 'revision':
254                 value = commitSet.revisionForRepository(value.repository);
255                 break;
256             case 'patch':
257                 const patch = commitSet.patchForRepository(value.repository);
258                 if (!patch)
259                     continue;
260                 value = patch.url();
261                 break;
262             case 'roots':
263                 const rootFiles = commitSet.allRootFiles();
264                 if (!rootFiles.length)
265                     continue;
266                 value = JSON.stringify(rootFiles.map((file) => ({url: file.url()})));
267                 break;
268             case 'ownedRevisions':
269                 const ownedRepositories = commitSet.ownedRepositoriesForOwnerRepository(value.ownerRepository);
270                 if (!ownedRepositories)
271                     continue;
272
273                 const revisionInfo = {};
274                 revisionInfo[value.ownerRepository.name()] = ownedRepositories.map((ownedRepository) => {
275                     return {
276                         'revision': commitSet.revisionForRepository(ownedRepository),
277                         'repository': ownedRepository.name(),
278                         'ownerRevision': commitSet.ownerRevisionForRepository(ownedRepository)
279                     };
280                 });
281                 value = JSON.stringify(revisionInfo);
282                 break;
283             case 'conditional':
284                 switch (value.condition) {
285                 case 'built':
286                     if (!requestsInGroup.some((otherRequest) => otherRequest.isBuild() && otherRequest.commitSet() == buildRequest.commitSet()))
287                         continue;
288                     break;
289                 case 'requiresBuild':
290                     const requiresBuild = value.repositoriesToCheck.some((repository) => commitSet.requiresBuildForRepository(repository));
291                     if (!requiresBuild)
292                         continue;
293                     break;
294                 }
295                 value = value.value;
296             }
297             properties[propertyName] = value;
298         }
299         properties[this._buildRequestPropertyName] = buildRequest.id();
300
301         return properties;
302     }
303
304     _revisionSetFromCommitSetWithExclusionList(commitSet, exclusionList)
305     {
306         const revisionSet = {};
307         for (let repository of commitSet.repositories()) {
308             if (exclusionList.indexOf(repository.name()) >= 0)
309                 continue;
310             const commit = commitSet.commitForRepository(repository);
311             revisionSet[repository.name()] = {
312                 id: commit.id(),
313                 time: +commit.time(),
314                 repository: repository.name(),
315                 revision: commit.revision(),
316             };
317         }
318         return revisionSet;
319     }
320
321     static _loadConfig(remote, config, builderNameToIDMap)
322     {
323         assert(builderNameToIDMap);
324         const types = config['types'] || {};
325         const builders = config['builders'] || {};
326
327         assert(config.buildRequestArgument, 'buildRequestArgument must specify the name of the property used to store the build request ID');
328
329         assert.equal(typeof(config.repositoryGroups), 'object', 'repositoryGroups must specify a dictionary from the name to its definition');
330
331         const repositoryGroups = {};
332         for (const name in config.repositoryGroups)
333             repositoryGroups[name] = this._parseRepositoryGroup(name, config.repositoryGroups[name]);
334
335         const commonConfigurations = {
336             repositoryGroups,
337             slaveArgument: config.slaveArgument,
338             buildRequestArgument: config.buildRequestArgument,
339             platformArgument: config.platformArgument,
340         };
341
342         const syncerByBuilder = new Map;
343         const ensureBuildbotSyncer = (builderInfo) => {
344             let builderSyncer = syncerByBuilder.get(builderInfo.builder);
345             if (!builderSyncer) {
346                 builderSyncer = new BuildbotSyncer(remote, builderInfo, commonConfigurations);
347                 syncerByBuilder.set(builderInfo.builder, builderSyncer);
348             }
349             return builderSyncer;
350         }
351
352         assert(Array.isArray(config['testConfigurations']), `The test configuration must be an array`);
353         this._resolveBuildersWithPlatforms('test', config['testConfigurations'], builders, builderNameToIDMap).forEach((entry, configurationIndex) => {
354             assert(Array.isArray(entry['types']), `The test configuration ${configurationIndex} does not specify "types" as an array`);
355             for (const type of entry['types']) {
356                 const typeConfig = this._validateAndMergeConfig({}, entry.builderConfig);
357                 assert(types[type], `"${type}" is not a valid type in the configuration`);
358                 this._validateAndMergeConfig(typeConfig, types[type]);
359
360                 const testPath = typeConfig.test;
361                 const test = Test.findByPath(testPath);
362                 assert(test, `"${testPath.join('", "')}" is not a valid test path in the test configuration ${configurationIndex}`);
363
364                 ensureBuildbotSyncer(entry.builderConfig).addTestConfiguration(test, entry.platform, typeConfig.properties);
365             }
366         });
367
368         const buildConfigurations = config['buildConfigurations'];
369         if (buildConfigurations) {
370             assert(Array.isArray(buildConfigurations), `The test configuration must be an array`);
371             this._resolveBuildersWithPlatforms('test', buildConfigurations, builders, builderNameToIDMap).forEach((entry, configurationIndex) => {
372                 const syncer = ensureBuildbotSyncer(entry.builderConfig);
373                 assert(!syncer.isTester(), `The build configuration ${configurationIndex} uses a tester: ${syncer.builderName()}`);
374                 syncer.addBuildConfiguration(entry.platform, entry.builderConfig.properties);
375             });
376         }
377
378         return Array.from(syncerByBuilder.values());
379     }
380
381     static _resolveBuildersWithPlatforms(configurationType, configurationList, builders, builderNameToIDMap)
382     {
383         const resolvedConfigurations = [];
384         let configurationIndex = 0;
385         for (const entry of configurationList) {
386             configurationIndex++;
387             assert(Array.isArray(entry['builders']), `The ${configurationType} configuration ${configurationIndex} does not specify "builders" as an array`);
388             assert(Array.isArray(entry['platforms']), `The ${configurationType} configuration ${configurationIndex} does not specify "platforms" as an array`);
389             for (const builderKey of entry['builders']) {
390                 const matchingBuilder = builders[builderKey];
391                 assert(matchingBuilder, `"${builderKey}" is not a valid builder in the configuration`);
392                 assert('builder' in matchingBuilder, `Builder ${builderKey} does not specify a buildbot builder name`);
393                 assert(matchingBuilder.builder in builderNameToIDMap, `Builder ${matchingBuilder.builder} not found in Buildbot configuration.`);
394                 matchingBuilder['builderID'] = builderNameToIDMap[matchingBuilder.builder];
395                 const builderConfig = this._validateAndMergeConfig({}, matchingBuilder);
396                 for (const platformName of entry['platforms']) {
397                     const platform = Platform.findByName(platformName);
398                     assert(platform, `${platformName} is not a valid platform name`);
399                     resolvedConfigurations.push({types: entry.types, builderConfig, platform});
400                 }
401             }
402         }
403         return resolvedConfigurations;
404     }
405
406     static _parseRepositoryGroup(name, group)
407     {
408         assert.equal(typeof(group.repositories), 'object',
409             `Repository group "${name}" does not specify a dictionary of repositories`);
410         assert(!('description' in group) || typeof(group['description']) == 'string',
411             `Repository group "${name}" have an invalid description`);
412         assert([undefined, true, false].includes(group.acceptsRoots),
413             `Repository group "${name}" contains invalid acceptsRoots value: ${JSON.stringify(group.acceptsRoots)}`);
414
415         const repositoryByName = {};
416         const parsedRepositoryList = [];
417         const patchAcceptingRepositoryList = new Set;
418         for (const repositoryName in group.repositories) {
419             const options = group.repositories[repositoryName];
420             const repository = Repository.findTopLevelByName(repositoryName);
421             assert(repository, `"${repositoryName}" is not a valid repository name`);
422             repositoryByName[repositoryName] = repository;
423             assert.equal(typeof(options), 'object', `"${repositoryName}" specifies a non-dictionary value`);
424             assert([undefined, true, false].includes(options.acceptsPatch),
425                 `"${repositoryName}" contains invalid acceptsPatch value: ${JSON.stringify(options.acceptsPatch)}`);
426             if (options.acceptsPatch)
427                 patchAcceptingRepositoryList.add(repository);
428             repositoryByName[repositoryName] = repository;
429             parsedRepositoryList.push({repository: repository.id(), acceptsPatch: options.acceptsPatch});
430         }
431         assert(parsedRepositoryList.length, `Repository group "${name}" does not specify any repository`);
432
433         assert.equal(typeof(group.testProperties), 'object', `Repository group "${name}" specifies the test configurations with an invalid type`);
434
435         const resolveRepository = (repositoryName) => {
436             const repository = repositoryByName[repositoryName];
437             assert(repository, `Repository group "${name}" an invalid repository "${repositoryName}"`);
438             return repository;
439         }
440
441         const testRepositories = new Set;
442         let specifiesRoots = false;
443         const testPropertiesTemplate = this._parseRepositoryGroupPropertyTemplate('test', name, group.testProperties, (type, value) => {
444             assert(type != 'patch', `Repository group "${name}" specifies a patch for "${value}" in the properties for testing`);
445             switch (type) {
446             case 'revision':
447                 const repository = resolveRepository(value);
448                 testRepositories.add(repository);
449                 return {type, repository};
450             case 'roots':
451                 assert(group.acceptsRoots, `Repository group "${name}" specifies roots in a property but it does not accept roots`);
452                 specifiesRoots = true;
453                 return {type};
454             case 'ifBuilt':
455                 return {type: 'conditional', condition: 'built', value};
456             }
457             return null;
458         });
459         assert(!group.acceptsRoots == !specifiesRoots,
460             `Repository group "${name}" accepts roots but does not specify roots in testProperties`);
461         assert.equal(parsedRepositoryList.length, testRepositories.size,
462             `Repository group "${name}" does not use some of the repositories listed in testing`);
463
464         let buildPropertiesTemplate = null;
465         if ('buildProperties' in group) {
466             assert(patchAcceptingRepositoryList.size, `Repository group "${name}" specifies the properties for building but does not accept any patches`);
467             assert(group.acceptsRoots, `Repository group "${name}" specifies the properties for building but does not accept roots in testing`);
468             const revisionRepositories = new Set;
469             const patchRepositories = new Set;
470             buildPropertiesTemplate = this._parseRepositoryGroupPropertyTemplate('build', name, group.buildProperties, (type, value, condition) => {
471                 assert(type != 'roots', `Repository group "${name}" specifies roots in the properties for building`);
472                 let repository = null;
473                 switch (type) {
474                 case 'patch':
475                     repository = resolveRepository(value);
476                     assert(patchAcceptingRepositoryList.has(repository), `Repository group "${name}" specifies a patch for "${value}" but it does not accept a patch`);
477                     patchRepositories.add(repository);
478                     return {type, repository};
479                 case 'revision':
480                     repository = resolveRepository(value);
481                     revisionRepositories.add(repository);
482                     return {type, repository};
483                 case 'ownedRevisions':
484                     return {type, ownerRepository: resolveRepository(value)};
485                 case 'ifRepositorySet':
486                     assert(condition, 'condition must set if type is "ifRepositorySet"');
487                     return {type: 'conditional', condition: 'requiresBuild', value, repositoriesToCheck: condition.map(resolveRepository)};
488                 }
489                 return null;
490             });
491             for (const repository of patchRepositories)
492                 assert(revisionRepositories.has(repository), `Repository group "${name}" specifies a patch for "${repository.name()}" but does not specify a revision`);
493             assert.equal(patchAcceptingRepositoryList.size, patchRepositories.size,
494                 `Repository group "${name}" does not use some of the repositories listed in building a patch`);
495         }
496
497         return {
498             name: group.name,
499             description: group.description,
500             acceptsRoots: group.acceptsRoots,
501             testPropertiesTemplate: testPropertiesTemplate,
502             buildPropertiesTemplate: buildPropertiesTemplate,
503             repositoryList: parsedRepositoryList,
504         };
505     }
506
507     static _parseRepositoryGroupPropertyTemplate(parsingMode, groupName, properties, makeOption)
508     {
509         const propertiesTemplate = {};
510         for (const propertyName in properties) {
511             let value = properties[propertyName];
512             const isDictionary = typeof(value) == 'object';
513             assert(isDictionary || typeof(value) == 'string',
514                 `Repository group "${groupName}" uses an invalid value "${value}" in property "${propertyName}"`);
515
516             if (!isDictionary) {
517                 propertiesTemplate[propertyName] = value;
518                 continue;
519             }
520
521             const keys = Object.keys(value);
522             assert(keys.length == 1 || keys.length == 2,
523                 `Repository group "${groupName}" specifies more than two types in property "${propertyName}": "${keys.join('", "')}"`);
524             let type;
525             let condition = null;
526             let optionValue;
527             if (keys.length == 2) {
528                 assert(keys.includes('value'), `Repository group "${groupName}" with two types in property "${propertyName}": "${keys.join('", "')}" should contains 'value' as one type`);
529                 type = keys.find((key) => key != 'value');
530                 optionValue = value.value;
531                 condition = value[type];
532             }
533             else {
534                 type = keys[0];
535                 optionValue = value[type];
536             }
537             const option = makeOption(type, optionValue, condition);
538             assert(option, `Repository group "${groupName}" specifies an invalid type "${type}" in property "${propertyName}"`);
539             propertiesTemplate[propertyName] = option;
540         }
541         return propertiesTemplate;
542     }
543
544     static _validateAndMergeConfig(config, valuesToMerge, excludedProperty)
545     {
546         for (const name in valuesToMerge) {
547             const value = valuesToMerge[name];
548             if (name == excludedProperty)
549                 continue;
550
551             switch (name) {
552             case 'properties': // Fallthrough
553                 assert.equal(typeof(value), 'object', 'Build properties should be a dictionary');
554                 if (!config['properties'])
555                     config['properties'] = {};
556                 const properties = config['properties'];
557                 for (const name in value) {
558                     assert.equal(typeof(value[name]), 'string', `Build properties "${name}" specifies a non-string value of type "${typeof(value)}"`);
559                     properties[name] = value[name];
560                 }
561                 break;
562             case 'test': // Fallthrough
563             case 'slaveList': // Fallthrough
564                 assert(value instanceof Array, `${name} should be an array`);
565                 assert(value.every(function (part) { return typeof part == 'string'; }), `${name} should be an array of strings`);
566                 config[name] = value.slice();
567                 break;
568             case 'builder': // Fallthrough
569                 assert.equal(typeof(value), 'string', `${name} should be of string type`);
570                 config[name] = value;
571                 break;
572             case 'builderID':
573                 assert(value, 'builderID should not be undefined.');
574                 config[name] = value;
575                 break;
576             default:
577                 assert(false, `Unrecognized parameter "${name}"`);
578             }
579         }
580         return config;
581     }
582
583 }
584
585 if (typeof module != 'undefined') {
586     module.exports.BuildbotSyncer = BuildbotSyncer;
587     module.exports.BuildbotBuildEntry = BuildbotBuildEntry;
588 }