47a08854dacdfd12b33a95872a22dc258e1d3a34
[WebKit-https.git] / Websites / perf.webkit.org / tools / js / buildbot-syncer.js
1 'use strict';
2
3 let assert = require('assert');
4
5 require('./v3-models.js');
6
7 class BuildbotBuildEntry {
8     constructor(syncer, rawData)
9     {
10         assert.equal(syncer.builderName(), rawData['builderName']);
11
12         this._syncer = syncer;
13         this._slaveName = null;
14         this._buildRequestId = null;
15         this._isInProgress = rawData['currentStep'] || (rawData['times'] && !rawData['times'][1]);
16         this._buildNumber = rawData['number'];
17
18         for (let propertyTuple of (rawData['properties'] || [])) {
19             // e.g. ['build_request_id', '16733', 'Force Build Form']
20             const name = propertyTuple[0];
21             const value = propertyTuple[1];
22             if (name == syncer._slavePropertyName)
23                 this._slaveName = value;
24             else if (name == syncer._buildRequestPropertyName)
25                 this._buildRequestId = value;
26         }
27     }
28
29     syncer() { return this._syncer; }
30     buildNumber() { return this._buildNumber; }
31     slaveName() { return this._slaveName; }
32     buildRequestId() { return this._buildRequestId; }
33     isPending() { return typeof(this._buildNumber) != 'number'; }
34     isInProgress() { return this._isInProgress; }
35     hasFinished() { return !this.isPending() && !this.isInProgress(); }
36     url() { return this.isPending() ? this._syncer.url() : this._syncer.urlForBuildNumber(this._buildNumber); }
37
38     buildRequestStatusIfUpdateIsNeeded(request)
39     {
40         assert.equal(request.id(), this._buildRequestId);
41         if (!request)
42             return null;
43         if (this.isPending()) {
44             if (request.isPending())
45                 return 'scheduled';
46         } else if (this.isInProgress()) {
47             if (!request.hasStarted() || request.isScheduled())
48                 return 'running';
49         } else if (this.hasFinished()) {
50             if (!request.hasFinished())
51                 return 'failedIfNotCompleted';
52         }
53         return null;
54     }
55 }
56
57
58 class BuildbotSyncer {
59
60     constructor(remote, object, commonConfigurations)
61     {
62         this._remote = remote;
63         this._type = null;
64         this._configurations = [];
65         this._repositoryGroups = commonConfigurations.repositoryGroups;
66         this._slavePropertyName = commonConfigurations.slaveArgument;
67         this._buildRequestPropertyName = commonConfigurations.buildRequestArgument;
68         this._builderName = object.builder;
69         this._slaveList = object.slaveList;
70         this._entryList = null;
71         this._slavesWithNewRequests = new Set;
72     }
73
74     builderName() { return this._builderName; }
75
76     addTestConfiguration(test, platform, propertiesTemplate)
77     {
78         assert(test instanceof Test);
79         assert(platform instanceof Platform);
80         assert(this._type == null || this._type == 'tester');
81         this._type = 'tester';
82         this._configurations.push({test, platform, propertiesTemplate});
83     }
84     testConfigurations() { return this._type == 'tester' ? this._configurations : []; }
85
86     addBuildConfiguration(platform, propertiesTemplate)
87     {
88         assert(platform instanceof Platform);
89         assert(this._type == null || this._type == 'builder');
90         this._type = 'builder';
91         this._configurations.push({test: null, platform, propertiesTemplate});
92     }
93     buildConfigurations() { return this._type == 'builder' ? this._configurations : []; }
94
95     isTester() { return this._type == 'tester'; }
96
97     repositoryGroups() { return this._repositoryGroups; }
98
99     matchesConfiguration(request)
100     {
101         return this._configurations.some((config) => config.platform == request.platform() && config.test == request.test());
102     }
103
104     scheduleRequest(newRequest, requestsInGroup, slaveName)
105     {
106         assert(!this._slavesWithNewRequests.has(slaveName));
107         let properties = this._propertiesForBuildRequest(newRequest, requestsInGroup);
108
109         assert.equal(!this._slavePropertyName, !slaveName);
110         if (this._slavePropertyName)
111             properties[this._slavePropertyName] = slaveName;
112
113         this._slavesWithNewRequests.add(slaveName);
114         return this._remote.postFormUrlencodedData(this.pathForForceBuild(), properties);
115     }
116
117     scheduleRequestInGroupIfAvailable(newRequest, requestsInGroup, slaveName)
118     {
119         assert(newRequest instanceof BuildRequest);
120
121         if (!this.matchesConfiguration(newRequest))
122             return null;
123
124         let hasPendingBuildsWithoutSlaveNameSpecified = false;
125         let usedSlaves = new Set;
126         for (let entry of this._entryList) {
127             let entryPreventsNewRequest = entry.isPending();
128             if (entry.isInProgress()) {
129                 const requestInProgress = BuildRequest.findById(entry.buildRequestId());
130                 if (!requestInProgress || requestInProgress.testGroupId() != newRequest.testGroupId())
131                     entryPreventsNewRequest = true;
132             }
133             if (entryPreventsNewRequest) {
134                 if (!entry.slaveName())
135                     hasPendingBuildsWithoutSlaveNameSpecified = true;
136                 usedSlaves.add(entry.slaveName());
137             }
138         }
139
140         if (!this._slaveList || hasPendingBuildsWithoutSlaveNameSpecified) {
141             if (usedSlaves.size || this._slavesWithNewRequests.size)
142                 return null;
143             return this.scheduleRequest(newRequest, requestsInGroup, null);
144         }
145
146         if (slaveName) {
147             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
148                 return this.scheduleRequest(newRequest, requestsInGroup, slaveName);
149             return null;
150         }
151
152         for (let slaveName of this._slaveList) {
153             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
154                 return this.scheduleRequest(newRequest, requestsInGroup, slaveName);
155         }
156
157         return null;
158     }
159
160     pullBuildbot(count)
161     {
162         return this._remote.getJSON(this.pathForPendingBuildsJSON()).then((content) => {
163             let pendingEntries = content.map((entry) => new BuildbotBuildEntry(this, entry));
164             return this._pullRecentBuilds(count).then((entries) => {
165                 let entryByRequest = {};
166
167                 for (let entry of pendingEntries)
168                     entryByRequest[entry.buildRequestId()] = entry;
169
170                 for (let entry of entries)
171                     entryByRequest[entry.buildRequestId()] = entry;
172
173                 let entryList = [];
174                 for (let id in entryByRequest)
175                     entryList.push(entryByRequest[id]);
176
177                 this._entryList = entryList;
178                 this._slavesWithNewRequests.clear();
179
180                 return entryList;
181             });
182         });
183     }
184
185     _pullRecentBuilds(count)
186     {
187         if (!count)
188             return Promise.resolve([]);
189
190         let selectedBuilds = new Array(count);
191         for (let i = 0; i < count; i++)
192             selectedBuilds[i] = -i - 1;
193
194         return this._remote.getJSON(this.pathForBuildJSON(selectedBuilds)).then((content) => {
195             const entries = [];
196             for (let index of selectedBuilds) {
197                 const entry = content[index];
198                 if (entry && !entry['error'])
199                     entries.push(new BuildbotBuildEntry(this, entry));
200             }
201             return entries;
202         });
203     }
204
205     pathForPendingBuildsJSON() { return `/json/builders/${escape(this._builderName)}/pendingBuilds`; }
206     pathForBuildJSON(selectedBuilds)
207     {
208         return `/json/builders/${escape(this._builderName)}/builds/?` + selectedBuilds.map((number) => 'select=' + number).join('&');
209     }
210     pathForForceBuild() { return `/builders/${escape(this._builderName)}/force`; }
211
212     url() { return this._remote.url(`/builders/${escape(this._builderName)}/`); }
213     urlForBuildNumber(number) { return this._remote.url(`/builders/${escape(this._builderName)}/builds/${number}`); }
214
215     _propertiesForBuildRequest(buildRequest, requestsInGroup)
216     {
217         assert(buildRequest instanceof BuildRequest);
218         assert(requestsInGroup[0] instanceof BuildRequest);
219
220         const commitSet = buildRequest.commitSet();
221         assert(commitSet instanceof CommitSet);
222
223         const repositoryByName = {};
224         for (let repository of commitSet.repositories())
225             repositoryByName[repository.name()] = repository;
226
227         const matchingConfiguration = this._configurations.find((config) => config.platform == buildRequest.platform() && config.test == buildRequest.test());
228         assert(matchingConfiguration, `Build request ${buildRequest.id()} does not match a configuration in the builder "${this._builderName}"`);
229         const propertiesTemplate = matchingConfiguration.propertiesTemplate;
230
231         const repositoryGroup = buildRequest.repositoryGroup();
232         assert(repositoryGroup.accepts(commitSet), `Build request ${buildRequest.id()} does not specify a commit set accepted by the repository group ${repositoryGroup.id()}`);
233
234         const repositoryGroupConfiguration = this._repositoryGroups[repositoryGroup.name()];
235         assert(repositoryGroupConfiguration, `Build request ${buildRequest.id()} uses an unsupported repository group "${repositoryGroup.name()}"`);
236
237         const properties = {};
238         for (let propertyName in propertiesTemplate)
239             properties[propertyName] = propertiesTemplate[propertyName];
240
241         const repositoryGroupTemplate = buildRequest.isBuild() ? repositoryGroupConfiguration.buildPropertiesTemplate : repositoryGroupConfiguration.testPropertiesTemplate;
242         for (let propertyName in repositoryGroupTemplate) {
243             let value = repositoryGroupTemplate[propertyName];
244             const type = typeof(value) == 'object' ? value.type : 'string';
245             switch (type) {
246             case 'string':
247                 break;
248             case 'revision':
249                 value = commitSet.revisionForRepository(value.repository);
250                 break;
251             case 'patch':
252                 const patch = commitSet.patchForRepository(value.repository);
253                 if (!patch)
254                     continue;
255                 value = patch.url();
256                 break;
257             case 'roots':
258                 const rootFiles = commitSet.allRootFiles();
259                 if (!rootFiles.length)
260                     continue;
261                 value = JSON.stringify(rootFiles.map((file) => ({url: file.url()})));
262                 break;
263             case 'ownedRevisions':
264                 const ownedRepositories = commitSet.ownedRepositoriesForOwnerRepository(value.ownerRepository);
265                 if (!ownedRepositories)
266                     continue;
267
268                 const revisionInfo = {};
269                 revisionInfo[value.ownerRepository.name()] = ownedRepositories.map((ownedRepository) => {
270                     return {
271                         'revision': commitSet.revisionForRepository(ownedRepository),
272                         'repository': ownedRepository.name(),
273                         'ownerRevision': commitSet.ownerRevisionForRepository(ownedRepository)
274                     };
275                 });
276                 value = JSON.stringify(revisionInfo);
277                 break;
278             case 'conditional':
279                 switch (value.condition) {
280                 case 'built':
281                     if (!requestsInGroup.some((otherRequest) => otherRequest.isBuild() && otherRequest.commitSet() == buildRequest.commitSet()))
282                         continue;
283                     break;
284                 case 'requiresBuild':
285                     const requiresBuild = value.repositoriesToCheck.some((repository) => commitSet.requiresBuildForRepository(repository));
286                     if (!requiresBuild)
287                         continue;
288                     break;
289                 }
290                 value = value.value;
291             }
292             properties[propertyName] = value;
293         }
294         properties[this._buildRequestPropertyName] = buildRequest.id();
295
296         return properties;
297     }
298
299     _revisionSetFromCommitSetWithExclusionList(commitSet, exclusionList)
300     {
301         const revisionSet = {};
302         for (let repository of commitSet.repositories()) {
303             if (exclusionList.indexOf(repository.name()) >= 0)
304                 continue;
305             const commit = commitSet.commitForRepository(repository);
306             revisionSet[repository.name()] = {
307                 id: commit.id(),
308                 time: +commit.time(),
309                 repository: repository.name(),
310                 revision: commit.revision(),
311             };
312         }
313         return revisionSet;
314     }
315
316     static _loadConfig(remote, config)
317     {
318         const types = config['types'] || {};
319         const builders = config['builders'] || {};
320
321         assert(config.buildRequestArgument, 'buildRequestArgument must specify the name of the property used to store the build request ID');
322
323         assert.equal(typeof(config.repositoryGroups), 'object', 'repositoryGroups must specify a dictionary from the name to its definition');
324
325         const repositoryGroups = {};
326         for (const name in config.repositoryGroups)
327             repositoryGroups[name] = this._parseRepositoryGroup(name, config.repositoryGroups[name]);
328
329         const commonConfigurations = {
330             repositoryGroups,
331             slaveArgument: config.slaveArgument,
332             buildRequestArgument: config.buildRequestArgument,
333         };
334
335         const syncerByBuilder = new Map;
336         const ensureBuildbotSyncer = (builderInfo) => {
337             let builderSyncer = syncerByBuilder.get(builderInfo.builder);
338             if (!builderSyncer) {
339                 builderSyncer = new BuildbotSyncer(remote, builderInfo, commonConfigurations);
340                 syncerByBuilder.set(builderInfo.builder, builderSyncer);
341             }
342             return builderSyncer;
343         }
344
345         assert(Array.isArray(config['testConfigurations']), `The test configuration must be an array`);
346         this._resolveBuildersWithPlatforms('test', config['testConfigurations'], builders).forEach((entry, configurationIndex) => {
347             assert(Array.isArray(entry['types']), `The test configuration ${configurationIndex} does not specify "types" as an array`);
348             for (const type of entry['types']) {
349                 const typeConfig = this._validateAndMergeConfig({}, entry.builderConfig);
350                 assert(types[type], `"${type}" is not a valid type in the configuration`);
351                 this._validateAndMergeConfig(typeConfig, types[type]);
352
353                 const testPath = typeConfig.test;
354                 const test = Test.findByPath(testPath);
355                 assert(test, `"${testPath.join('", "')}" is not a valid test path in the test configuration ${configurationIndex}`);
356
357                 ensureBuildbotSyncer(entry.builderConfig).addTestConfiguration(test, entry.platform, typeConfig.properties);
358             }
359         });
360
361         const buildConfigurations = config['buildConfigurations'];
362         if (buildConfigurations) {
363             assert(Array.isArray(buildConfigurations), `The test configuration must be an array`);
364             this._resolveBuildersWithPlatforms('test', buildConfigurations, builders).forEach((entry, configurationIndex) => {
365                 const syncer = ensureBuildbotSyncer(entry.builderConfig);
366                 assert(!syncer.isTester(), `The build configuration ${configurationIndex} uses a tester: ${syncer.builderName()}`);
367                 syncer.addBuildConfiguration(entry.platform, entry.builderConfig.properties);
368             });
369         }
370
371         return Array.from(syncerByBuilder.values());
372     }
373
374     static _resolveBuildersWithPlatforms(configurationType, configurationList, builders)
375     {
376         const resolvedConfigurations = [];
377         let configurationIndex = 0;
378         for (const entry of configurationList) {
379             configurationIndex++;
380             assert(Array.isArray(entry['builders']), `The ${configurationType} configuration ${configurationIndex} does not specify "builders" as an array`);
381             assert(Array.isArray(entry['platforms']), `The ${configurationType} configuration ${configurationIndex} does not specify "platforms" as an array`);
382             for (const builderKey of entry['builders']) {
383                 const matchingBuilder = builders[builderKey];
384                 assert(matchingBuilder, `"${builderKey}" is not a valid builder in the configuration`);
385                 assert('builder' in matchingBuilder, `Builder ${builderKey} does not specify a buildbot builder name`);
386                 const builderConfig = this._validateAndMergeConfig({}, matchingBuilder);
387                 for (const platformName of entry['platforms']) {
388                     const platform = Platform.findByName(platformName);
389                     assert(platform, `${platformName} is not a valid platform name`);
390                     resolvedConfigurations.push({types: entry.types, builderConfig, platform});
391                 }
392             }
393         }
394         return resolvedConfigurations;
395     }
396
397     static _parseRepositoryGroup(name, group)
398     {
399         assert.equal(typeof(group.repositories), 'object',
400             `Repository group "${name}" does not specify a dictionary of repositories`);
401         assert(!('description' in group) || typeof(group['description']) == 'string',
402             `Repository group "${name}" have an invalid description`);
403         assert([undefined, true, false].includes(group.acceptsRoots),
404             `Repository group "${name}" contains invalid acceptsRoots value: ${JSON.stringify(group.acceptsRoots)}`);
405
406         const repositoryByName = {};
407         const parsedRepositoryList = [];
408         const patchAcceptingRepositoryList = new Set;
409         for (const repositoryName in group.repositories) {
410             const options = group.repositories[repositoryName];
411             const repository = Repository.findTopLevelByName(repositoryName);
412             assert(repository, `"${repositoryName}" is not a valid repository name`);
413             repositoryByName[repositoryName] = repository;
414             assert.equal(typeof(options), 'object', `"${repositoryName}" specifies a non-dictionary value`);
415             assert([undefined, true, false].includes(options.acceptsPatch),
416                 `"${repositoryName}" contains invalid acceptsPatch value: ${JSON.stringify(options.acceptsPatch)}`);
417             if (options.acceptsPatch)
418                 patchAcceptingRepositoryList.add(repository);
419             repositoryByName[repositoryName] = repository;
420             parsedRepositoryList.push({repository: repository.id(), acceptsPatch: options.acceptsPatch});
421         }
422         assert(parsedRepositoryList.length, `Repository group "${name}" does not specify any repository`);
423
424         assert.equal(typeof(group.testProperties), 'object', `Repository group "${name}" specifies the test configurations with an invalid type`);
425
426         const resolveRepository = (repositoryName) => {
427             const repository = repositoryByName[repositoryName];
428             assert(repository, `Repository group "${name}" an invalid repository "${repositoryName}"`);
429             return repository;
430         }
431
432         const testRepositories = new Set;
433         let specifiesRoots = false;
434         const testPropertiesTemplate = this._parseRepositoryGroupPropertyTemplate('test', name, group.testProperties, (type, value) => {
435             assert(type != 'patch', `Repository group "${name}" specifies a patch for "${value}" in the properties for testing`);
436             switch (type) {
437             case 'revision':
438                 const repository = resolveRepository(value);
439                 testRepositories.add(repository);
440                 return {type, repository};
441             case 'roots':
442                 assert(group.acceptsRoots, `Repository group "${name}" specifies roots in a property but it does not accept roots`);
443                 specifiesRoots = true;
444                 return {type};
445             case 'ifBuilt':
446                 return {type: 'conditional', condition: 'built', value};
447             }
448             return null;
449         });
450         assert(!group.acceptsRoots == !specifiesRoots,
451             `Repository group "${name}" accepts roots but does not specify roots in testProperties`);
452         assert.equal(parsedRepositoryList.length, testRepositories.size,
453             `Repository group "${name}" does not use some of the repositories listed in testing`);
454
455         let buildPropertiesTemplate = null;
456         if ('buildProperties' in group) {
457             assert(patchAcceptingRepositoryList.size, `Repository group "${name}" specifies the properties for building but does not accept any patches`);
458             assert(group.acceptsRoots, `Repository group "${name}" specifies the properties for building but does not accept roots in testing`);
459             const revisionRepositories = new Set;
460             const patchRepositories = new Set;
461             buildPropertiesTemplate = this._parseRepositoryGroupPropertyTemplate('build', name, group.buildProperties, (type, value, condition) => {
462                 assert(type != 'roots', `Repository group "${name}" specifies roots in the properties for building`);
463                 let repository = null;
464                 switch (type) {
465                 case 'patch':
466                     repository = resolveRepository(value);
467                     assert(patchAcceptingRepositoryList.has(repository), `Repository group "${name}" specifies a patch for "${value}" but it does not accept a patch`);
468                     patchRepositories.add(repository);
469                     return {type, repository};
470                 case 'revision':
471                     repository = resolveRepository(value);
472                     revisionRepositories.add(repository);
473                     return {type, repository};
474                 case 'ownedRevisions':
475                     return {type, ownerRepository: resolveRepository(value)};
476                 case 'ifRepositorySet':
477                     assert(condition, 'condition must set if type is "ifRepositorySet"');
478                     return {type: 'conditional', condition: 'requiresBuild', value, repositoriesToCheck: condition.map(resolveRepository)};
479                 }
480                 return null;
481             });
482             for (const repository of patchRepositories)
483                 assert(revisionRepositories.has(repository), `Repository group "${name}" specifies a patch for "${repository.name()}" but does not specify a revision`);
484             assert.equal(patchAcceptingRepositoryList.size, patchRepositories.size,
485                 `Repository group "${name}" does not use some of the repositories listed in building a patch`);
486         }
487
488         return {
489             name: group.name,
490             description: group.description,
491             acceptsRoots: group.acceptsRoots,
492             testPropertiesTemplate: testPropertiesTemplate,
493             buildPropertiesTemplate: buildPropertiesTemplate,
494             repositoryList: parsedRepositoryList,
495         };
496     }
497
498     static _parseRepositoryGroupPropertyTemplate(parsingMode, groupName, properties, makeOption)
499     {
500         const propertiesTemplate = {};
501         for (const propertyName in properties) {
502             let value = properties[propertyName];
503             const isDictionary = typeof(value) == 'object';
504             assert(isDictionary || typeof(value) == 'string',
505                 `Repository group "${groupName}" uses an invalid value "${value}" in property "${propertyName}"`);
506
507             if (!isDictionary) {
508                 propertiesTemplate[propertyName] = value;
509                 continue;
510             }
511
512             const keys = Object.keys(value);
513             assert(keys.length == 1 || keys.length == 2,
514                 `Repository group "${groupName}" specifies more than two types in property "${propertyName}": "${keys.join('", "')}"`);
515             let type;
516             let condition = null;
517             let optionValue;
518             if (keys.length == 2) {
519                 assert(keys.includes('value'), `Repository group "${groupName}" with two types in property "${propertyName}": "${keys.join('", "')}" should contains 'value' as one type`);
520                 type = keys.find((key) => key != 'value');
521                 optionValue = value.value;
522                 condition = value[type];
523             }
524             else {
525                 type = keys[0];
526                 optionValue = value[type];
527             }
528             const option = makeOption(type, optionValue, condition);
529             assert(option, `Repository group "${groupName}" specifies an invalid type "${type}" in property "${propertyName}"`);
530             propertiesTemplate[propertyName] = option;
531         }
532         return propertiesTemplate;
533     }
534
535     static _validateAndMergeConfig(config, valuesToMerge, excludedProperty)
536     {
537         for (const name in valuesToMerge) {
538             const value = valuesToMerge[name];
539             if (name == excludedProperty)
540                 continue;
541
542             switch (name) {
543             case 'properties': // Fallthrough
544                 assert.equal(typeof(value), 'object', 'Build properties should be a dictionary');
545                 if (!config['properties'])
546                     config['properties'] = {};
547                 const properties = config['properties'];
548                 for (const name in value) {
549                     assert.equal(typeof(value[name]), 'string', `Build properties "${name}" specifies a non-string value of type "${typeof(value)}"`);
550                     properties[name] = value[name];
551                 }
552                 break;
553             case 'test': // Fallthrough
554             case 'slaveList': // Fallthrough
555                 assert(value instanceof Array, `${name} should be an array`);
556                 assert(value.every(function (part) { return typeof part == 'string'; }), `${name} should be an array of strings`);
557                 config[name] = value.slice();
558                 break;
559             case 'builder': // Fallthrough
560                 assert.equal(typeof(value), 'string', `${name} should be of string type`);
561                 config[name] = value;
562                 break;
563             default:
564                 assert(false, `Unrecognized parameter "${name}"`);
565             }
566         }
567         return config;
568     }
569
570 }
571
572 if (typeof module != 'undefined') {
573     module.exports.BuildbotSyncer = BuildbotSyncer;
574     module.exports.BuildbotBuildEntry = BuildbotBuildEntry;
575 }