Websites/perf.webkit.org/tools/js/buildbot-syncer.js

   1 'use strict';
   2
   3 let assert = require('assert');
   4
   5 require('./v3-models.js');
   6
   7 class BuildbotBuildEntry {
   8     constructor(syncer, rawData)
   9     {
  10         assert.equal(syncer.builderName(), rawData['builderName']);
  11
  12         this._syncer = syncer;
  13         this._slaveName = null;
  14         this._buildRequestId = null;
  15         this._isInProgress = rawData['currentStep'] || (rawData['times'] && !rawData['times'][1]);
  16         this._buildNumber = rawData['number'];
  17
  18         for (let propertyTuple of (rawData['properties'] || [])) {
  19             // e.g. ['build_request_id', '16733', 'Force Build Form']
  20             const name = propertyTuple[0];
  21             const value = propertyTuple[1];
  22             if (name == syncer._slavePropertyName)
  23                 this._slaveName = value;
  24             else if (name == syncer._buildRequestPropertyName)
  25                 this._buildRequestId = value;
  26         }
  27     }
  28
  29     syncer() { return this._syncer; }
  30     buildNumber() { return this._buildNumber; }
  31     slaveName() { return this._slaveName; }
  32     buildRequestId() { return this._buildRequestId; }
  33     isPending() { return typeof(this._buildNumber) != 'number'; }
  34     isInProgress() { return this._isInProgress; }
  35     hasFinished() { return !this.isPending() && !this.isInProgress(); }
  36     url() { return this.isPending() ? this._syncer.url() : this._syncer.urlForBuildNumber(this._buildNumber); }
  37
  38     buildRequestStatusIfUpdateIsNeeded(request)
  39     {
  40         assert.equal(request.id(), this._buildRequestId);
  41         if (!request)
  42             return null;
  43         if (this.isPending()) {
  44             if (request.isPending())
  45                 return 'scheduled';
  46         } else if (this.isInProgress()) {
  47             if (!request.hasStarted() || request.isScheduled())
  48                 return 'running';
  49         } else if (this.hasFinished()) {
  50             if (!request.hasFinished())
  51                 return 'failedIfNotCompleted';
  52         }
  53         return null;
  54     }
  55 }
  56
  57
  58 class BuildbotSyncer {
  59
  60     constructor(remote, object, commonConfigurations)
  61     {
  62         this._remote = remote;
  63         this._testConfigurations = [];
  64         this._repositoryGroups = commonConfigurations.repositoryGroups;
  65         this._slavePropertyName = commonConfigurations.slaveArgument;
  66         this._buildRequestPropertyName = commonConfigurations.buildRequestArgument;
  67         this._builderName = object.builder;
  68         this._slaveList = object.slaveList;
  69         this._entryList = null;
  70         this._slavesWithNewRequests = new Set;
  71     }
  72
  73     builderName() { return this._builderName; }
  74
  75     addTestConfiguration(test, platform, propertiesTemplate)
  76     {
  77         assert(test instanceof Test);
  78         assert(platform instanceof Platform);
  79         this._testConfigurations.push({test, platform, propertiesTemplate});
  80     }
  81     testConfigurations() { return this._testConfigurations; }
  82     repositoryGroups() { return this._repositoryGroups; }
  83
  84     matchesConfiguration(request)
  85     {
  86         return this._testConfigurations.some((config) => config.platform == request.platform() && config.test == request.test());
  87     }
  88
  89     scheduleRequest(newRequest, slaveName)
  90     {
  91         assert(!this._slavesWithNewRequests.has(slaveName));
  92         let properties = this._propertiesForBuildRequest(newRequest);
  93
  94         assert.equal(!this._slavePropertyName, !slaveName);
  95         if (this._slavePropertyName)
  96             properties[this._slavePropertyName] = slaveName;
  97
  98         this._slavesWithNewRequests.add(slaveName);
  99         return this._remote.postFormUrlencodedData(this.pathForForceBuild(), properties);
 100     }
 101
 102     scheduleRequestInGroupIfAvailable(newRequest, slaveName)
 103     {
 104         assert(newRequest instanceof BuildRequest);
 105
 106         if (!this.matchesConfiguration(newRequest))
 107             return null;
 108
 109         let hasPendingBuildsWithoutSlaveNameSpecified = false;
 110         let usedSlaves = new Set;
 111         for (let entry of this._entryList) {
 112             if (entry.isPending()) {
 113                 if (!entry.slaveName())
 114                     hasPendingBuildsWithoutSlaveNameSpecified = true;
 115                 usedSlaves.add(entry.slaveName());
 116             }
 117         }
 118
 119         if (!this._slaveList || hasPendingBuildsWithoutSlaveNameSpecified) {
 120             if (usedSlaves.size || this._slavesWithNewRequests.size)
 121                 return null;
 122             return this.scheduleRequest(newRequest, null);
 123         }
 124
 125         if (slaveName) {
 126             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
 127                 return this.scheduleRequest(newRequest, slaveName);
 128             return null;
 129         }
 130
 131         for (let slaveName of this._slaveList) {
 132             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
 133                 return this.scheduleRequest(newRequest, slaveName);
 134         }
 135
 136         return null;
 137     }
 138
 139     pullBuildbot(count)
 140     {
 141         return this._remote.getJSON(this.pathForPendingBuildsJSON()).then((content) => {
 142             let pendingEntries = content.map((entry) => new BuildbotBuildEntry(this, entry));
 143             return this._pullRecentBuilds(count).then((entries) => {
 144                 let entryByRequest = {};
 145
 146                 for (let entry of pendingEntries)
 147                     entryByRequest[entry.buildRequestId()] = entry;
 148
 149                 for (let entry of entries)
 150                     entryByRequest[entry.buildRequestId()] = entry;
 151
 152                 let entryList = [];
 153                 for (let id in entryByRequest)
 154                     entryList.push(entryByRequest[id]);
 155
 156                 this._entryList = entryList;
 157                 this._slavesWithNewRequests.clear();
 158
 159                 return entryList;
 160             });
 161         });
 162     }
 163
 164     _pullRecentBuilds(count)
 165     {
 166         if (!count)
 167             return Promise.resolve([]);
 168
 169         let selectedBuilds = new Array(count);
 170         for (let i = 0; i < count; i++)
 171             selectedBuilds[i] = -i - 1;
 172
 173         return this._remote.getJSON(this.pathForBuildJSON(selectedBuilds)).then((content) => {
 174             const entries = [];
 175             for (let index of selectedBuilds) {
 176                 const entry = content[index];
 177                 if (entry && !entry['error'])
 178                     entries.push(new BuildbotBuildEntry(this, entry));
 179             }
 180             return entries;
 181         });
 182     }
 183
 184     pathForPendingBuildsJSON() { return `/json/builders/${escape(this._builderName)}/pendingBuilds`; }
 185     pathForBuildJSON(selectedBuilds)
 186     {
 187         return `/json/builders/${escape(this._builderName)}/builds/?` + selectedBuilds.map((number) => 'select=' + number).join('&');
 188     }
 189     pathForForceBuild() { return `/builders/${escape(this._builderName)}/force`; }
 190
 191     url() { return this._remote.url(`/builders/${escape(this._builderName)}/`); }
 192     urlForBuildNumber(number) { return this._remote.url(`/builders/${escape(this._builderName)}/builds/${number}`); }
 193
 194     _propertiesForBuildRequest(buildRequest)
 195     {
 196         assert(buildRequest instanceof BuildRequest);
 197
 198         const commitSet = buildRequest.commitSet();
 199         assert(commitSet instanceof CommitSet);
 200
 201         const repositoryByName = {};
 202         for (let repository of commitSet.repositories())
 203             repositoryByName[repository.name()] = repository;
 204
 205         const matchingConfiguration = this._testConfigurations.find((config) => config.platform == buildRequest.platform() && config.test == buildRequest.test());
 206         assert(matchingConfiguration, `Build request ${buildRequest.id()} does not match a configuration in the builder "${this._builderName}"`);
 207         const propertiesTemplate = matchingConfiguration.propertiesTemplate;
 208
 209         const repositoryGroup = buildRequest.repositoryGroup();
 210         assert(repositoryGroup.accepts(commitSet), `Build request ${buildRequest.id()} does not specify a commit set accepted by the repository group ${repositoryGroup.id()}`);
 211
 212         const repositoryGroupConfiguration = this._repositoryGroups[repositoryGroup.name()];
 213         assert(repositoryGroupConfiguration, `Build request ${buildRequest.id()} uses an unsupported repository group "${repositoryGroup.name()}"`);
 214
 215         const properties = {};
 216         for (let propertyName in propertiesTemplate)
 217             properties[propertyName] = propertiesTemplate[propertyName];
 218
 219         const repositoryGroupTemplate = repositoryGroupConfiguration.propertiesTemplate;
 220         for (let propertyName in repositoryGroupTemplate) {
 221             const value = repositoryGroupTemplate[propertyName];
 222             properties[propertyName] = value instanceof Repository ? commitSet.revisionForRepository(value) : value;
 223         }
 224         properties[this._buildRequestPropertyName] = buildRequest.id();
 225
 226         return properties;
 227     }
 228
 229     _revisionSetFromCommitSetWithExclusionList(commitSet, exclusionList)
 230     {
 231         const revisionSet = {};
 232         for (let repository of commitSet.repositories()) {
 233             if (exclusionList.indexOf(repository.name()) >= 0)
 234                 continue;
 235             const commit = commitSet.commitForRepository(repository);
 236             revisionSet[repository.name()] = {
 237                 id: commit.id(),
 238                 time: +commit.time(),
 239                 repository: repository.name(),
 240                 revision: commit.revision(),
 241             };
 242         }
 243         return revisionSet;
 244     }
 245
 246     static _loadConfig(remote, config)
 247     {
 248         const types = config['types'] || {};
 249         const builders = config['builders'] || {};
 250
 251         assert(config.buildRequestArgument, 'buildRequestArgument must specify the name of the property used to store the build request ID');
 252
 253         assert.equal(typeof(config.repositoryGroups), 'object', 'repositoryGroups must specify a dictionary from the name to its definition');
 254
 255         const repositoryGroups = {};
 256         for (const name in config.repositoryGroups)
 257             repositoryGroups[name] = this._parseRepositoryGroup(name, config.repositoryGroups[name]);
 258
 259         const commonConfigurations = {
 260             repositoryGroups,
 261             slaveArgument: config.slaveArgument,
 262             buildRequestArgument: config.buildRequestArgument,
 263         };
 264
 265         let syncerByBuilder = new Map;
 266         const expandedConfigurations = [];
 267         for (let entry of config['configurations']) {
 268             for (const expandedConfig of this._expandTypesAndPlatforms(entry))
 269                 expandedConfigurations.push(expandedConfig);
 270         }
 271
 272         for (let entry of expandedConfigurations) {
 273             const mergedConfig = {};
 274             this._validateAndMergeConfig(mergedConfig, entry);
 275
 276             if ('type' in mergedConfig) {
 277                 const type = mergedConfig['type'];
 278                 assert(type, `${type} is not a valid type in the configuration`);
 279                 this._validateAndMergeConfig(mergedConfig, types[type]);
 280             }
 281
 282             const builder = entry['builder'];
 283             if (builders[builder])
 284                 this._validateAndMergeConfig(mergedConfig, builders[builder]);
 285
 286             this._createTestConfiguration(remote, syncerByBuilder, mergedConfig, commonConfigurations);
 287         }
 288
 289         return Array.from(syncerByBuilder.values());
 290     }
 291
 292     static _parseRepositoryGroup(name, group)
 293     {
 294         assert(Array.isArray(group.repositories), 'Each repository group must specify a list of repositories');
 295         assert(group.repositories.length, 'Each repository group must specify a list of repositories');
 296         assert(!('description' in group) || typeof(group['description']) == 'string', 'The description of a repository group must be a string');
 297         assert.equal(typeof(group.properties), 'object', 'Each repository group must specify a dictionary of properties');
 298
 299         const repositoryByName = {};
 300         const repositories = group.repositories.map((repositoryName) => {
 301             const repository = Repository.findTopLevelByName(repositoryName);
 302             assert(repository, `"${repositoryName}" is not a valid repository name`);
 303             repositoryByName[repositoryName] = repository;
 304             return repository;
 305         });
 306         const propertiesTemplate = {};
 307         const usedRepositories = [];
 308         for (const propertyName in group.properties) {
 309             let value = group.properties[propertyName];
 310             const match = value.match(/^<(.+)>$/);
 311             if (match) {
 312                 const repositoryName = match[1];
 313                 value = repositoryByName[repositoryName];
 314                 assert(value, `Repository group "${name}" uses "${repositoryName}" in its property but does not list in the list of repositories`);
 315                 usedRepositories.push(value);
 316             }
 317             propertiesTemplate[propertyName] = value;
 318         }
 319         assert.equal(repositories.length, usedRepositories.length, `Repository group "${name}" does not use some of the listed repositories`);
 320         return {
 321             name: group.name,
 322             description: group.description,
 323             propertiesTemplate,
 324             arguments: group.arguments,
 325             repositories: repositories.map((repository) => repository.id()),
 326         };
 327     }
 328
 329     static _expandTypesAndPlatforms(unresolvedConfig)
 330     {
 331         const typeExpanded = [];
 332         if ('types' in unresolvedConfig) {
 333             for (let type of unresolvedConfig['types'])
 334                 typeExpanded.push(this._validateAndMergeConfig({'type': type}, unresolvedConfig, 'types'));
 335         } else
 336             typeExpanded.push(unresolvedConfig);
 337
 338         const configurations = [];
 339         for (let config of typeExpanded) {
 340             if ('platforms' in config) {
 341                 for (let platform of config['platforms'])
 342                     configurations.push(this._validateAndMergeConfig({'platform': platform}, config, 'platforms'));
 343             } else
 344                 configurations.push(config);
 345         }
 346
 347         return configurations;
 348     }
 349
 350     static _createTestConfiguration(remote, syncerByBuilder, newConfig, commonConfigurations)
 351     {
 352         assert('platform' in newConfig, 'configuration must specify a platform');
 353         assert('test' in newConfig, 'configuration must specify a test');
 354         assert('builder' in newConfig, 'configuration must specify a builder');
 355
 356         const test = Test.findByPath(newConfig.test);
 357         assert(test, `${newConfig.test} is not a valid test path`);
 358
 359         const platform = Platform.findByName(newConfig.platform);
 360         assert(platform, `${newConfig.platform} is not a valid platform name`);
 361
 362         let syncer = syncerByBuilder.get(newConfig.builder);
 363         if (!syncer) {
 364             syncer = new BuildbotSyncer(remote, newConfig, commonConfigurations);
 365             syncerByBuilder.set(newConfig.builder, syncer);
 366         }
 367         syncer.addTestConfiguration(test, platform, newConfig.properties || {});
 368     }
 369
 370     static _validateAndMergeConfig(config, valuesToMerge, excludedProperty)
 371     {
 372         for (const name in valuesToMerge) {
 373             const value = valuesToMerge[name];
 374             if (name == excludedProperty)
 375                 continue;
 376
 377             switch (name) {
 378             case 'properties': // Fallthrough
 379             case 'arguments':
 380                 assert.equal(typeof(value), 'object', 'arguments should be a dictionary');
 381                 if (!config['properties'])
 382                     config['properties'] = {};
 383                 this._validateAndMergeProperties(config['properties'], value);
 384                 break;
 385             case 'test': // Fallthrough
 386             case 'slaveList': // Fallthrough
 387             case 'platforms': // Fallthrough
 388             case 'types':
 389                 assert(value instanceof Array, `${name} should be an array`);
 390                 assert(value.every(function (part) { return typeof part == 'string'; }), `${name} should be an array of strings`);
 391                 config[name] = value.slice();
 392                 break;
 393             case 'type': // Fallthrough
 394             case 'builder': // Fallthrough
 395             case 'platform': // Fallthrough
 396                 assert.equal(typeof(value), 'string', `${name} should be of string type`);
 397                 config[name] = value;
 398                 break;
 399             default:
 400                 assert(false, `Unrecognized parameter ${name}`);
 401             }
 402         }
 403         return config;
 404     }
 405
 406     static _validateAndMergeProperties(properties, configArguments)
 407     {
 408         for (let name in configArguments) {
 409             const value = configArguments[name];
 410             if (typeof(value) == 'string') {
 411                 properties[name] = value;
 412                 continue;
 413             }
 414             assert.equal(typeof(value), 'object', 'A argument value must be either a string or a dictionary');
 415
 416             const keys = Object.keys(value);
 417             assert.equal(keys.length, 1, 'arguments value cannot contain more than one key');
 418             let namedValue = value[keys[0]];
 419             switch (keys[0]) {
 420             case 'root':
 421                 assert.equal(typeof(namedValue), 'string', 'root name must be a string');
 422                 break;
 423             case 'rootOptions': // Fallthrough
 424             case 'rootsExcluding':
 425                 assert(namedValue instanceof Array, `${keys[0]} must specify an array`);
 426                 for (let excludedRootName of namedValue)
 427                     assert.equal(typeof(excludedRootName), 'string', `${keys[0]} must specify an array of strings`);
 428                 namedValue = namedValue.slice();
 429                 break;
 430             default:
 431                 assert(false, `Unrecognized named argument ${keys[0]}`);
 432             }
 433             properties[name] = {[keys[0]]: namedValue};
 434         }
 435     }
 436
 437 }
 438
 439 if (typeof module != 'undefined') {
 440     module.exports.BuildbotSyncer = BuildbotSyncer;
 441     module.exports.BuildbotBuildEntry = BuildbotBuildEntry;
 442 }