Websites/perf.webkit.org/tools/js/buildbot-syncer.js

   1 'use strict';
   2
   3 let assert = require('assert');
   4
   5 require('./v3-models.js');
   6
   7 class BuildbotBuildEntry {
   8     constructor(syncer, rawData)
   9     {
  10         assert.equal(syncer.builderName(), rawData['builderName']);
  11
  12         this._syncer = syncer;
  13         this._slaveName = null;
  14         this._buildRequestId = null;
  15         this._isInProgress = rawData['currentStep'] || (rawData['times'] && !rawData['times'][1]);
  16         this._buildNumber = rawData['number'];
  17
  18         for (let propertyTuple of (rawData['properties'] || [])) {
  19             // e.g. ['build_request_id', '16733', 'Force Build Form']
  20             const name = propertyTuple[0];
  21             const value = propertyTuple[1];
  22             if (name == syncer._slavePropertyName)
  23                 this._slaveName = value;
  24             else if (name == syncer._buildRequestPropertyName)
  25                 this._buildRequestId = value;
  26         }
  27     }
  28
  29     syncer() { return this._syncer; }
  30     buildNumber() { return this._buildNumber; }
  31     slaveName() { return this._slaveName; }
  32     buildRequestId() { return this._buildRequestId; }
  33     isPending() { return typeof(this._buildNumber) != 'number'; }
  34     isInProgress() { return this._isInProgress; }
  35     hasFinished() { return !this.isPending() && !this.isInProgress(); }
  36     url() { return this.isPending() ? this._syncer.url() : this._syncer.urlForBuildNumber(this._buildNumber); }
  37
  38     buildRequestStatusIfUpdateIsNeeded(request)
  39     {
  40         assert.equal(request.id(), this._buildRequestId);
  41         if (!request)
  42             return null;
  43         if (this.isPending()) {
  44             if (request.isPending())
  45                 return 'scheduled';
  46         } else if (this.isInProgress()) {
  47             if (!request.hasStarted() || request.isScheduled())
  48                 return 'running';
  49         } else if (this.hasFinished()) {
  50             if (!request.hasFinished())
  51                 return 'failedIfNotCompleted';
  52         }
  53         return null;
  54     }
  55 }
  56
  57
  58 class BuildbotSyncer {
  59
  60     constructor(remote, object)
  61     {
  62         this._remote = remote;
  63         this._testConfigurations = [];
  64         this._builderName = object.builder;
  65         this._slavePropertyName = object.slaveArgument;
  66         this._slaveList = object.slaveList;
  67         this._buildRequestPropertyName = object.buildRequestArgument;
  68         this._entryList = null;
  69         this._slavesWithNewRequests = new Set;
  70     }
  71
  72     builderName() { return this._builderName; }
  73
  74     addTestConfiguration(test, platform, propertiesTemplate)
  75     {
  76         assert(test instanceof Test);
  77         assert(platform instanceof Platform);
  78         this._testConfigurations.push({test, platform, propertiesTemplate});
  79     }
  80     testConfigurations() { return this._testConfigurations; }
  81
  82     matchesConfiguration(request)
  83     {
  84         return this._testConfigurations.some((config) => config.platform == request.platform() && config.test == request.test());
  85     }
  86
  87     scheduleRequest(newRequest, slaveName)
  88     {
  89         assert(!this._slavesWithNewRequests.has(slaveName));
  90         let properties = this._propertiesForBuildRequest(newRequest);
  91
  92         assert.equal(!this._slavePropertyName, !slaveName);
  93         if (this._slavePropertyName)
  94             properties[this._slavePropertyName] = slaveName;
  95
  96         this._slavesWithNewRequests.add(slaveName);
  97         return this._remote.postFormUrlencodedData(this.pathForForceBuild(), properties);
  98     }
  99
 100     scheduleRequestInGroupIfAvailable(newRequest, slaveName)
 101     {
 102         assert(newRequest instanceof BuildRequest);
 103
 104         if (!this.matchesConfiguration(newRequest))
 105             return null;
 106
 107         let hasPendingBuildsWithoutSlaveNameSpecified = false;
 108         let usedSlaves = new Set;
 109         for (let entry of this._entryList) {
 110             if (entry.isPending()) {
 111                 if (!entry.slaveName())
 112                     hasPendingBuildsWithoutSlaveNameSpecified = true;
 113                 usedSlaves.add(entry.slaveName());
 114             }
 115         }
 116
 117         if (!this._slaveList || hasPendingBuildsWithoutSlaveNameSpecified) {
 118             if (usedSlaves.size || this._slavesWithNewRequests.size)
 119                 return null;
 120             return this.scheduleRequest(newRequest, null);
 121         }
 122
 123         if (slaveName) {
 124             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
 125                 return this.scheduleRequest(newRequest, slaveName);
 126             return null;
 127         }
 128
 129         for (let slaveName of this._slaveList) {
 130             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
 131                 return this.scheduleRequest(newRequest, slaveName);
 132         }
 133
 134         return null;
 135     }
 136
 137     pullBuildbot(count)
 138     {
 139         return this._remote.getJSON(this.pathForPendingBuildsJSON()).then((content) => {
 140             let pendingEntries = content.map((entry) => new BuildbotBuildEntry(this, entry));
 141             return this._pullRecentBuilds(count).then((entries) => {
 142                 let entryByRequest = {};
 143
 144                 for (let entry of pendingEntries)
 145                     entryByRequest[entry.buildRequestId()] = entry;
 146
 147                 for (let entry of entries)
 148                     entryByRequest[entry.buildRequestId()] = entry;
 149
 150                 let entryList = [];
 151                 for (let id in entryByRequest)
 152                     entryList.push(entryByRequest[id]);
 153
 154                 this._entryList = entryList;
 155                 this._slavesWithNewRequests.clear();
 156
 157                 return entryList;
 158             });
 159         });
 160     }
 161
 162     _pullRecentBuilds(count)
 163     {
 164         if (!count)
 165             return Promise.resolve([]);
 166
 167         let selectedBuilds = new Array(count);
 168         for (let i = 0; i < count; i++)
 169             selectedBuilds[i] = -i - 1;
 170
 171         return this._remote.getJSON(this.pathForBuildJSON(selectedBuilds)).then((content) => {
 172             const entries = [];
 173             for (let index of selectedBuilds) {
 174                 const entry = content[index];
 175                 if (entry && !entry['error'])
 176                     entries.push(new BuildbotBuildEntry(this, entry));
 177             }
 178             return entries;
 179         });
 180     }
 181
 182     pathForPendingBuildsJSON() { return `/json/builders/${escape(this._builderName)}/pendingBuilds`; }
 183     pathForBuildJSON(selectedBuilds)
 184     {
 185         return `/json/builders/${escape(this._builderName)}/builds/?` + selectedBuilds.map((number) => 'select=' + number).join('&');
 186     }
 187     pathForForceBuild() { return `/builders/${escape(this._builderName)}/force`; }
 188
 189     url() { return this._remote.url(`/builders/${escape(this._builderName)}/`); }
 190     urlForBuildNumber(number) { return this._remote.url(`/builders/${escape(this._builderName)}/builds/${number}`); }
 191
 192     _propertiesForBuildRequest(buildRequest)
 193     {
 194         assert(buildRequest instanceof BuildRequest);
 195
 196         const commitSet = buildRequest.commitSet();
 197         assert(commitSet instanceof CommitSet);
 198
 199         const repositoryByName = {};
 200         for (let repository of commitSet.repositories())
 201             repositoryByName[repository.name()] = repository;
 202
 203         let propertiesTemplate = null;
 204         for (let config of this._testConfigurations) {
 205             if (config.platform == buildRequest.platform() && config.test == buildRequest.test())
 206                 propertiesTemplate = config.propertiesTemplate;
 207         }
 208         assert(propertiesTemplate);
 209
 210         const properties = {};
 211         for (let key in propertiesTemplate) {
 212             const value = propertiesTemplate[key];
 213             if (typeof(value) != 'object')
 214                 properties[key] = value;
 215             else if ('root' in value) {
 216                 const repositoryName = value['root'];
 217                 const repository = repositoryByName[repositoryName];
 218                 assert(repository, `"${repositoryName}" must be specified`);
 219                 properties[key] = commitSet.revisionForRepository(repository);
 220             } else if ('rootOptions' in value) {
 221                 const filteredOptions = value['rootOptions'].filter((option) => option in repositoryByName);
 222                 assert.equal(filteredOptions.length, 1, `There should be exactly one valid root among "${value['rootOptions']}".`);
 223                 properties[key] = commitSet.revisionForRepository(repositoryByName[filteredOptions[0]]);
 224             }
 225             else if ('rootsExcluding' in value) {
 226                 const revisionSet = this._revisionSetFromCommitSetWithExclusionList(commitSet, value['rootsExcluding']);
 227                 properties[key] = JSON.stringify(revisionSet);
 228             }
 229         }
 230
 231         properties[this._buildRequestPropertyName] = buildRequest.id();
 232
 233         return properties;
 234     }
 235
 236     _revisionSetFromCommitSetWithExclusionList(commitSet, exclusionList)
 237     {
 238         const revisionSet = {};
 239         for (let repository of commitSet.repositories()) {
 240             if (exclusionList.indexOf(repository.name()) >= 0)
 241                 continue;
 242             const commit = commitSet.commitForRepository(repository);
 243             revisionSet[repository.name()] = {
 244                 id: commit.id(),
 245                 time: +commit.time(),
 246                 repository: repository.name(),
 247                 revision: commit.revision(),
 248             };
 249         }
 250         return revisionSet;
 251     }
 252
 253     static _loadConfig(remote, config)
 254     {
 255         const shared = config['shared'] || {};
 256         const types = config['types'] || {};
 257         const builders = config['builders'] || {};
 258
 259         let syncerByBuilder = new Map;
 260         for (let entry of config['configurations']) {
 261             const newConfig = {};
 262             this._validateAndMergeConfig(newConfig, shared);
 263             this._validateAndMergeConfig(newConfig, entry);
 264
 265             const expandedConfigurations = this._expandTypesAndPlatforms(newConfig);
 266             for (let config of expandedConfigurations) {
 267                 if ('type' in config) {
 268                     const type = config['type'];
 269                     assert(type, `${type} is not a valid type in the configuration`);
 270                     this._validateAndMergeConfig(config, types[type]);
 271                 }
 272
 273                 const builder = entry['builder'];
 274                 if (builders[builder])
 275                     this._validateAndMergeConfig(config, builders[builder]);
 276
 277                 this._createTestConfiguration(remote, syncerByBuilder, config);
 278             }
 279         }
 280
 281         return Array.from(syncerByBuilder.values());
 282     }
 283
 284     static _expandTypesAndPlatforms(unresolvedConfig)
 285     {
 286         const typeExpanded = [];
 287         if ('types' in unresolvedConfig) {
 288             for (let type of unresolvedConfig['types'])
 289                 typeExpanded.push(this._validateAndMergeConfig({'type': type}, unresolvedConfig, 'types'));
 290         } else
 291             typeExpanded.push(unresolvedConfig);
 292
 293         const configurations = [];
 294         for (let config of typeExpanded) {
 295             if ('platforms' in config) {
 296                 for (let platform of config['platforms'])
 297                     configurations.push(this._validateAndMergeConfig({'platform': platform}, config, 'platforms'));
 298             } else
 299                 configurations.push(config);
 300         }
 301
 302         return configurations;
 303     }
 304
 305     static _createTestConfiguration(remote, syncerByBuilder, newConfig)
 306     {
 307         assert('platform' in newConfig, 'configuration must specify a platform');
 308         assert('test' in newConfig, 'configuration must specify a test');
 309         assert('builder' in newConfig, 'configuration must specify a builder');
 310         assert('properties' in newConfig, 'configuration must specify arguments to post on a builder');
 311         assert('buildRequestArgument' in newConfig, 'configuration must specify buildRequestArgument');
 312
 313         const test = Test.findByPath(newConfig.test);
 314         assert(test, `${newConfig.test} is not a valid test path`);
 315
 316         const platform = Platform.findByName(newConfig.platform);
 317         assert(platform, `${newConfig.platform} is not a valid platform name`);
 318
 319         let syncer = syncerByBuilder.get(newConfig.builder);
 320         if (!syncer) {
 321             syncer = new BuildbotSyncer(remote, newConfig);
 322             syncerByBuilder.set(newConfig.builder, syncer);
 323         }
 324         syncer.addTestConfiguration(test, platform, newConfig.properties);
 325     }
 326
 327     static _validateAndMergeConfig(config, valuesToMerge, excludedProperty)
 328     {
 329         for (const name in valuesToMerge) {
 330             const value = valuesToMerge[name];
 331             if (name == excludedProperty)
 332                 continue;
 333
 334             switch (name) {
 335             case 'properties': // Fallthrough
 336             case 'arguments':
 337                 assert.equal(typeof(value), 'object', 'arguments should be a dictionary');
 338                 if (!config['properties'])
 339                     config['properties'] = {};
 340                 this._validateAndMergeProperties(config['properties'], value);
 341                 break;
 342             case 'test': // Fallthrough
 343             case 'slaveList': // Fallthrough
 344             case 'platforms':
 345             case 'types':
 346                 assert(value instanceof Array, `${name} should be an array`);
 347                 assert(value.every(function (part) { return typeof part == 'string'; }), `${name} should be an array of strings`);
 348                 config[name] = value.slice();
 349                 break;
 350             case 'type': // Fallthrough
 351             case 'builder': // Fallthrough
 352             case 'platform': // Fallthrough
 353             case 'slaveArgument': // Fallthrough
 354             case 'buildRequestArgument':
 355                 assert.equal(typeof(value), 'string', `${name} should be of string type`);
 356                 config[name] = value;
 357                 break;
 358             default:
 359                 assert(false, `Unrecognized parameter ${name}`);
 360             }
 361         }
 362         return config;
 363     }
 364
 365     static _validateAndMergeProperties(properties, configArguments)
 366     {
 367         for (let name in configArguments) {
 368             const value = configArguments[name];
 369             if (typeof(value) == 'string') {
 370                 properties[name] = value;
 371                 continue;
 372             }
 373             assert.equal(typeof(value), 'object', 'A argument value must be either a string or a dictionary');
 374
 375             const keys = Object.keys(value);
 376             assert.equal(keys.length, 1, 'arguments value cannot contain more than one key');
 377             let namedValue = value[keys[0]];
 378             switch (keys[0]) {
 379             case 'root':
 380                 assert.equal(typeof(namedValue), 'string', 'root name must be a string');
 381                 break;
 382             case 'rootOptions': // Fallthrough
 383             case 'rootsExcluding':
 384                 assert(namedValue instanceof Array, `${keys[0]} must specify an array`);
 385                 for (let excludedRootName of namedValue)
 386                     assert.equal(typeof(excludedRootName), 'string', `${keys[0]} must specify an array of strings`);
 387                 namedValue = namedValue.slice();
 388                 break;
 389             default:
 390                 assert(false, `Unrecognized named argument ${keys[0]}`);
 391             }
 392             properties[name] = {[keys[0]]: namedValue};
 393         }
 394     }
 395
 396 }
 397
 398 if (typeof module != 'undefined') {
 399     module.exports.BuildbotSyncer = BuildbotSyncer;
 400     module.exports.BuildbotBuildEntry = BuildbotBuildEntry;
 401 }