Websites/perf.webkit.org/tools/js/buildbot-syncer.js

   1 'use strict';
   2
   3 let assert = require('assert');
   4
   5 require('./v3-models.js');
   6
   7 class BuildbotBuildEntry {
   8     constructor(syncer, rawData)
   9     {
  10         assert.equal(syncer.builderName(), rawData['builderName']);
  11
  12         this._syncer = syncer;
  13         this._slaveName = null;
  14         this._buildRequestId = null;
  15         this._isInProgress = rawData['currentStep'] || (rawData['times'] && !rawData['times'][1]);
  16         this._buildNumber = rawData['number'];
  17
  18         for (let propertyTuple of (rawData['properties'] || [])) {
  19             // e.g. ['build_request_id', '16733', 'Force Build Form']
  20             let name = propertyTuple[0];
  21             let value = propertyTuple[1];
  22             if (name == syncer._slavePropertyName)
  23                 this._slaveName = value;
  24             else if (name == syncer._buildRequestPropertyName)
  25                 this._buildRequestId = value;
  26         }
  27     }
  28
  29     syncer() { return this._syncer; }
  30     buildNumber() { return this._buildNumber; }
  31     slaveName() { return this._slaveName; }
  32     buildRequestId() { return this._buildRequestId; }
  33     isPending() { return !this._buildNumber; }
  34     isInProgress() { return this._isInProgress; }
  35     hasFinished() { return !this.isPending() && !this.isInProgress(); }
  36     url() { return this.isPending() ? this._syncer.url() : this._syncer.urlForBuildNumber(this._buildNumber); }
  37
  38     buildRequestStatusIfUpdateIsNeeded(request)
  39     {
  40         assert.equal(request.id(), this._buildRequestId);
  41         if (!request)
  42             return null;
  43         if (this.isPending()) {
  44             if (request.isPending())
  45                 return 'scheduled';
  46         } else if (this.isInProgress()) {
  47             if (!request.hasStarted())
  48                 return 'running';
  49         } else if (this.hasFinished()) {
  50             if (!request.hasFinished())
  51                 return 'failedIfNotCompleted';
  52         }
  53         return null;
  54     }
  55 }
  56
  57
  58 class BuildbotSyncer {
  59
  60     constructor(remote, object)
  61     {
  62         this._remote = remote;
  63         this._testConfigurations = [];
  64         this._builderName = object.builder;
  65         this._slavePropertyName = object.slaveArgument;
  66         this._slaveList = object.slaveList;
  67         this._buildRequestPropertyName = object.buildRequestArgument;
  68         this._entryList = null;
  69         this._slavesWithNewRequests = new Set;
  70     }
  71
  72     builderName() { return this._builderName; }
  73
  74     addTestConfiguration(test, platform, propertiesTemplate)
  75     {
  76         assert(test instanceof Test);
  77         assert(platform instanceof Platform);
  78         this._testConfigurations.push({test: test, platform: platform, propertiesTemplate: propertiesTemplate});
  79     }
  80     testConfigurations() { return this._testConfigurations; }
  81
  82     matchesConfiguration(request)
  83     {
  84         for (let config of this._testConfigurations) {
  85             if (config.platform == request.platform() && config.test == request.test())
  86                 return true;
  87         }
  88         return false;
  89     }
  90
  91     scheduleRequest(newRequest, slaveName)
  92     {
  93         assert(!this._slavesWithNewRequests.has(slaveName));
  94         let properties = this._propertiesForBuildRequest(newRequest);
  95
  96         assert.equal(!this._slavePropertyName, !slaveName);
  97         if (this._slavePropertyName)
  98             properties[this._slavePropertyName] = slaveName;
  99
 100         this._slavesWithNewRequests.add(slaveName);
 101         return this._remote.postFormUrlencodedData(this.pathForForceBuild(), properties);
 102     }
 103
 104     scheduleFirstRequestInGroupIfAvailable(newRequest)
 105     {
 106         assert(newRequest instanceof BuildRequest);
 107
 108         if (!this.matchesConfiguration(newRequest))
 109             return null;
 110
 111         let hasPendingBuildsWithoutSlaveNameSpecified = false;
 112         let usedSlaves = new Set;
 113         for (let entry of this._entryList) {
 114             if (entry.isPending()) {
 115                 if (!entry.slaveName())
 116                     hasPendingBuildsWithoutSlaveNameSpecified = true;
 117                 usedSlaves.add(entry.slaveName());
 118             }
 119         }
 120
 121         if (!this._slaveList || hasPendingBuildsWithoutSlaveNameSpecified) {
 122             if (usedSlaves.size || this._slavesWithNewRequests.size)
 123                 return null;
 124             return this.scheduleRequest(newRequest, null);
 125         }
 126
 127         for (let slaveName of this._slaveList) {
 128             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
 129                 return this.scheduleRequest(newRequest, slaveName);
 130         }
 131
 132         return null;
 133     }
 134
 135     pullBuildbot(count)
 136     {
 137         let self = this;
 138         return this._remote.getJSON(this.pathForPendingBuildsJSON()).then(function (content) {
 139             let pendingEntries = content.map(function (entry) { return new BuildbotBuildEntry(self, entry); });
 140             return self._pullRecentBuilds(count).then(function (entries) {
 141                 let entryByRequest = {};
 142
 143                 for (let entry of pendingEntries)
 144                     entryByRequest[entry.buildRequestId()] = entry;
 145
 146                 for (let entry of entries)
 147                     entryByRequest[entry.buildRequestId()] = entry;
 148
 149                 let entryList = [];
 150                 for (let id in entryByRequest)
 151                     entryList.push(entryByRequest[id]);
 152
 153                 self._entryList = entryList;
 154                 self._slavesWithNewRequests.clear();
 155
 156                 return entryList;
 157             });
 158         });
 159     }
 160
 161     _pullRecentBuilds(count)
 162     {
 163         if (!count)
 164             return Promise.resolve([]);
 165
 166         let selectedBuilds = new Array(count);
 167         for (let i = 0; i < count; i++)
 168             selectedBuilds[i] = -i - 1;
 169
 170         let self = this;
 171         return this._remote.getJSON(this.pathForBuildJSON(selectedBuilds)).then(function (content) {
 172             var entries = [];
 173             for (let index of selectedBuilds) {
 174                 let entry = content[index];
 175                 if (entry && !entry['error'])
 176                     entries.push(new BuildbotBuildEntry(self, entry));
 177             }
 178             return entries;
 179         });
 180     }
 181
 182     pathForPendingBuildsJSON() { return `/json/builders/${this._builderName}/pendingBuilds`; }
 183     pathForBuildJSON(selectedBuilds)
 184     {
 185         return `/json/builders/${this._builderName}/builds/?`
 186             + selectedBuilds.map(function (number) { return 'select=' + number; }).join('&');
 187     }
 188     pathForForceBuild() { return `/builders/${this._builderName}/force`; }
 189
 190     url() { return this._remote.url(`/builders/${this._builderName}/`); }
 191     urlForBuildNumber(number) { return this._remote.url(`/builders/${this._builderName}/builds/${number}`); }
 192
 193     _propertiesForBuildRequest(buildRequest)
 194     {
 195         assert(buildRequest instanceof BuildRequest);
 196
 197         let rootSet = buildRequest.rootSet();
 198         assert(rootSet instanceof RootSet);
 199
 200         let repositoryByName = {};
 201         for (let repository of rootSet.repositories())
 202             repositoryByName[repository.name()] = repository;
 203
 204         let propertiesTemplate = null;
 205         for (let config of this._testConfigurations) {
 206             if (config.platform == buildRequest.platform() && config.test == buildRequest.test())
 207                 propertiesTemplate = config.propertiesTemplate;
 208         }
 209         assert(propertiesTemplate);
 210
 211         let properties = {};
 212         for (let key in propertiesTemplate) {
 213             let value = propertiesTemplate[key];
 214             if (typeof(value) != 'object')
 215                 properties[key] = value;
 216             else if ('root' in value) {
 217                 let repositoryName = value['root'];
 218                 let repository = repositoryByName[repositoryName];
 219                 assert(repository, '"${repositoryName}" must be specified');
 220                 properties[key] = rootSet.revisionForRepository(repository);
 221             } else if ('rootsExcluding' in value) {
 222                 let revisionSet = this._revisionSetFromRootSetWithExclusionList(rootSet, value['rootsExcluding']);
 223                 properties[key] = JSON.stringify(revisionSet);
 224             }
 225         }
 226
 227         properties[this._buildRequestPropertyName] = buildRequest.id();
 228
 229         return properties;
 230     }
 231
 232     _revisionSetFromRootSetWithExclusionList(rootSet, exclusionList)
 233     {
 234         let revisionSet = {};
 235         for (let repository of rootSet.repositories()) {
 236             if (exclusionList.indexOf(repository.name()) >= 0)
 237                 continue;
 238             let commit = rootSet.commitForRepository(repository);
 239             revisionSet[repository.name()] = {
 240                 id: commit.id(),
 241                 time: +commit.time(),
 242                 repository: repository.name(),
 243                 revision: commit.revision(),
 244             };
 245         }
 246         return revisionSet;
 247     }
 248
 249     static _loadConfig(remote, config)
 250     {
 251         let shared = config['shared'] || {};
 252         let types = config['types'] || {};
 253         let builders = config['builders'] || {};
 254
 255         let syncerByBuilder = new Map;
 256         for (let entry of config['configurations']) {
 257             let newConfig = {};
 258             this._validateAndMergeConfig(newConfig, shared);
 259
 260             this._validateAndMergeConfig(newConfig, entry);
 261
 262             let type = entry['type'];
 263             if (type) {
 264                 assert(types[type]);
 265                 this._validateAndMergeConfig(newConfig, types[type]);
 266             }
 267
 268             let builder = entry['builder'];
 269             if (builders[builder])
 270                 this._validateAndMergeConfig(newConfig, builders[builder]);
 271
 272             assert('platform' in newConfig, 'configuration must specify a platform');
 273             assert('test' in newConfig, 'configuration must specify a test');
 274             assert('builder' in newConfig, 'configuration must specify a builder');
 275             assert('properties' in newConfig, 'configuration must specify arguments to post on a builder');
 276             assert('buildRequestArgument' in newConfig, 'configuration must specify buildRequestArgument');
 277
 278             let test = Test.findByPath(newConfig.test);
 279             assert(test, `${newConfig.test} is not a valid test path`);
 280
 281             let platform = Platform.findByName(newConfig.platform);
 282             assert(platform, `${newConfig.platform} is not a valid platform name`);
 283
 284             let syncer = syncerByBuilder.get(newConfig.builder);
 285             if (!syncer) {
 286                 syncer = new BuildbotSyncer(remote, newConfig);
 287                 syncerByBuilder.set(newConfig.builder, syncer);
 288             }
 289             syncer.addTestConfiguration(test, platform, newConfig.properties);
 290         }
 291
 292         return Array.from(syncerByBuilder.values());
 293     }
 294
 295     static _validateAndMergeConfig(config, valuesToMerge)
 296     {
 297         for (let name in valuesToMerge) {
 298             let value = valuesToMerge[name];
 299             switch (name) {
 300             case 'arguments':
 301                 assert.equal(typeof(value), 'object', 'arguments should be a dictionary');
 302                 if (!config['properties'])
 303                     config['properties'] = {};
 304                 this._validateAndMergeProperties(config['properties'], value);
 305                 break;
 306             case 'test':
 307                 assert(value instanceof Array, 'test should be an array');
 308                 assert(value.every(function (part) { return typeof part == 'string'; }), 'test should be an array of strings');
 309                 config[name] = value.slice();
 310                 break;
 311             case 'slaveList':
 312                 assert(value instanceof Array, 'slaveList should be an array');
 313                 assert(value.every(function (part) { return typeof part == 'string'; }), 'slaveList should be an array of strings');
 314                 config[name] = value;
 315                 break;
 316             case 'type': // fallthrough
 317             case 'builder': // fallthrough
 318             case 'platform': // fallthrough
 319             case 'slaveArgument': // fallthrough
 320             case 'buildRequestArgument':
 321                 assert.equal(typeof(value), 'string', `${name} should be of string type`);
 322                 config[name] = value;
 323                 break;
 324             default:
 325                 assert(false, `Unrecognized parameter ${name}`);
 326             }
 327         }
 328     }
 329
 330     static _validateAndMergeProperties(properties, configArguments)
 331     {
 332         for (let name in configArguments) {
 333             let value = configArguments[name];
 334             if (typeof(value) == 'string') {
 335                 properties[name] = value;
 336                 continue;
 337             }
 338             assert.equal(typeof(value), 'object', 'A argument value must be either a string or a dictionary');
 339
 340             let keys = Object.keys(value);
 341             assert.equal(keys.length, 1, 'arguments value cannot contain more than one key');
 342             let namedValue = value[keys[0]];
 343             switch (keys[0]) {
 344             case 'root':
 345                 assert.equal(typeof(namedValue), 'string', 'root name must be a string');
 346                 break;
 347             case 'rootsExcluding':
 348                 assert(namedValue instanceof Array, 'rootsExcluding must specify an array');
 349                 for (let excludedRootName of namedValue)
 350                     assert.equal(typeof(excludedRootName), 'string', 'rootsExcluding must specify an array of strings');
 351                 namedValue = namedValue.slice();
 352                 break;
 353             default:
 354                 assert(false, `Unrecognized named argument ${keys[0]}`);
 355             }
 356             properties[name] = {[keys[0]]: namedValue};
 357         }
 358     }
 359
 360 }
 361
 362 if (typeof module != 'undefined') {
 363     module.exports.BuildbotSyncer = BuildbotSyncer;
 364     module.exports.BuildbotBuildEntry = BuildbotBuildEntry;
 365 }