Websites/perf.webkit.org/tools/js/buildbot-syncer.js

   1 'use strict';
   2
   3 let assert = require('assert');
   4
   5 require('./v3-models.js');
   6
   7 class BuildbotBuildEntry {
   8     constructor(syncer, rawData)
   9     {
  10         assert.equal(syncer.builderName(), rawData['builderName']);
  11
  12         this._syncer = syncer;
  13         this._slaveName = null;
  14         this._buildRequestId = null;
  15         this._isInProgress = rawData['currentStep'] || (rawData['times'] && !rawData['times'][1]);
  16         this._buildNumber = rawData['number'];
  17
  18         for (let propertyTuple of (rawData['properties'] || [])) {
  19             // e.g. ['build_request_id', '16733', 'Force Build Form']
  20             let name = propertyTuple[0];
  21             let value = propertyTuple[1];
  22             if (name == syncer._slavePropertyName)
  23                 this._slaveName = value;
  24             else if (name == syncer._buildRequestPropertyName)
  25                 this._buildRequestId = value;
  26         }
  27     }
  28
  29     syncer() { return this._syncer; }
  30     buildNumber() { return this._buildNumber; }
  31     slaveName() { return this._slaveName; }
  32     buildRequestId() { return this._buildRequestId; }
  33     isPending() { return typeof(this._buildNumber) != 'number'; }
  34     isInProgress() { return this._isInProgress; }
  35     hasFinished() { return !this.isPending() && !this.isInProgress(); }
  36     url() { return this.isPending() ? this._syncer.url() : this._syncer.urlForBuildNumber(this._buildNumber); }
  37
  38     buildRequestStatusIfUpdateIsNeeded(request)
  39     {
  40         assert.equal(request.id(), this._buildRequestId);
  41         if (!request)
  42             return null;
  43         if (this.isPending()) {
  44             if (request.isPending())
  45                 return 'scheduled';
  46         } else if (this.isInProgress()) {
  47             if (!request.hasStarted() || request.isScheduled())
  48                 return 'running';
  49         } else if (this.hasFinished()) {
  50             if (!request.hasFinished())
  51                 return 'failedIfNotCompleted';
  52         }
  53         return null;
  54     }
  55 }
  56
  57
  58 class BuildbotSyncer {
  59
  60     constructor(remote, object)
  61     {
  62         this._remote = remote;
  63         this._testConfigurations = [];
  64         this._builderName = object.builder;
  65         this._slavePropertyName = object.slaveArgument;
  66         this._slaveList = object.slaveList;
  67         this._buildRequestPropertyName = object.buildRequestArgument;
  68         this._entryList = null;
  69         this._slavesWithNewRequests = new Set;
  70     }
  71
  72     builderName() { return this._builderName; }
  73
  74     addTestConfiguration(test, platform, propertiesTemplate)
  75     {
  76         assert(test instanceof Test);
  77         assert(platform instanceof Platform);
  78         this._testConfigurations.push({test: test, platform: platform, propertiesTemplate: propertiesTemplate});
  79     }
  80     testConfigurations() { return this._testConfigurations; }
  81
  82     matchesConfiguration(request)
  83     {
  84         for (let config of this._testConfigurations) {
  85             if (config.platform == request.platform() && config.test == request.test())
  86                 return true;
  87         }
  88         return false;
  89     }
  90
  91     scheduleRequest(newRequest, slaveName)
  92     {
  93         assert(!this._slavesWithNewRequests.has(slaveName));
  94         let properties = this._propertiesForBuildRequest(newRequest);
  95
  96         assert.equal(!this._slavePropertyName, !slaveName);
  97         if (this._slavePropertyName)
  98             properties[this._slavePropertyName] = slaveName;
  99
 100         this._slavesWithNewRequests.add(slaveName);
 101         return this._remote.postFormUrlencodedData(this.pathForForceBuild(), properties);
 102     }
 103
 104     scheduleRequestInGroupIfAvailable(newRequest, slaveName)
 105     {
 106         assert(newRequest instanceof BuildRequest);
 107
 108         if (!this.matchesConfiguration(newRequest))
 109             return null;
 110
 111         let hasPendingBuildsWithoutSlaveNameSpecified = false;
 112         let usedSlaves = new Set;
 113         for (let entry of this._entryList) {
 114             if (entry.isPending()) {
 115                 if (!entry.slaveName())
 116                     hasPendingBuildsWithoutSlaveNameSpecified = true;
 117                 usedSlaves.add(entry.slaveName());
 118             }
 119         }
 120
 121         if (!this._slaveList || hasPendingBuildsWithoutSlaveNameSpecified) {
 122             if (usedSlaves.size || this._slavesWithNewRequests.size)
 123                 return null;
 124             return this.scheduleRequest(newRequest, null);
 125         }
 126
 127         if (slaveName) {
 128             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
 129                 return this.scheduleRequest(newRequest, slaveName);
 130             return null;
 131         }
 132
 133         for (let slaveName of this._slaveList) {
 134             if (!usedSlaves.has(slaveName) && !this._slavesWithNewRequests.has(slaveName))
 135                 return this.scheduleRequest(newRequest, slaveName);
 136         }
 137
 138         return null;
 139     }
 140
 141     pullBuildbot(count)
 142     {
 143         let self = this;
 144         return this._remote.getJSON(this.pathForPendingBuildsJSON()).then(function (content) {
 145             let pendingEntries = content.map(function (entry) { return new BuildbotBuildEntry(self, entry); });
 146             return self._pullRecentBuilds(count).then(function (entries) {
 147                 let entryByRequest = {};
 148
 149                 for (let entry of pendingEntries)
 150                     entryByRequest[entry.buildRequestId()] = entry;
 151
 152                 for (let entry of entries)
 153                     entryByRequest[entry.buildRequestId()] = entry;
 154
 155                 let entryList = [];
 156                 for (let id in entryByRequest)
 157                     entryList.push(entryByRequest[id]);
 158
 159                 self._entryList = entryList;
 160                 self._slavesWithNewRequests.clear();
 161
 162                 return entryList;
 163             });
 164         });
 165     }
 166
 167     _pullRecentBuilds(count)
 168     {
 169         if (!count)
 170             return Promise.resolve([]);
 171
 172         let selectedBuilds = new Array(count);
 173         for (let i = 0; i < count; i++)
 174             selectedBuilds[i] = -i - 1;
 175
 176         let self = this;
 177         return this._remote.getJSON(this.pathForBuildJSON(selectedBuilds)).then(function (content) {
 178             var entries = [];
 179             for (let index of selectedBuilds) {
 180                 let entry = content[index];
 181                 if (entry && !entry['error'])
 182                     entries.push(new BuildbotBuildEntry(self, entry));
 183             }
 184             return entries;
 185         });
 186     }
 187
 188     pathForPendingBuildsJSON() { return `/json/builders/${escape(this._builderName)}/pendingBuilds`; }
 189     pathForBuildJSON(selectedBuilds)
 190     {
 191         return `/json/builders/${escape(this._builderName)}/builds/?`
 192             + selectedBuilds.map(function (number) { return 'select=' + number; }).join('&');
 193     }
 194     pathForForceBuild() { return `/builders/${escape(this._builderName)}/force`; }
 195
 196     url() { return this._remote.url(`/builders/${escape(this._builderName)}/`); }
 197     urlForBuildNumber(number) { return this._remote.url(`/builders/${escape(this._builderName)}/builds/${number}`); }
 198
 199     _propertiesForBuildRequest(buildRequest)
 200     {
 201         assert(buildRequest instanceof BuildRequest);
 202
 203         let rootSet = buildRequest.rootSet();
 204         assert(rootSet instanceof RootSet);
 205
 206         let repositoryByName = {};
 207         for (let repository of rootSet.repositories())
 208             repositoryByName[repository.name()] = repository;
 209
 210         let propertiesTemplate = null;
 211         for (let config of this._testConfigurations) {
 212             if (config.platform == buildRequest.platform() && config.test == buildRequest.test())
 213                 propertiesTemplate = config.propertiesTemplate;
 214         }
 215         assert(propertiesTemplate);
 216
 217         let properties = {};
 218         for (let key in propertiesTemplate) {
 219             let value = propertiesTemplate[key];
 220             if (typeof(value) != 'object')
 221                 properties[key] = value;
 222             else if ('root' in value) {
 223                 let repositoryName = value['root'];
 224                 let repository = repositoryByName[repositoryName];
 225                 assert(repository, '"${repositoryName}" must be specified');
 226                 properties[key] = rootSet.revisionForRepository(repository);
 227             } else if ('rootsExcluding' in value) {
 228                 let revisionSet = this._revisionSetFromRootSetWithExclusionList(rootSet, value['rootsExcluding']);
 229                 properties[key] = JSON.stringify(revisionSet);
 230             }
 231         }
 232
 233         properties[this._buildRequestPropertyName] = buildRequest.id();
 234
 235         return properties;
 236     }
 237
 238     _revisionSetFromRootSetWithExclusionList(rootSet, exclusionList)
 239     {
 240         let revisionSet = {};
 241         for (let repository of rootSet.repositories()) {
 242             if (exclusionList.indexOf(repository.name()) >= 0)
 243                 continue;
 244             let commit = rootSet.commitForRepository(repository);
 245             revisionSet[repository.name()] = {
 246                 id: commit.id(),
 247                 time: +commit.time(),
 248                 repository: repository.name(),
 249                 revision: commit.revision(),
 250             };
 251         }
 252         return revisionSet;
 253     }
 254
 255     static _loadConfig(remote, config)
 256     {
 257         let shared = config['shared'] || {};
 258         let types = config['types'] || {};
 259         let builders = config['builders'] || {};
 260
 261         let syncerByBuilder = new Map;
 262         for (let entry of config['configurations']) {
 263             let newConfig = {};
 264             this._validateAndMergeConfig(newConfig, shared);
 265
 266             this._validateAndMergeConfig(newConfig, entry);
 267
 268             let type = entry['type'];
 269             if (type) {
 270                 assert(types[type]);
 271                 this._validateAndMergeConfig(newConfig, types[type]);
 272             }
 273
 274             let builder = entry['builder'];
 275             if (builders[builder])
 276                 this._validateAndMergeConfig(newConfig, builders[builder]);
 277
 278             assert('platform' in newConfig, 'configuration must specify a platform');
 279             assert('test' in newConfig, 'configuration must specify a test');
 280             assert('builder' in newConfig, 'configuration must specify a builder');
 281             assert('properties' in newConfig, 'configuration must specify arguments to post on a builder');
 282             assert('buildRequestArgument' in newConfig, 'configuration must specify buildRequestArgument');
 283
 284             let test = Test.findByPath(newConfig.test);
 285             assert(test, `${newConfig.test} is not a valid test path`);
 286
 287             let platform = Platform.findByName(newConfig.platform);
 288             assert(platform, `${newConfig.platform} is not a valid platform name`);
 289
 290             let syncer = syncerByBuilder.get(newConfig.builder);
 291             if (!syncer) {
 292                 syncer = new BuildbotSyncer(remote, newConfig);
 293                 syncerByBuilder.set(newConfig.builder, syncer);
 294             }
 295             syncer.addTestConfiguration(test, platform, newConfig.properties);
 296         }
 297
 298         return Array.from(syncerByBuilder.values());
 299     }
 300
 301     static _validateAndMergeConfig(config, valuesToMerge)
 302     {
 303         for (let name in valuesToMerge) {
 304             let value = valuesToMerge[name];
 305             switch (name) {
 306             case 'arguments':
 307                 assert.equal(typeof(value), 'object', 'arguments should be a dictionary');
 308                 if (!config['properties'])
 309                     config['properties'] = {};
 310                 this._validateAndMergeProperties(config['properties'], value);
 311                 break;
 312             case 'test':
 313                 assert(value instanceof Array, 'test should be an array');
 314                 assert(value.every(function (part) { return typeof part == 'string'; }), 'test should be an array of strings');
 315                 config[name] = value.slice();
 316                 break;
 317             case 'slaveList':
 318                 assert(value instanceof Array, 'slaveList should be an array');
 319                 assert(value.every(function (part) { return typeof part == 'string'; }), 'slaveList should be an array of strings');
 320                 config[name] = value;
 321                 break;
 322             case 'type': // fallthrough
 323             case 'builder': // fallthrough
 324             case 'platform': // fallthrough
 325             case 'slaveArgument': // fallthrough
 326             case 'buildRequestArgument':
 327                 assert.equal(typeof(value), 'string', `${name} should be of string type`);
 328                 config[name] = value;
 329                 break;
 330             default:
 331                 assert(false, `Unrecognized parameter ${name}`);
 332             }
 333         }
 334     }
 335
 336     static _validateAndMergeProperties(properties, configArguments)
 337     {
 338         for (let name in configArguments) {
 339             let value = configArguments[name];
 340             if (typeof(value) == 'string') {
 341                 properties[name] = value;
 342                 continue;
 343             }
 344             assert.equal(typeof(value), 'object', 'A argument value must be either a string or a dictionary');
 345
 346             let keys = Object.keys(value);
 347             assert.equal(keys.length, 1, 'arguments value cannot contain more than one key');
 348             let namedValue = value[keys[0]];
 349             switch (keys[0]) {
 350             case 'root':
 351                 assert.equal(typeof(namedValue), 'string', 'root name must be a string');
 352                 break;
 353             case 'rootsExcluding':
 354                 assert(namedValue instanceof Array, 'rootsExcluding must specify an array');
 355                 for (let excludedRootName of namedValue)
 356                     assert.equal(typeof(excludedRootName), 'string', 'rootsExcluding must specify an array of strings');
 357                 namedValue = namedValue.slice();
 358                 break;
 359             default:
 360                 assert(false, `Unrecognized named argument ${keys[0]}`);
 361             }
 362             properties[name] = {[keys[0]]: namedValue};
 363         }
 364     }
 365
 366 }
 367
 368 if (typeof module != 'undefined') {
 369     module.exports.BuildbotSyncer = BuildbotSyncer;
 370     module.exports.BuildbotBuildEntry = BuildbotBuildEntry;
 371 }