A/B testing bots should prioritize user created test groups
[WebKit-https.git] / Websites / perf.webkit.org / unit-tests / buildbot-syncer-tests.js
1 'use strict';
2
3 let assert = require('assert');
4
5 require('../tools/js/v3-models.js');
6 let MockRemoteAPI = require('./resources/mock-remote-api.js').MockRemoteAPI;
7 let MockModels = require('./resources/mock-v3-models.js').MockModels;
8
9 let BuildbotBuildEntry = require('../tools/js/buildbot-syncer.js').BuildbotBuildEntry;
10 let BuildbotSyncer = require('../tools/js/buildbot-syncer.js').BuildbotSyncer;
11
12 function sampleiOSConfig()
13 {
14     return {
15         'shared':
16             {
17                 'arguments': {
18                     'desired_image': {'root': 'iOS'},
19                     'roots_dict': {'rootsExcluding': ['iOS']}
20                 },
21                 'slaveArgument': 'slavename',
22                 'buildRequestArgument': 'build_request_id'
23             },
24         'types': {
25             'speedometer': {
26                 'test': ['Speedometer'],
27                 'arguments': {'test_name': 'speedometer'}
28             },
29             'jetstream': {
30                 'test': ['JetStream'],
31                 'arguments': {'test_name': 'jetstream'}
32             },
33             'dromaeo-dom': {
34                 'test': ['Dromaeo', 'DOM Core Tests'],
35                 'arguments': {'tests': 'dromaeo-dom'}
36             },
37         },
38         'builders': {
39             'iPhone-bench': {
40                 'builder': 'ABTest-iPhone-RunBenchmark-Tests',
41                 'arguments': { 'forcescheduler': 'ABTest-iPhone-RunBenchmark-Tests-ForceScheduler' },
42                 'slaveList': ['ABTest-iPhone-0'],
43             },
44             'iPad-bench': {
45                 'builder': 'ABTest-iPad-RunBenchmark-Tests',
46                 'arguments': { 'forcescheduler': 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler' },
47                 'slaveList': ['ABTest-iPad-0', 'ABTest-iPad-1'],
48             }
49         },
50         'configurations': [
51             {'type': 'speedometer', 'builder': 'iPhone-bench', 'platform': 'iPhone'},
52             {'type': 'jetstream', 'builder': 'iPhone-bench', 'platform': 'iPhone'},
53             {'type': 'dromaeo-dom', 'builder': 'iPhone-bench', 'platform': 'iPhone'},
54
55             {'type': 'speedometer', 'builder': 'iPad-bench', 'platform': 'iPad'},
56             {'type': 'jetstream', 'builder': 'iPad-bench', 'platform': 'iPad'},
57         ]
58     };
59 }
60
61 let sampleRootSetData = {
62     'WebKit': {
63         'id': '111127',
64         'time': 1456955807334,
65         'repository': 'WebKit',
66         'revision': '197463',
67     },
68     'Shared': {
69         'id': '111237',
70         'time': 1456931874000,
71         'repository': 'Shared',
72         'revision': '80229',
73     }
74 };
75
76 function smallConfiguration()
77 {
78     return {
79         'builder': 'some builder',
80         'platform': 'Some platform',
81         'test': ['Some test'],
82         'arguments': {},
83         'buildRequestArgument': 'id'};
84 }
85
86 function smallPendingBuild()
87 {
88     return {
89         'builderName': 'some builder',
90         'builds': [],
91         'properties': [],
92         'source': {
93             'branch': '',
94             'changes': [],
95             'codebase': 'WebKit',
96             'hasPatch': false,
97             'project': '',
98             'repository': '',
99             'revision': ''
100         },
101     };
102 }
103
104 function smallInProgressBuild()
105 {
106     return {
107         'builderName': 'some builder',
108         'builds': [],
109         'properties': [],
110         'currentStep': { },
111         'eta': 123,
112         'number': 456,
113         'source': {
114             'branch': '',
115             'changes': [],
116             'codebase': 'WebKit',
117             'hasPatch': false,
118             'project': '',
119             'repository': '',
120             'revision': ''
121         },
122     };
123 }
124
125 function smallFinishedBuild()
126 {
127     return {
128         'builderName': 'some builder',
129         'builds': [],
130         'properties': [],
131         'currentStep': null,
132         'eta': null,
133         'number': 789,
134         'source': {
135             'branch': '',
136             'changes': [],
137             'codebase': 'WebKit',
138             'hasPatch': false,
139             'project': '',
140             'repository': '',
141             'revision': ''
142         },
143         'times': [0, 1],
144     };
145 }
146
147 function createSampleBuildRequest(platform, test)
148 {
149     assert(platform instanceof Platform);
150     assert(test instanceof Test);
151
152     let rootSet = RootSet.ensureSingleton('4197', {roots: [
153         {'id': '111127', 'time': 1456955807334, 'repository': MockModels.webkit, 'revision': '197463'},
154         {'id': '111237', 'time': 1456931874000, 'repository': MockModels.sharedRepository, 'revision': '80229'},
155         {'id': '88930', 'time': 0, 'repository': MockModels.ios, 'revision': '13A452'},
156     ]});
157
158     let request = BuildRequest.ensureSingleton('16733-' + platform.id(), {'rootSet': rootSet, 'status': 'pending', 'platform': platform, 'test': test});
159     return request;
160 }
161
162 function samplePendingBuild(buildRequestId, buildTime, slaveName)
163 {
164     return {
165         'builderName': 'ABTest-iPad-RunBenchmark-Tests',
166         'builds': [],
167         'properties': [
168             ['build_request_id', buildRequestId || '16733', 'Force Build Form'],
169             ['desired_image', '13A452', 'Force Build Form'],
170             ['owner', '<unknown>', 'Force Build Form'],
171             ['test_name', 'speedometer', 'Force Build Form'],
172             ['reason', 'force build','Force Build Form'],
173             [
174                 'roots_dict',
175                 JSON.stringify(sampleRootSetData),
176                 'Force Build Form'
177             ],
178             ['slavename', slaveName, ''],
179             ['scheduler', 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler', 'Scheduler']
180         ],
181         'source': {
182             'branch': '',
183             'changes': [],
184             'codebase': 'compiler-rt',
185             'hasPatch': false,
186             'project': '',
187             'repository': '',
188             'revision': ''
189         },
190         'submittedAt': buildTime || 1458704983
191     };
192 }
193
194 function sampleInProgressBuild(slaveName)
195 {
196     return {
197         'blame': [],
198         'builderName': 'ABTest-iPad-RunBenchmark-Tests',
199         'currentStep': {
200             'eta': 0.26548067698460565,
201             'expectations': [['output', 845, 1315.0]],
202             'hidden': false,
203             'isFinished': false,
204             'isStarted': true,
205             'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/Some%20step/logs/stdio']],
206             'name': 'Some step',
207             'results': [null,[]],
208             'statistics': {},
209             'step_number': 1,
210             'text': [''],
211             'times': [1458718657.581628, null],
212             'urls': {}
213         },
214         'eta': 6497.991612434387,
215         'logs': [['stdio','https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/shell/logs/stdio']],
216         'number': 614,
217         'properties': [
218             ['build_request_id', '16733', 'Force Build Form'],
219             ['buildername', 'ABTest-iPad-RunBenchmark-Tests', 'Builder'],
220             ['buildnumber', 614, 'Build'],
221             ['desired_image', '13A452', 'Force Build Form'],
222             ['owner', '<unknown>', 'Force Build Form'],
223             ['reason', 'force build', 'Force Build Form'],
224             ['roots_dict', JSON.stringify(sampleRootSetData), 'Force Build Form'],
225             ['scheduler', 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler', 'Scheduler'],
226             ['slavename', slaveName || 'ABTest-iPad-0', 'BuildSlave'],
227         ],
228         'reason': 'A build was forced by \'<unknown>\': force build',
229         'results': null,
230         'slave': 'ABTest-iPad-0',
231         'sourceStamps': [{'branch': '', 'changes': [], 'codebase': 'compiler-rt', 'hasPatch': false, 'project': '', 'repository': '', 'revision': ''}],
232         'steps': [
233             {
234                 'eta': null,
235                 'expectations': [['output',2309,2309.0]],
236                 'hidden': false,
237                 'isFinished': true,
238                 'isStarted': true,
239                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/shell/logs/stdio']],
240                 'name': 'Finished step',
241                 'results': [0, []],
242                 'statistics': {},
243                 'step_number': 0,
244                 'text': [''],
245                 'times': [1458718655.419865, 1458718655.453633],
246                 'urls': {}
247             },
248             {
249                 'eta': 0.26548067698460565,
250                 'expectations': [['output', 845, 1315.0]],
251                 'hidden': false,
252                 'isFinished': false,
253                 'isStarted': true,
254                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/Some%20step/logs/stdio']],
255                 'name': 'Some step',
256                 'results': [null,[]],
257                 'statistics': {},
258                 'step_number': 1,
259                 'text': [''],
260                 'times': [1458718657.581628, null],
261                 'urls': {}
262             },
263             {
264                 'eta': null,
265                 'expectations': [['output', null, null]],
266                 'hidden': false,
267                 'isFinished': false,
268                 'isStarted': false,
269                 'logs': [],
270                 'name': 'Some other step',
271                 'results': [null, []],
272                 'statistics': {},
273                 'step_number': 2,
274                 'text': [],
275                 'times': [null, null],
276                 'urls': {}
277             },
278         ],
279         'text': [],
280         'times': [1458718655.415821, null]
281     };
282 }
283
284 function sampleFinishedBuild(buildRequestId, slaveName)
285 {
286     return {
287         'blame': [],
288         'builderName': 'ABTest-iPad-RunBenchmark-Tests',
289         'currentStep': null,
290         'eta': null,
291         'logs': [['stdio','https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755/steps/shell/logs/stdio']],
292         'number': 1755,
293         'properties': [
294             ['build_request_id', buildRequestId || '18935', 'Force Build Form'],
295             ['buildername', 'ABTest-iPad-RunBenchmark-Tests', 'Builder'],
296             ['buildnumber', 1755, 'Build'],
297             ['desired_image', '13A452', 'Force Build Form'],
298             ['owner', '<unknown>', 'Force Build Form'],
299             ['reason', 'force build', 'Force Build Form'],
300             ['roots_dict', JSON.stringify(sampleRootSetData), 'Force Build Form'],
301             ['scheduler', 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler', 'Scheduler'],
302             ['slavename', slaveName || 'ABTest-iPad-0', 'BuildSlave'],
303         ],
304         'reason': 'A build was forced by \'<unknown>\': force build',
305         'results': 2,
306         'slave': 'ABTest-iPad-0',
307         'sourceStamps': [{'branch': '', 'changes': [], 'codebase': 'compiler-rt', 'hasPatch': false, 'project': '', 'repository': '', 'revision': ''}],
308         'steps': [
309             {
310                 'eta': null,
311                 'expectations': [['output',2309,2309.0]],
312                 'hidden': false,
313                 'isFinished': true,
314                 'isStarted': true,
315                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/shell/logs/stdio']],
316                 'name': 'Finished step',
317                 'results': [0, []],
318                 'statistics': {},
319                 'step_number': 0,
320                 'text': [''],
321                 'times': [1458718655.419865, 1458718655.453633],
322                 'urls': {}
323             },
324             {
325                 'eta': null,
326                 'expectations': [['output', 845, 1315.0]],
327                 'hidden': false,
328                 'isFinished': true,
329                 'isStarted': true,
330                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/Some%20step/logs/stdio']],
331                 'name': 'Some step',
332                 'results': [null,[]],
333                 'statistics': {},
334                 'step_number': 1,
335                 'text': [''],
336                 'times': [1458718657.581628, null],
337                 'urls': {}
338             },
339             {
340                 'eta': null,
341                 'expectations': [['output', null, null]],
342                 'hidden': false,
343                 'isFinished': true,
344                 'isStarted': true,
345                 'logs': [],
346                 'name': 'Some other step',
347                 'results': [null, []],
348                 'statistics': {},
349                 'step_number': 2,
350                 'text': [],
351                 'times': [null, null],
352                 'urls': {}
353             },
354         ],
355         'text': [],
356         'times': [1458937478.25837, 1458946147.173785]
357     };
358 }
359
360 describe('BuildbotSyncer', function () {
361     MockModels.inject();
362     let requests = MockRemoteAPI.inject('http://build.webkit.org');
363
364     describe('_loadConfig', function () {
365
366         it('should create BuildbotSyncer objects for a configuration that specify all required options', function () {
367             let syncers = BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [smallConfiguration()]});
368             assert.equal(syncers.length, 1);
369         });
370
371         it('should throw when some required options are missing', function () {
372             assert.throws(function () {
373                 let config = smallConfiguration();
374                 delete config['builder'];
375                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
376             }, 'builder should be a required option');
377             assert.throws(function () {
378                 let config = smallConfiguration();
379                 delete config['platform'];
380                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
381             }, 'platform should be a required option');
382             assert.throws(function () {
383                 let config = smallConfiguration();
384                 delete config['test'];
385                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
386             }, 'test should be a required option');
387             assert.throws(function () {
388                 let config = smallConfiguration();
389                 delete config['arguments'];
390                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
391             });
392             assert.throws(function () {
393                 let config = smallConfiguration();
394                 delete config['buildRequestArgument'];
395                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
396             });
397         });
398
399         it('should throw when a test name is not an array of strings', function () {
400             assert.throws(function () {
401                 let config = smallConfiguration();
402                 config.test = 'some test';
403                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
404             });
405             assert.throws(function () {
406                 let config = smallConfiguration();
407                 config.test = [1];
408                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
409             });
410         });
411
412         it('should throw when arguments is not an object', function () {
413             assert.throws(function () {
414                 let config = smallConfiguration();
415                 config.arguments = 'hello';
416                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
417             });
418         });
419
420         it('should throw when arguments\'s values are malformed', function () {
421             assert.throws(function () {
422                 let config = smallConfiguration();
423                 config.arguments = {'some': {'root': 'some root', 'rootsExcluding': ['other root']}};
424                 BuildbotSyncer._loadConfig(RemoteAPI, {'configurations': [config]});
425             });
426             assert.throws(function () {
427                 let config = smallConfiguration();
428                 config.arguments = {'some': {'otherKey': 'some root'}};
429                 BuildbotSyncer._loadConfig(RemoteAPI, {'configurations': [config]});
430             });
431             assert.throws(function () {
432                 let config = smallConfiguration();
433                 config.arguments = {'some': {'root': ['a', 'b']}};
434                 BuildbotSyncer._loadConfig(RemoteAPI, {'configurations': [config]});
435             });
436             assert.throws(function () {
437                 let config = smallConfiguration();
438                 config.arguments = {'some': {'root': 1}};
439                 BuildbotSyncer._loadConfig(RemoteAPI, {'configurations': [config]});
440             });
441             assert.throws(function () {
442                 let config = smallConfiguration();
443                 config.arguments = {'some': {'rootsExcluding': 'a'}};
444                 BuildbotSyncer._loadConfig(RemoteAPI, {'configurations': [config]});
445             });
446             assert.throws(function () {
447                 let config = smallConfiguration();
448                 config.arguments = {'some': {'rootsExcluding': [1]}};
449                 BuildbotSyncer._loadConfig(RemoteAPI, {'configurations': [config]});
450             });
451         });
452
453         it('should create BuildbotSyncer objects for valid configurations', function () {
454             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
455             assert.equal(syncers.length, 2);
456             assert.ok(syncers[0] instanceof BuildbotSyncer);
457             assert.ok(syncers[1] instanceof BuildbotSyncer);
458         });
459
460         it('should parse builder names correctly', function () {
461             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
462             assert.equal(syncers[0].builderName(), 'ABTest-iPhone-RunBenchmark-Tests');
463             assert.equal(syncers[1].builderName(), 'ABTest-iPad-RunBenchmark-Tests');
464         });
465
466         it('should parse test configurations correctly', function () {
467             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
468
469             let configurations = syncers[0].testConfigurations();
470             assert.equal(configurations.length, 3);
471             assert.equal(configurations[0].platform, MockModels.iphone);
472             assert.equal(configurations[0].test, MockModels.speedometer);
473             assert.equal(configurations[1].platform, MockModels.iphone);
474             assert.equal(configurations[1].test, MockModels.jetstream);
475             assert.equal(configurations[2].platform, MockModels.iphone);
476             assert.equal(configurations[2].test, MockModels.domcore);
477
478             configurations = syncers[1].testConfigurations();
479             assert.equal(configurations.length, 2);
480             assert.equal(configurations[0].platform, MockModels.ipad);
481             assert.equal(configurations[0].test, MockModels.speedometer);
482             assert.equal(configurations[1].platform, MockModels.ipad);
483             assert.equal(configurations[1].test, MockModels.jetstream);
484         });
485     });
486
487     describe('_propertiesForBuildRequest', function () {
488         it('should include all properties specified in a given configuration', function () {
489             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
490             let properties = syncers[0]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
491             assert.deepEqual(Object.keys(properties), ['desired_image', 'roots_dict', 'test_name', 'forcescheduler', 'build_request_id']);
492         });
493
494         it('should preserve non-parametric property values', function () {
495             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
496             let properties = syncers[0]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
497             assert.equal(properties['test_name'], 'speedometer');
498             assert.equal(properties['forcescheduler'], 'ABTest-iPhone-RunBenchmark-Tests-ForceScheduler');
499
500             properties = syncers[1]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.ipad, MockModels.jetstream));
501             assert.equal(properties['test_name'], 'jetstream');
502             assert.equal(properties['forcescheduler'], 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler');
503         });
504
505         it('should resolve "root"', function () {
506             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
507             let properties = syncers[0]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
508             assert.equal(properties['desired_image'], '13A452');
509         });
510
511         it('should resolve "rootsExcluding"', function () {
512             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
513             let properties = syncers[0]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
514             assert.equal(properties['roots_dict'], JSON.stringify(sampleRootSetData));
515         });
516
517         it('should set the property for the build request id', function () {
518             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
519             let request = createSampleBuildRequest(MockModels.iphone, MockModels.speedometer);
520             let properties = syncers[0]._propertiesForBuildRequest(request);
521             assert.equal(properties['build_request_id'], request.id());
522         });
523     });
524
525     describe('pullBuildbot', function () {
526         it('should fetch pending builds from the right URL', function () {
527             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
528             assert.equal(syncer.builderName(), 'ABTest-iPad-RunBenchmark-Tests');
529             let expectedURL = '/json/builders/ABTest-iPad-RunBenchmark-Tests/pendingBuilds';
530             assert.equal(syncer.pathForPendingBuildsJSON(), expectedURL);
531             syncer.pullBuildbot();
532             assert.equal(requests.length, 1);
533             assert.equal(requests[0].url, expectedURL);
534         });
535
536         it('should fetch recent builds once pending builds have been fetched', function (done) {
537             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
538             assert.equal(syncer.builderName(), 'ABTest-iPad-RunBenchmark-Tests');
539
540             syncer.pullBuildbot(1);
541             assert.equal(requests.length, 1);
542             assert.equal(requests[0].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/pendingBuilds');
543             requests[0].resolve([]);
544             Promise.resolve().then(function () {
545                 assert.equal(requests.length, 2);
546                 assert.equal(requests[1].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/builds/?select=-1');
547                 done();
548             }).catch(done);
549         });
550
551         it('should fetch the right number of recent builds', function (done) {
552             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
553
554             syncer.pullBuildbot(3);
555             assert.equal(requests.length, 1);
556             assert.equal(requests[0].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/pendingBuilds');
557             requests[0].resolve([]);
558             Promise.resolve().then(function () {
559                 assert.equal(requests.length, 2);
560                 assert.equal(requests[1].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/builds/?select=-1&select=-2&select=-3');
561                 done();
562             }).catch(done);
563         });
564
565         it('should create BuildbotBuildEntry for pending builds', function (done) {
566             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
567             let promise = syncer.pullBuildbot();
568             requests[0].resolve([samplePendingBuild()]);
569             promise.then(function (entries) {
570                 assert.equal(entries.length, 1);
571                 let entry = entries[0];
572                 assert.ok(entry instanceof BuildbotBuildEntry);
573                 assert.ok(!entry.buildNumber());
574                 assert.ok(!entry.slaveName());
575                 assert.equal(entry.buildRequestId(), 16733);
576                 assert.ok(entry.isPending());
577                 assert.ok(!entry.isInProgress());
578                 assert.ok(!entry.hasFinished());
579                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
580                 done();
581             }).catch(done);
582         });
583
584         it('should create BuildbotBuildEntry for in-progress builds', function (done) {
585             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
586
587             let promise = syncer.pullBuildbot(1);
588             assert.equal(requests.length, 1);
589             requests[0].resolve([]);
590             Promise.resolve().then(function () {
591                 assert.equal(requests.length, 2);
592                 requests[1].resolve({[-1]: sampleInProgressBuild()});
593             }).catch(done);
594
595             promise.then(function (entries) {
596                 assert.equal(entries.length, 1);
597                 let entry = entries[0];
598                 assert.ok(entry instanceof BuildbotBuildEntry);
599                 assert.equal(entry.buildNumber(), 614);
600                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
601                 assert.equal(entry.buildRequestId(), 16733);
602                 assert.ok(!entry.isPending());
603                 assert.ok(entry.isInProgress());
604                 assert.ok(!entry.hasFinished());
605                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
606                 done();
607             }).catch(done);
608         });
609
610         it('should create BuildbotBuildEntry for finished builds', function (done) {
611             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
612
613             let promise = syncer.pullBuildbot(1);
614             assert.equal(requests.length, 1);
615             requests[0].resolve([]);
616             Promise.resolve().then(function () {
617                 assert.equal(requests.length, 2);
618                 requests[1].resolve({[-1]: sampleFinishedBuild()});
619             }).catch(done);
620
621             promise.then(function (entries) {
622                 assert.deepEqual(entries.length, 1);
623                 let entry = entries[0];
624                 assert.ok(entry instanceof BuildbotBuildEntry);
625                 assert.equal(entry.buildNumber(), 1755);
626                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
627                 assert.equal(entry.buildRequestId(), 18935);
628                 assert.ok(!entry.isPending());
629                 assert.ok(!entry.isInProgress());
630                 assert.ok(entry.hasFinished());
631                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
632                 done();
633             }).catch(done);
634         });
635
636         it('should create BuildbotBuildEntry for mixed pending, in-progress, finished, and missing builds', function (done) {
637             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
638
639             let promise = syncer.pullBuildbot(5);
640             assert.equal(requests.length, 1);
641
642             requests[0].resolve([samplePendingBuild(123)]);
643
644             Promise.resolve().then(function () {
645                 assert.equal(requests.length, 2);
646                 requests[1].resolve({[-1]: sampleFinishedBuild(), [-2]: {'error': 'Not available'}, [-4]: sampleInProgressBuild()});
647             }).catch(done);
648
649             promise.then(function (entries) {
650                 assert.deepEqual(entries.length, 3);
651
652                 let entry = entries[0];
653                 assert.ok(entry instanceof BuildbotBuildEntry);
654                 assert.equal(entry.buildNumber(), null);
655                 assert.equal(entry.slaveName(), null);
656                 assert.equal(entry.buildRequestId(), 123);
657                 assert.ok(entry.isPending());
658                 assert.ok(!entry.isInProgress());
659                 assert.ok(!entry.hasFinished());
660                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
661
662                 entry = entries[1];
663                 assert.ok(entry instanceof BuildbotBuildEntry);
664                 assert.equal(entry.buildNumber(), 614);
665                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
666                 assert.equal(entry.buildRequestId(), 16733);
667                 assert.ok(!entry.isPending());
668                 assert.ok(entry.isInProgress());
669                 assert.ok(!entry.hasFinished());
670                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
671
672                 entry = entries[2];
673                 assert.ok(entry instanceof BuildbotBuildEntry);
674                 assert.equal(entry.buildNumber(), 1755);
675                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
676                 assert.equal(entry.buildRequestId(), 18935);
677                 assert.ok(!entry.isPending());
678                 assert.ok(!entry.isInProgress());
679                 assert.ok(entry.hasFinished());
680                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
681
682                 done();
683             }).catch(done);
684         });
685
686         it('should sort BuildbotBuildEntry by order', function (done) {
687             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
688
689             let promise = syncer.pullBuildbot(5);
690             assert.equal(requests.length, 1);
691
692             requests[0].resolve([samplePendingBuild(456, 2), samplePendingBuild(123, 1)]);
693
694             Promise.resolve().then(function () {
695                 assert.equal(requests.length, 2);
696                 requests[1].resolve({[-3]: sampleFinishedBuild(), [-1]: {'error': 'Not available'}, [-2]: sampleInProgressBuild()});
697             }).catch(done);
698
699             promise.then(function (entries) {
700                 assert.deepEqual(entries.length, 4);
701
702                 let entry = entries[0];
703                 assert.ok(entry instanceof BuildbotBuildEntry);
704                 assert.equal(entry.buildNumber(), null);
705                 assert.equal(entry.slaveName(), null);
706                 assert.equal(entry.buildRequestId(), 123);
707                 assert.ok(entry.isPending());
708                 assert.ok(!entry.isInProgress());
709                 assert.ok(!entry.hasFinished());
710                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
711
712                 entry = entries[1];
713                 assert.ok(entry instanceof BuildbotBuildEntry);
714                 assert.equal(entry.buildNumber(), null);
715                 assert.equal(entry.slaveName(), null);
716                 assert.equal(entry.buildRequestId(), 456);
717                 assert.ok(entry.isPending());
718                 assert.ok(!entry.isInProgress());
719                 assert.ok(!entry.hasFinished());
720                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
721
722                 entry = entries[2];
723                 assert.ok(entry instanceof BuildbotBuildEntry);
724                 assert.equal(entry.buildNumber(), 614);
725                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
726                 assert.equal(entry.buildRequestId(), 16733);
727                 assert.ok(!entry.isPending());
728                 assert.ok(entry.isInProgress());
729                 assert.ok(!entry.hasFinished());
730                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
731
732                 entry = entries[3];
733                 assert.ok(entry instanceof BuildbotBuildEntry);
734                 assert.equal(entry.buildNumber(), 1755);
735                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
736                 assert.equal(entry.buildRequestId(), 18935);
737                 assert.ok(!entry.isPending());
738                 assert.ok(!entry.isInProgress());
739                 assert.ok(entry.hasFinished());
740                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
741
742                 done();
743             }).catch(done);
744         });
745
746         it('should override BuildbotBuildEntry for pending builds by in-progress builds', function (done) {
747             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
748
749             let promise = syncer.pullBuildbot(5);
750             assert.equal(requests.length, 1);
751
752             requests[0].resolve([samplePendingBuild()]);
753
754             Promise.resolve().then(function () {
755                 assert.equal(requests.length, 2);
756                 requests[1].resolve({[-1]: sampleInProgressBuild()});
757             }).catch(done);
758
759             promise.then(function (entries) {
760                 assert.equal(entries.length, 1);
761
762                 let entry = entries[0];
763                 assert.ok(entry instanceof BuildbotBuildEntry);
764                 assert.equal(entry.buildNumber(), 614);
765                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
766                 assert.equal(entry.buildRequestId(), 16733);
767                 assert.ok(!entry.isPending());
768                 assert.ok(entry.isInProgress());
769                 assert.ok(!entry.hasFinished());
770                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
771
772                 done();
773             }).catch(done);
774         });
775
776         it('should override BuildbotBuildEntry for pending builds by finished builds', function (done) {
777             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
778
779             let promise = syncer.pullBuildbot(5);
780             assert.equal(requests.length, 1);
781
782             requests[0].resolve([samplePendingBuild()]);
783
784             Promise.resolve().then(function () {
785                 assert.equal(requests.length, 2);
786                 requests[1].resolve({[-1]: sampleFinishedBuild(16733)});
787             }).catch(done);
788
789             promise.then(function (entries) {
790                 assert.equal(entries.length, 1);
791
792                 let entry = entries[0];
793                 assert.ok(entry instanceof BuildbotBuildEntry);
794                 assert.equal(entry.buildNumber(), 1755);
795                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
796                 assert.equal(entry.buildRequestId(), 16733);
797                 assert.ok(!entry.isPending());
798                 assert.ok(!entry.isInProgress());
799                 assert.ok(entry.hasFinished());
800                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
801
802                 done();
803             }).catch(done);
804         });
805     });
806
807     describe('scheduleRequest', function () {
808         it('should schedule a build request on a specified slave', function (done) {
809             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[0];
810
811             syncer.scheduleRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer), 'some-slave');
812             Promise.resolve().then(function () {
813                 assert.equal(requests.length, 1);
814                 assert.equal(requests[0].url, '/builders/ABTest-iPhone-RunBenchmark-Tests/force');
815                 assert.equal(requests[0].method, 'POST');
816                 assert.deepEqual(requests[0].data, {
817                     'build_request_id': '16733-' + MockModels.iphone.id(),
818                     'desired_image': '13A452',
819                     'forcescheduler': 'ABTest-iPhone-RunBenchmark-Tests-ForceScheduler',
820                     'roots_dict': '{"WebKit":{"id":"111127","time":1456955807334,"repository":"WebKit","revision":"197463"},'
821                         + '"Shared":{"id":"111237","time":1456931874000,"repository":"Shared","revision":"80229"}}',
822                     'slavename': 'some-slave',
823                     'test_name': 'speedometer'
824                 });
825                 done();
826             }).catch(done);
827         });
828     });
829
830     describe('scheduleFirstRequestInGroupIfAvailable', function () {
831
832         function pullBuildbotWithAssertion(syncer, pendingBuilds, inProgressAndFinishedBuilds)
833         {
834             let promise = syncer.pullBuildbot(5);
835             assert.equal(requests.length, 1);
836             requests[0].resolve(pendingBuilds);
837             return Promise.resolve().then(function () {
838                 assert.equal(requests.length, 2);
839                 requests[1].resolve(inProgressAndFinishedBuilds);
840                 requests.length = 0;
841             }).then(function () {
842                 return promise;
843             });
844         }
845
846         it('should schedule a build if builder has no builds if slaveList is not specified', function (done) {
847             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [smallConfiguration()]})[0];
848
849             pullBuildbotWithAssertion(syncer, [], {}).then(function () {
850                 syncer.scheduleFirstRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
851             }).then(function () {
852                 assert.equal(requests.length, 1);
853                 assert.equal(requests[0].url, '/builders/some builder/force');
854                 assert.equal(requests[0].method, 'POST');
855                 assert.deepEqual(requests[0].data, {id: '16733-' + MockModels.somePlatform.id()});
856                 done();
857             }).catch(done);
858         });
859
860         it('should schedule a build if builder only has finished builds if slaveList is not specified', function (done) {
861             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [smallConfiguration()]})[0];
862
863             pullBuildbotWithAssertion(syncer, [], {[-1]: smallFinishedBuild()}).then(function () {
864                 syncer.scheduleFirstRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
865             }).then(function () {
866                 assert.equal(requests.length, 1);
867                 assert.equal(requests[0].url, '/builders/some builder/force');
868                 assert.equal(requests[0].method, 'POST');
869                 assert.deepEqual(requests[0].data, {id: '16733-' + MockModels.somePlatform.id()});
870                 done();
871             }).catch(done);
872         });
873
874         it('should not schedule a build if builder has a pending build if slaveList is not specified', function (done) {
875             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [smallConfiguration()]})[0];
876
877             pullBuildbotWithAssertion(syncer, [smallPendingBuild()], {}).then(function () {
878                 syncer.scheduleFirstRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
879             }).then(function () {
880                 assert.equal(requests.length, 0);
881                 done();
882             }).catch(done);
883         });
884
885         it('should schedule a build if builder does not have pending or completed builds on the matching slave', function (done) {
886             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[0];
887
888             pullBuildbotWithAssertion(syncer, [], {}).then(function () {
889                 syncer.scheduleFirstRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
890             }).then(function () {
891                 assert.equal(requests.length, 1);
892                 assert.equal(requests[0].url, '/builders/ABTest-iPhone-RunBenchmark-Tests/force');
893                 assert.equal(requests[0].method, 'POST');
894                 done();
895             }).catch(done);
896         });
897
898         it('should schedule a build if builder only has finished builds on the matching slave', function (done) {
899             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
900
901             pullBuildbotWithAssertion(syncer, [], {[-1]: sampleFinishedBuild()}).then(function () {
902                 syncer.scheduleFirstRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
903             }).then(function () {
904                 assert.equal(requests.length, 1);
905                 assert.equal(requests[0].url, '/builders/ABTest-iPad-RunBenchmark-Tests/force');
906                 assert.equal(requests[0].method, 'POST');
907                 done();
908             }).catch(done);
909         });
910
911         it('should not schedule a build if builder has a pending build on the maching slave', function (done) {
912             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
913
914             pullBuildbotWithAssertion(syncer, [samplePendingBuild()], {}).then(function () {
915                 syncer.scheduleFirstRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
916             }).then(function () {
917                 assert.equal(requests.length, 0);
918                 done();
919             }).catch(done);
920         });
921
922         it('should schedule a build if builder only has a pending build on a non-maching slave', function (done) {
923             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
924
925             pullBuildbotWithAssertion(syncer, [samplePendingBuild(1, 1, 'another-slave')], {}).then(function () {
926                 syncer.scheduleFirstRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
927             }).then(function () {
928                 assert.equal(requests.length, 1);
929                 done();
930             }).catch(done);
931         });
932
933         it('should schedule a build if builder only has an in-progress build on the matching slave', function (done) {
934             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
935
936             pullBuildbotWithAssertion(syncer, [], {[-1]: sampleInProgressBuild()}).then(function () {
937                 syncer.scheduleFirstRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
938             }).then(function () {
939                 assert.equal(requests.length, 1);
940                 done();
941             }).catch(done);
942         });
943
944         it('should schedule a build if builder has an in-progress build on another slave', function (done) {
945             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
946
947             pullBuildbotWithAssertion(syncer, [], {[-1]: sampleInProgressBuild('other-slave')}).then(function () {
948                 syncer.scheduleFirstRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
949             }).then(function () {
950                 assert.equal(requests.length, 1);
951                 done();
952             }).catch(done);
953         });
954
955         it('should not schedule a build if the request does not match any configuration', function (done) {
956             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[0];
957
958             pullBuildbotWithAssertion(syncer, [], {}).then(function () {
959                 syncer.scheduleFirstRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
960             }).then(function () {
961                 assert.equal(requests.length, 0);
962                 done();
963             }).catch(done);
964         });
965
966         it('should not schedule a build if a new request had been submitted to the same slave', function (done) {
967             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
968
969             pullBuildbotWithAssertion(syncer, [], {}).then(function () {
970                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-0');
971                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-1');
972             }).then(function () {
973                 assert.equal(requests.length, 2);
974                 syncer.scheduleFirstRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
975             }).then(function () {
976                 assert.equal(requests.length, 2);
977                 done();
978             }).catch(done);
979         });
980
981         it('should schedule a build if a new request had been submitted to another slave', function (done) {
982             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
983
984             pullBuildbotWithAssertion(syncer, [], {}).then(function () {
985                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-0');
986             }).then(function () {
987                 assert.equal(requests.length, 1);
988                 syncer.scheduleFirstRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-1');
989             }).then(function () {
990                 assert.equal(requests.length, 2);
991                 done();
992             }).catch(done);
993         });
994
995         it('should not schedule a build if a new request had been submitted to the same builder without slaveList', function (done) {
996             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [smallConfiguration()]})[0];
997
998             pullBuildbotWithAssertion(syncer, [], {}).then(function () {
999                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest), null);
1000             }).then(function () {
1001                 assert.equal(requests.length, 1);
1002                 syncer.scheduleFirstRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
1003             }).then(function () {
1004                 assert.equal(requests.length, 1);
1005                 done();
1006             }).catch(done);
1007         });
1008     });
1009 });