'buildbot-syncer.js' should be able to determine force build argument from a list...
[WebKit-https.git] / Websites / perf.webkit.org / unit-tests / buildbot-syncer-tests.js
1 'use strict';
2
3 let assert = require('assert');
4
5 require('../tools/js/v3-models.js');
6 let MockRemoteAPI = require('./resources/mock-remote-api.js').MockRemoteAPI;
7 let MockModels = require('./resources/mock-v3-models.js').MockModels;
8
9 let BuildbotBuildEntry = require('../tools/js/buildbot-syncer.js').BuildbotBuildEntry;
10 let BuildbotSyncer = require('../tools/js/buildbot-syncer.js').BuildbotSyncer;
11
12 function sampleiOSConfig()
13 {
14     return {
15         'shared':
16             {
17                 'arguments': {
18                     'desired_image': {'root': 'iOS'},
19                     'opensource': {'rootOptions': ['WebKit-SVN', 'WebKit-Git']},
20                     'roots_dict': {'rootsExcluding': ['iOS']}
21                 },
22                 'slaveArgument': 'slavename',
23                 'buildRequestArgument': 'build_request_id'
24             },
25         'types': {
26             'speedometer': {
27                 'test': ['Speedometer'],
28                 'arguments': {'test_name': 'speedometer'}
29             },
30             'jetstream': {
31                 'test': ['JetStream'],
32                 'arguments': {'test_name': 'jetstream'}
33             },
34             'dromaeo-dom': {
35                 'test': ['Dromaeo', 'DOM Core Tests'],
36                 'arguments': {'tests': 'dromaeo-dom'}
37             },
38         },
39         'builders': {
40             'iPhone-bench': {
41                 'builder': 'ABTest-iPhone-RunBenchmark-Tests',
42                 'arguments': { 'forcescheduler': 'ABTest-iPhone-RunBenchmark-Tests-ForceScheduler' },
43                 'slaveList': ['ABTest-iPhone-0'],
44             },
45             'iPad-bench': {
46                 'builder': 'ABTest-iPad-RunBenchmark-Tests',
47                 'arguments': { 'forcescheduler': 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler' },
48                 'slaveList': ['ABTest-iPad-0', 'ABTest-iPad-1'],
49             }
50         },
51         'configurations': [
52             {'type': 'speedometer', 'builder': 'iPhone-bench', 'platform': 'iPhone'},
53             {'type': 'jetstream', 'builder': 'iPhone-bench', 'platform': 'iPhone'},
54             {'type': 'dromaeo-dom', 'builder': 'iPhone-bench', 'platform': 'iPhone'},
55
56             {'type': 'speedometer', 'builder': 'iPad-bench', 'platform': 'iPad'},
57             {'type': 'jetstream', 'builder': 'iPad-bench', 'platform': 'iPad'},
58         ]
59     };
60 }
61
62 function sampleiOSConfigWithExpansions()
63 {
64     return {
65         "triggerableName": "build-webkit-ios",
66         "shared":
67             {
68                 "arguments": {
69                     "webkit-revision": {"root": "WebKit"},
70                     "os-version": {"root": "iOS"}
71                 },
72                 "buildRequestArgument": "build-request-id"
73             },
74         "types": {
75             "iphone-plt": {
76                 "test": ["PLT-iPhone"],
77                 "arguments": {"test_name": "plt"}
78             },
79             "ipad-plt": {
80                 "test": ["PLT-iPad"],
81                 "arguments": {"test_name": "plt"}
82             },
83             "speedometer": {
84                 "test": ["Speedometer"],
85                 "arguments": {"tests": "speedometer"}
86             },
87         },
88         "builders": {
89             "iphone": {
90                 "builder": "iPhone AB Tests",
91                 "arguments": {"forcescheduler": "force-iphone-ab-tests"}
92             },
93             "ipad": {
94                 "builder": "iPad AB Tests",
95                 "arguments": {"forcescheduler": "force-ipad-ab-tests"}
96             },
97         },
98         "configurations": [
99             {
100                 "builder": "iphone",
101                 "platforms": ["iPhone", "iOS 10 iPhone"],
102                 "types": ["iphone-plt", "speedometer"],
103             },
104             {
105                 "builder": "ipad",
106                 "platforms": ["iPad"],
107                 "types": ["ipad-plt", "speedometer"],
108             },
109         ]
110     }
111 }
112
113 let sampleRootSetData = {
114     'WebKit': {
115         'id': '111127',
116         'time': 1456955807334,
117         'repository': 'WebKit',
118         'revision': '197463',
119     },
120     'Shared': {
121         'id': '111237',
122         'time': 1456931874000,
123         'repository': 'Shared',
124         'revision': '80229',
125     },
126     'WebKit-Git': {
127         "id":"111239",
128         "time":1456931874000,
129         "repository":"WebKit-Git",
130         "revision":"9abcdef",
131     },
132 };
133
134 function smallConfiguration()
135 {
136     return {
137         'builder': 'some builder',
138         'platform': 'Some platform',
139         'test': ['Some test'],
140         'arguments': {},
141         'buildRequestArgument': 'id'};
142 }
143
144 function smallPendingBuild()
145 {
146     return {
147         'builderName': 'some builder',
148         'builds': [],
149         'properties': [],
150         'source': {
151             'branch': '',
152             'changes': [],
153             'codebase': 'WebKit',
154             'hasPatch': false,
155             'project': '',
156             'repository': '',
157             'revision': ''
158         },
159     };
160 }
161
162 function smallInProgressBuild()
163 {
164     return {
165         'builderName': 'some builder',
166         'builds': [],
167         'properties': [],
168         'currentStep': { },
169         'eta': 123,
170         'number': 456,
171         'source': {
172             'branch': '',
173             'changes': [],
174             'codebase': 'WebKit',
175             'hasPatch': false,
176             'project': '',
177             'repository': '',
178             'revision': ''
179         },
180     };
181 }
182
183 function smallFinishedBuild()
184 {
185     return {
186         'builderName': 'some builder',
187         'builds': [],
188         'properties': [],
189         'currentStep': null,
190         'eta': null,
191         'number': 789,
192         'source': {
193             'branch': '',
194             'changes': [],
195             'codebase': 'WebKit',
196             'hasPatch': false,
197             'project': '',
198             'repository': '',
199             'revision': ''
200         },
201         'times': [0, 1],
202     };
203 }
204
205 function createSampleBuildRequest(platform, test)
206 {
207     assert(platform instanceof Platform);
208     assert(test instanceof Test);
209
210     let rootSet = RootSet.ensureSingleton('4197', {roots: [
211         {'id': '111127', 'time': 1456955807334, 'repository': MockModels.webkit, 'revision': '197463'},
212         {'id': '111237', 'time': 1456931874000, 'repository': MockModels.sharedRepository, 'revision': '80229'},
213         {'id': '111239', 'time': 1456931874000, 'repository': MockModels.webkitGit, 'revision': '9abcdef'},
214         {'id': '88930', 'time': 0, 'repository': MockModels.ios, 'revision': '13A452'},
215     ]});
216
217     let request = BuildRequest.ensureSingleton('16733-' + platform.id(), {'rootSet': rootSet, 'status': 'pending', 'platform': platform, 'test': test});
218     return request;
219 }
220
221 function samplePendingBuild(buildRequestId, buildTime, slaveName)
222 {
223     return {
224         'builderName': 'ABTest-iPad-RunBenchmark-Tests',
225         'builds': [],
226         'properties': [
227             ['build_request_id', buildRequestId || '16733', 'Force Build Form'],
228             ['desired_image', '13A452', 'Force Build Form'],
229             ['owner', '<unknown>', 'Force Build Form'],
230             ['test_name', 'speedometer', 'Force Build Form'],
231             ['reason', 'force build','Force Build Form'],
232             [
233                 'roots_dict',
234                 JSON.stringify(sampleRootSetData),
235                 'Force Build Form'
236             ],
237             ['slavename', slaveName, ''],
238             ['scheduler', 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler', 'Scheduler']
239         ],
240         'source': {
241             'branch': '',
242             'changes': [],
243             'codebase': 'compiler-rt',
244             'hasPatch': false,
245             'project': '',
246             'repository': '',
247             'revision': ''
248         },
249         'submittedAt': buildTime || 1458704983
250     };
251 }
252
253 function sampleInProgressBuild(slaveName)
254 {
255     return {
256         'blame': [],
257         'builderName': 'ABTest-iPad-RunBenchmark-Tests',
258         'currentStep': {
259             'eta': 0.26548067698460565,
260             'expectations': [['output', 845, 1315.0]],
261             'hidden': false,
262             'isFinished': false,
263             'isStarted': true,
264             'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/Some%20step/logs/stdio']],
265             'name': 'Some step',
266             'results': [null,[]],
267             'statistics': {},
268             'step_number': 1,
269             'text': [''],
270             'times': [1458718657.581628, null],
271             'urls': {}
272         },
273         'eta': 6497.991612434387,
274         'logs': [['stdio','https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/shell/logs/stdio']],
275         'number': 614,
276         'properties': [
277             ['build_request_id', '16733', 'Force Build Form'],
278             ['buildername', 'ABTest-iPad-RunBenchmark-Tests', 'Builder'],
279             ['buildnumber', 614, 'Build'],
280             ['desired_image', '13A452', 'Force Build Form'],
281             ['owner', '<unknown>', 'Force Build Form'],
282             ['reason', 'force build', 'Force Build Form'],
283             ['roots_dict', JSON.stringify(sampleRootSetData), 'Force Build Form'],
284             ['scheduler', 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler', 'Scheduler'],
285             ['slavename', slaveName || 'ABTest-iPad-0', 'BuildSlave'],
286         ],
287         'reason': 'A build was forced by \'<unknown>\': force build',
288         'results': null,
289         'slave': 'ABTest-iPad-0',
290         'sourceStamps': [{'branch': '', 'changes': [], 'codebase': 'compiler-rt', 'hasPatch': false, 'project': '', 'repository': '', 'revision': ''}],
291         'steps': [
292             {
293                 'eta': null,
294                 'expectations': [['output',2309,2309.0]],
295                 'hidden': false,
296                 'isFinished': true,
297                 'isStarted': true,
298                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/shell/logs/stdio']],
299                 'name': 'Finished step',
300                 'results': [0, []],
301                 'statistics': {},
302                 'step_number': 0,
303                 'text': [''],
304                 'times': [1458718655.419865, 1458718655.453633],
305                 'urls': {}
306             },
307             {
308                 'eta': 0.26548067698460565,
309                 'expectations': [['output', 845, 1315.0]],
310                 'hidden': false,
311                 'isFinished': false,
312                 'isStarted': true,
313                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/Some%20step/logs/stdio']],
314                 'name': 'Some step',
315                 'results': [null,[]],
316                 'statistics': {},
317                 'step_number': 1,
318                 'text': [''],
319                 'times': [1458718657.581628, null],
320                 'urls': {}
321             },
322             {
323                 'eta': null,
324                 'expectations': [['output', null, null]],
325                 'hidden': false,
326                 'isFinished': false,
327                 'isStarted': false,
328                 'logs': [],
329                 'name': 'Some other step',
330                 'results': [null, []],
331                 'statistics': {},
332                 'step_number': 2,
333                 'text': [],
334                 'times': [null, null],
335                 'urls': {}
336             },
337         ],
338         'text': [],
339         'times': [1458718655.415821, null]
340     };
341 }
342
343 function sampleFinishedBuild(buildRequestId, slaveName)
344 {
345     return {
346         'blame': [],
347         'builderName': 'ABTest-iPad-RunBenchmark-Tests',
348         'currentStep': null,
349         'eta': null,
350         'logs': [['stdio','https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755/steps/shell/logs/stdio']],
351         'number': 1755,
352         'properties': [
353             ['build_request_id', buildRequestId || '18935', 'Force Build Form'],
354             ['buildername', 'ABTest-iPad-RunBenchmark-Tests', 'Builder'],
355             ['buildnumber', 1755, 'Build'],
356             ['desired_image', '13A452', 'Force Build Form'],
357             ['owner', '<unknown>', 'Force Build Form'],
358             ['reason', 'force build', 'Force Build Form'],
359             ['roots_dict', JSON.stringify(sampleRootSetData), 'Force Build Form'],
360             ['scheduler', 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler', 'Scheduler'],
361             ['slavename', slaveName || 'ABTest-iPad-0', 'BuildSlave'],
362         ],
363         'reason': 'A build was forced by \'<unknown>\': force build',
364         'results': 2,
365         'slave': 'ABTest-iPad-0',
366         'sourceStamps': [{'branch': '', 'changes': [], 'codebase': 'compiler-rt', 'hasPatch': false, 'project': '', 'repository': '', 'revision': ''}],
367         'steps': [
368             {
369                 'eta': null,
370                 'expectations': [['output',2309,2309.0]],
371                 'hidden': false,
372                 'isFinished': true,
373                 'isStarted': true,
374                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/shell/logs/stdio']],
375                 'name': 'Finished step',
376                 'results': [0, []],
377                 'statistics': {},
378                 'step_number': 0,
379                 'text': [''],
380                 'times': [1458718655.419865, 1458718655.453633],
381                 'urls': {}
382             },
383             {
384                 'eta': null,
385                 'expectations': [['output', 845, 1315.0]],
386                 'hidden': false,
387                 'isFinished': true,
388                 'isStarted': true,
389                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/Some%20step/logs/stdio']],
390                 'name': 'Some step',
391                 'results': [null,[]],
392                 'statistics': {},
393                 'step_number': 1,
394                 'text': [''],
395                 'times': [1458718657.581628, null],
396                 'urls': {}
397             },
398             {
399                 'eta': null,
400                 'expectations': [['output', null, null]],
401                 'hidden': false,
402                 'isFinished': true,
403                 'isStarted': true,
404                 'logs': [],
405                 'name': 'Some other step',
406                 'results': [null, []],
407                 'statistics': {},
408                 'step_number': 2,
409                 'text': [],
410                 'times': [null, null],
411                 'urls': {}
412             },
413         ],
414         'text': [],
415         'times': [1458937478.25837, 1458946147.173785]
416     };
417 }
418
419 describe('BuildbotSyncer', function () {
420     MockModels.inject();
421     let requests = MockRemoteAPI.inject('http://build.webkit.org');
422
423     describe('_loadConfig', function () {
424
425         it('should create BuildbotSyncer objects for a configuration that specify all required options', function () {
426             let syncers = BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [smallConfiguration()]});
427             assert.equal(syncers.length, 1);
428         });
429
430         it('should throw when some required options are missing', function () {
431             assert.throws(function () {
432                 let config = smallConfiguration();
433                 delete config['builder'];
434                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
435             }, 'builder should be a required option');
436             assert.throws(function () {
437                 let config = smallConfiguration();
438                 delete config['platform'];
439                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
440             }, 'platform should be a required option');
441             assert.throws(function () {
442                 let config = smallConfiguration();
443                 delete config['test'];
444                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
445             }, 'test should be a required option');
446             assert.throws(function () {
447                 let config = smallConfiguration();
448                 delete config['arguments'];
449                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
450             });
451             assert.throws(function () {
452                 let config = smallConfiguration();
453                 delete config['buildRequestArgument'];
454                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
455             });
456         });
457
458         it('should throw when a test name is not an array of strings', function () {
459             assert.throws(function () {
460                 let config = smallConfiguration();
461                 config.test = 'some test';
462                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
463             });
464             assert.throws(function () {
465                 let config = smallConfiguration();
466                 config.test = [1];
467                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
468             });
469         });
470
471         it('should throw when arguments is not an object', function () {
472             assert.throws(function () {
473                 let config = smallConfiguration();
474                 config.arguments = 'hello';
475                 BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [config]});
476             });
477         });
478
479         it('should throw when arguments\'s values are malformed', function () {
480             assert.throws(function () {
481                 let config = smallConfiguration();
482                 config.arguments = {'some': {'root': 'some root', 'rootsExcluding': ['other root']}};
483                 BuildbotSyncer._loadConfig(RemoteAPI, {'configurations': [config]});
484             });
485             assert.throws(function () {
486                 let config = smallConfiguration();
487                 config.arguments = {'some': {'otherKey': 'some root'}};
488                 BuildbotSyncer._loadConfig(RemoteAPI, {'configurations': [config]});
489             });
490             assert.throws(function () {
491                 let config = smallConfiguration();
492                 config.arguments = {'some': {'root': ['a', 'b']}};
493                 BuildbotSyncer._loadConfig(RemoteAPI, {'configurations': [config]});
494             });
495             assert.throws(function () {
496                 let config = smallConfiguration();
497                 config.arguments = {'some': {'root': 1}};
498                 BuildbotSyncer._loadConfig(RemoteAPI, {'configurations': [config]});
499             });
500             assert.throws(function () {
501                 let config = smallConfiguration();
502                 config.arguments = {'some': {'rootsExcluding': 'a'}};
503                 BuildbotSyncer._loadConfig(RemoteAPI, {'configurations': [config]});
504             });
505             assert.throws(function () {
506                 let config = smallConfiguration();
507                 config.arguments = {'some': {'rootsExcluding': [1]}};
508                 BuildbotSyncer._loadConfig(RemoteAPI, {'configurations': [config]});
509             });
510         });
511
512         it('should create BuildbotSyncer objects for valid configurations', function () {
513             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
514             assert.equal(syncers.length, 2);
515             assert.ok(syncers[0] instanceof BuildbotSyncer);
516             assert.ok(syncers[1] instanceof BuildbotSyncer);
517         });
518
519         it('should parse builder names correctly', function () {
520             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
521             assert.equal(syncers[0].builderName(), 'ABTest-iPhone-RunBenchmark-Tests');
522             assert.equal(syncers[1].builderName(), 'ABTest-iPad-RunBenchmark-Tests');
523         });
524
525         it('should parse test configurations correctly', function () {
526             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
527
528             let configurations = syncers[0].testConfigurations();
529             assert.equal(configurations.length, 3);
530             assert.equal(configurations[0].platform, MockModels.iphone);
531             assert.equal(configurations[0].test, MockModels.speedometer);
532             assert.equal(configurations[1].platform, MockModels.iphone);
533             assert.equal(configurations[1].test, MockModels.jetstream);
534             assert.equal(configurations[2].platform, MockModels.iphone);
535             assert.equal(configurations[2].test, MockModels.domcore);
536
537             configurations = syncers[1].testConfigurations();
538             assert.equal(configurations.length, 2);
539             assert.equal(configurations[0].platform, MockModels.ipad);
540             assert.equal(configurations[0].test, MockModels.speedometer);
541             assert.equal(configurations[1].platform, MockModels.ipad);
542             assert.equal(configurations[1].test, MockModels.jetstream);
543         });
544
545         it('should parse test configurations with types and platforms expansions correctly', function () {
546             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfigWithExpansions());
547
548             assert.equal(syncers.length, 2);
549
550             let configurations = syncers[0].testConfigurations();
551             assert.equal(configurations.length, 4);
552             assert.equal(configurations[0].platform, MockModels.iphone);
553             assert.equal(configurations[0].test, MockModels.iPhonePLT);
554             assert.equal(configurations[1].platform, MockModels.iOS10iPhone);
555             assert.equal(configurations[1].test, MockModels.iPhonePLT);
556             assert.equal(configurations[2].platform, MockModels.iphone);
557             assert.equal(configurations[2].test, MockModels.speedometer);
558             assert.equal(configurations[3].platform, MockModels.iOS10iPhone);
559             assert.equal(configurations[3].test, MockModels.speedometer);
560
561             configurations = syncers[1].testConfigurations();
562             assert.equal(configurations.length, 2);
563             assert.equal(configurations[0].platform, MockModels.ipad);
564             assert.equal(configurations[0].test, MockModels.iPadPLT);
565             assert.equal(configurations[1].platform, MockModels.ipad);
566             assert.equal(configurations[1].test, MockModels.speedometer);
567         });
568     });
569
570     describe('_propertiesForBuildRequest', function () {
571         it('should include all properties specified in a given configuration', function () {
572             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
573             let properties = syncers[0]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
574             assert.deepEqual(Object.keys(properties), ['desired_image', 'opensource', 'roots_dict', 'test_name', 'forcescheduler', 'build_request_id']);
575         });
576
577         it('should preserve non-parametric property values', function () {
578             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
579             let properties = syncers[0]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
580             assert.equal(properties['test_name'], 'speedometer');
581             assert.equal(properties['forcescheduler'], 'ABTest-iPhone-RunBenchmark-Tests-ForceScheduler');
582
583             properties = syncers[1]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.ipad, MockModels.jetstream));
584             assert.equal(properties['test_name'], 'jetstream');
585             assert.equal(properties['forcescheduler'], 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler');
586         });
587
588         it('should resolve "root"', function () {
589             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
590             let properties = syncers[0]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
591             assert.equal(properties['desired_image'], '13A452');
592         });
593
594         it('should resolve "rootOptions"', function () {
595             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
596             let properties = syncers[0]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
597             assert.equal(properties['roots_dict'], JSON.stringify(sampleRootSetData));
598         });
599
600         it('should resolve "rootsExcluding"', function () {
601             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
602             let properties = syncers[0]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
603             assert.equal(properties['roots_dict'], JSON.stringify(sampleRootSetData));
604         });
605
606         it('should set the property for the build request id', function () {
607             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
608             let request = createSampleBuildRequest(MockModels.iphone, MockModels.speedometer);
609             let properties = syncers[0]._propertiesForBuildRequest(request);
610             assert.equal(properties['build_request_id'], request.id());
611         });
612     });
613
614     describe('pullBuildbot', function () {
615         it('should fetch pending builds from the right URL', function () {
616             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
617             assert.equal(syncer.builderName(), 'ABTest-iPad-RunBenchmark-Tests');
618             let expectedURL = '/json/builders/ABTest-iPad-RunBenchmark-Tests/pendingBuilds';
619             assert.equal(syncer.pathForPendingBuildsJSON(), expectedURL);
620             syncer.pullBuildbot();
621             assert.equal(requests.length, 1);
622             assert.equal(requests[0].url, expectedURL);
623         });
624
625         it('should fetch recent builds once pending builds have been fetched', function (done) {
626             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
627             assert.equal(syncer.builderName(), 'ABTest-iPad-RunBenchmark-Tests');
628
629             syncer.pullBuildbot(1);
630             assert.equal(requests.length, 1);
631             assert.equal(requests[0].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/pendingBuilds');
632             requests[0].resolve([]);
633             Promise.resolve().then(function () {
634                 assert.equal(requests.length, 2);
635                 assert.equal(requests[1].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/builds/?select=-1');
636                 done();
637             }).catch(done);
638         });
639
640         it('should fetch the right number of recent builds', function (done) {
641             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
642
643             syncer.pullBuildbot(3);
644             assert.equal(requests.length, 1);
645             assert.equal(requests[0].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/pendingBuilds');
646             requests[0].resolve([]);
647             Promise.resolve().then(function () {
648                 assert.equal(requests.length, 2);
649                 assert.equal(requests[1].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/builds/?select=-1&select=-2&select=-3');
650                 done();
651             }).catch(done);
652         });
653
654         it('should create BuildbotBuildEntry for pending builds', function (done) {
655             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
656             let promise = syncer.pullBuildbot();
657             requests[0].resolve([samplePendingBuild()]);
658             promise.then(function (entries) {
659                 assert.equal(entries.length, 1);
660                 let entry = entries[0];
661                 assert.ok(entry instanceof BuildbotBuildEntry);
662                 assert.ok(!entry.buildNumber());
663                 assert.ok(!entry.slaveName());
664                 assert.equal(entry.buildRequestId(), 16733);
665                 assert.ok(entry.isPending());
666                 assert.ok(!entry.isInProgress());
667                 assert.ok(!entry.hasFinished());
668                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
669                 done();
670             }).catch(done);
671         });
672
673         it('should create BuildbotBuildEntry for in-progress builds', function (done) {
674             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
675
676             let promise = syncer.pullBuildbot(1);
677             assert.equal(requests.length, 1);
678             requests[0].resolve([]);
679             Promise.resolve().then(function () {
680                 assert.equal(requests.length, 2);
681                 requests[1].resolve({[-1]: sampleInProgressBuild()});
682             }).catch(done);
683
684             promise.then(function (entries) {
685                 assert.equal(entries.length, 1);
686                 let entry = entries[0];
687                 assert.ok(entry instanceof BuildbotBuildEntry);
688                 assert.equal(entry.buildNumber(), 614);
689                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
690                 assert.equal(entry.buildRequestId(), 16733);
691                 assert.ok(!entry.isPending());
692                 assert.ok(entry.isInProgress());
693                 assert.ok(!entry.hasFinished());
694                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
695                 done();
696             }).catch(done);
697         });
698
699         it('should create BuildbotBuildEntry for finished builds', function (done) {
700             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
701
702             let promise = syncer.pullBuildbot(1);
703             assert.equal(requests.length, 1);
704             requests[0].resolve([]);
705             Promise.resolve().then(function () {
706                 assert.equal(requests.length, 2);
707                 requests[1].resolve({[-1]: sampleFinishedBuild()});
708             }).catch(done);
709
710             promise.then(function (entries) {
711                 assert.deepEqual(entries.length, 1);
712                 let entry = entries[0];
713                 assert.ok(entry instanceof BuildbotBuildEntry);
714                 assert.equal(entry.buildNumber(), 1755);
715                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
716                 assert.equal(entry.buildRequestId(), 18935);
717                 assert.ok(!entry.isPending());
718                 assert.ok(!entry.isInProgress());
719                 assert.ok(entry.hasFinished());
720                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
721                 done();
722             }).catch(done);
723         });
724
725         it('should create BuildbotBuildEntry for mixed pending, in-progress, finished, and missing builds', function (done) {
726             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
727
728             let promise = syncer.pullBuildbot(5);
729             assert.equal(requests.length, 1);
730
731             requests[0].resolve([samplePendingBuild(123)]);
732
733             Promise.resolve().then(function () {
734                 assert.equal(requests.length, 2);
735                 requests[1].resolve({[-1]: sampleFinishedBuild(), [-2]: {'error': 'Not available'}, [-4]: sampleInProgressBuild()});
736             }).catch(done);
737
738             promise.then(function (entries) {
739                 assert.deepEqual(entries.length, 3);
740
741                 let entry = entries[0];
742                 assert.ok(entry instanceof BuildbotBuildEntry);
743                 assert.equal(entry.buildNumber(), null);
744                 assert.equal(entry.slaveName(), null);
745                 assert.equal(entry.buildRequestId(), 123);
746                 assert.ok(entry.isPending());
747                 assert.ok(!entry.isInProgress());
748                 assert.ok(!entry.hasFinished());
749                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
750
751                 entry = entries[1];
752                 assert.ok(entry instanceof BuildbotBuildEntry);
753                 assert.equal(entry.buildNumber(), 614);
754                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
755                 assert.equal(entry.buildRequestId(), 16733);
756                 assert.ok(!entry.isPending());
757                 assert.ok(entry.isInProgress());
758                 assert.ok(!entry.hasFinished());
759                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
760
761                 entry = entries[2];
762                 assert.ok(entry instanceof BuildbotBuildEntry);
763                 assert.equal(entry.buildNumber(), 1755);
764                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
765                 assert.equal(entry.buildRequestId(), 18935);
766                 assert.ok(!entry.isPending());
767                 assert.ok(!entry.isInProgress());
768                 assert.ok(entry.hasFinished());
769                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
770
771                 done();
772             }).catch(done);
773         });
774
775         it('should sort BuildbotBuildEntry by order', function (done) {
776             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
777
778             let promise = syncer.pullBuildbot(5);
779             assert.equal(requests.length, 1);
780
781             requests[0].resolve([samplePendingBuild(456, 2), samplePendingBuild(123, 1)]);
782
783             Promise.resolve().then(function () {
784                 assert.equal(requests.length, 2);
785                 requests[1].resolve({[-3]: sampleFinishedBuild(), [-1]: {'error': 'Not available'}, [-2]: sampleInProgressBuild()});
786             }).catch(done);
787
788             promise.then(function (entries) {
789                 assert.deepEqual(entries.length, 4);
790
791                 let entry = entries[0];
792                 assert.ok(entry instanceof BuildbotBuildEntry);
793                 assert.equal(entry.buildNumber(), null);
794                 assert.equal(entry.slaveName(), null);
795                 assert.equal(entry.buildRequestId(), 123);
796                 assert.ok(entry.isPending());
797                 assert.ok(!entry.isInProgress());
798                 assert.ok(!entry.hasFinished());
799                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
800
801                 entry = entries[1];
802                 assert.ok(entry instanceof BuildbotBuildEntry);
803                 assert.equal(entry.buildNumber(), null);
804                 assert.equal(entry.slaveName(), null);
805                 assert.equal(entry.buildRequestId(), 456);
806                 assert.ok(entry.isPending());
807                 assert.ok(!entry.isInProgress());
808                 assert.ok(!entry.hasFinished());
809                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
810
811                 entry = entries[2];
812                 assert.ok(entry instanceof BuildbotBuildEntry);
813                 assert.equal(entry.buildNumber(), 614);
814                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
815                 assert.equal(entry.buildRequestId(), 16733);
816                 assert.ok(!entry.isPending());
817                 assert.ok(entry.isInProgress());
818                 assert.ok(!entry.hasFinished());
819                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
820
821                 entry = entries[3];
822                 assert.ok(entry instanceof BuildbotBuildEntry);
823                 assert.equal(entry.buildNumber(), 1755);
824                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
825                 assert.equal(entry.buildRequestId(), 18935);
826                 assert.ok(!entry.isPending());
827                 assert.ok(!entry.isInProgress());
828                 assert.ok(entry.hasFinished());
829                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
830
831                 done();
832             }).catch(done);
833         });
834
835         it('should override BuildbotBuildEntry for pending builds by in-progress builds', function (done) {
836             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
837
838             let promise = syncer.pullBuildbot(5);
839             assert.equal(requests.length, 1);
840
841             requests[0].resolve([samplePendingBuild()]);
842
843             Promise.resolve().then(function () {
844                 assert.equal(requests.length, 2);
845                 requests[1].resolve({[-1]: sampleInProgressBuild()});
846             }).catch(done);
847
848             promise.then(function (entries) {
849                 assert.equal(entries.length, 1);
850
851                 let entry = entries[0];
852                 assert.ok(entry instanceof BuildbotBuildEntry);
853                 assert.equal(entry.buildNumber(), 614);
854                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
855                 assert.equal(entry.buildRequestId(), 16733);
856                 assert.ok(!entry.isPending());
857                 assert.ok(entry.isInProgress());
858                 assert.ok(!entry.hasFinished());
859                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
860
861                 done();
862             }).catch(done);
863         });
864
865         it('should override BuildbotBuildEntry for pending builds by finished builds', function (done) {
866             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
867
868             let promise = syncer.pullBuildbot(5);
869             assert.equal(requests.length, 1);
870
871             requests[0].resolve([samplePendingBuild()]);
872
873             Promise.resolve().then(function () {
874                 assert.equal(requests.length, 2);
875                 requests[1].resolve({[-1]: sampleFinishedBuild(16733)});
876             }).catch(done);
877
878             promise.then(function (entries) {
879                 assert.equal(entries.length, 1);
880
881                 let entry = entries[0];
882                 assert.ok(entry instanceof BuildbotBuildEntry);
883                 assert.equal(entry.buildNumber(), 1755);
884                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
885                 assert.equal(entry.buildRequestId(), 16733);
886                 assert.ok(!entry.isPending());
887                 assert.ok(!entry.isInProgress());
888                 assert.ok(entry.hasFinished());
889                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
890
891                 done();
892             }).catch(done);
893         });
894     });
895
896     describe('scheduleRequest', function () {
897         it('should schedule a build request on a specified slave', function (done) {
898             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[0];
899
900             syncer.scheduleRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer), 'some-slave');
901             Promise.resolve().then(function () {
902                 assert.equal(requests.length, 1);
903                 assert.equal(requests[0].url, '/builders/ABTest-iPhone-RunBenchmark-Tests/force');
904                 assert.equal(requests[0].method, 'POST');
905                 assert.deepEqual(requests[0].data, {
906                     'build_request_id': '16733-' + MockModels.iphone.id(),
907                     'desired_image': '13A452',
908                     "opensource": "9abcdef",
909                     'forcescheduler': 'ABTest-iPhone-RunBenchmark-Tests-ForceScheduler',
910                     'roots_dict': '{"WebKit":{"id":"111127","time":1456955807334,"repository":"WebKit","revision":"197463"},'
911                         + '"Shared":{"id":"111237","time":1456931874000,"repository":"Shared","revision":"80229"},'
912                         + '"WebKit-Git":{"id":"111239","time":1456931874000,"repository":"WebKit-Git","revision":"9abcdef"}}',
913                     'slavename': 'some-slave',
914                     'test_name': 'speedometer'
915                 });
916                 done();
917             }).catch(done);
918         });
919     });
920
921     describe('scheduleRequestInGroupIfAvailable', function () {
922
923         function pullBuildbotWithAssertion(syncer, pendingBuilds, inProgressAndFinishedBuilds)
924         {
925             let promise = syncer.pullBuildbot(5);
926             assert.equal(requests.length, 1);
927             requests[0].resolve(pendingBuilds);
928             return Promise.resolve().then(function () {
929                 assert.equal(requests.length, 2);
930                 requests[1].resolve(inProgressAndFinishedBuilds);
931                 requests.length = 0;
932             }).then(function () {
933                 return promise;
934             });
935         }
936
937         it('should schedule a build if builder has no builds if slaveList is not specified', function (done) {
938             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [smallConfiguration()]})[0];
939
940             pullBuildbotWithAssertion(syncer, [], {}).then(function () {
941                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
942             }).then(function () {
943                 assert.equal(requests.length, 1);
944                 assert.equal(requests[0].url, '/builders/some%20builder/force');
945                 assert.equal(requests[0].method, 'POST');
946                 assert.deepEqual(requests[0].data, {id: '16733-' + MockModels.somePlatform.id()});
947                 done();
948             }).catch(done);
949         });
950
951         it('should schedule a build if builder only has finished builds if slaveList is not specified', function (done) {
952             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [smallConfiguration()]})[0];
953
954             pullBuildbotWithAssertion(syncer, [], {[-1]: smallFinishedBuild()}).then(function () {
955                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
956             }).then(function () {
957                 assert.equal(requests.length, 1);
958                 assert.equal(requests[0].url, '/builders/some%20builder/force');
959                 assert.equal(requests[0].method, 'POST');
960                 assert.deepEqual(requests[0].data, {id: '16733-' + MockModels.somePlatform.id()});
961                 done();
962             }).catch(done);
963         });
964
965         it('should not schedule a build if builder has a pending build if slaveList is not specified', function (done) {
966             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [smallConfiguration()]})[0];
967
968             pullBuildbotWithAssertion(syncer, [smallPendingBuild()], {}).then(function () {
969                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
970             }).then(function () {
971                 assert.equal(requests.length, 0);
972                 done();
973             }).catch(done);
974         });
975
976         it('should schedule a build if builder does not have pending or completed builds on the matching slave', function (done) {
977             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[0];
978
979             pullBuildbotWithAssertion(syncer, [], {}).then(function () {
980                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
981             }).then(function () {
982                 assert.equal(requests.length, 1);
983                 assert.equal(requests[0].url, '/builders/ABTest-iPhone-RunBenchmark-Tests/force');
984                 assert.equal(requests[0].method, 'POST');
985                 done();
986             }).catch(done);
987         });
988
989         it('should schedule a build if builder only has finished builds on the matching slave', function (done) {
990             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
991
992             pullBuildbotWithAssertion(syncer, [], {[-1]: sampleFinishedBuild()}).then(function () {
993                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
994             }).then(function () {
995                 assert.equal(requests.length, 1);
996                 assert.equal(requests[0].url, '/builders/ABTest-iPad-RunBenchmark-Tests/force');
997                 assert.equal(requests[0].method, 'POST');
998                 done();
999             }).catch(done);
1000         });
1001
1002         it('should not schedule a build if builder has a pending build on the maching slave', function (done) {
1003             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1004
1005             pullBuildbotWithAssertion(syncer, [samplePendingBuild()], {}).then(function () {
1006                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1007             }).then(function () {
1008                 assert.equal(requests.length, 0);
1009                 done();
1010             }).catch(done);
1011         });
1012
1013         it('should schedule a build if builder only has a pending build on a non-maching slave', function (done) {
1014             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1015
1016             pullBuildbotWithAssertion(syncer, [samplePendingBuild(1, 1, 'another-slave')], {}).then(function () {
1017                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1018             }).then(function () {
1019                 assert.equal(requests.length, 1);
1020                 done();
1021             }).catch(done);
1022         });
1023
1024         it('should schedule a build if builder only has an in-progress build on the matching slave', function (done) {
1025             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1026
1027             pullBuildbotWithAssertion(syncer, [], {[-1]: sampleInProgressBuild()}).then(function () {
1028                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1029             }).then(function () {
1030                 assert.equal(requests.length, 1);
1031                 done();
1032             }).catch(done);
1033         });
1034
1035         it('should schedule a build if builder has an in-progress build on another slave', function (done) {
1036             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1037
1038             pullBuildbotWithAssertion(syncer, [], {[-1]: sampleInProgressBuild('other-slave')}).then(function () {
1039                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1040             }).then(function () {
1041                 assert.equal(requests.length, 1);
1042                 done();
1043             }).catch(done);
1044         });
1045
1046         it('should not schedule a build if the request does not match any configuration', function (done) {
1047             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[0];
1048
1049             pullBuildbotWithAssertion(syncer, [], {}).then(function () {
1050                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1051             }).then(function () {
1052                 assert.equal(requests.length, 0);
1053                 done();
1054             }).catch(done);
1055         });
1056
1057         it('should not schedule a build if a new request had been submitted to the same slave', function (done) {
1058             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1059
1060             pullBuildbotWithAssertion(syncer, [], {}).then(function () {
1061                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-0');
1062                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-1');
1063             }).then(function () {
1064                 assert.equal(requests.length, 2);
1065                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1066             }).then(function () {
1067                 assert.equal(requests.length, 2);
1068                 done();
1069             }).catch(done);
1070         });
1071
1072         it('should schedule a build if a new request had been submitted to another slave', function (done) {
1073             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1074
1075             pullBuildbotWithAssertion(syncer, [], {}).then(function () {
1076                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-0');
1077             }).then(function () {
1078                 assert.equal(requests.length, 1);
1079                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-1');
1080             }).then(function () {
1081                 assert.equal(requests.length, 2);
1082                 done();
1083             }).catch(done);
1084         });
1085
1086         it('should not schedule a build if a new request had been submitted to the same builder without slaveList', function (done) {
1087             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, {'configurations': [smallConfiguration()]})[0];
1088
1089             pullBuildbotWithAssertion(syncer, [], {}).then(function () {
1090                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest), null);
1091             }).then(function () {
1092                 assert.equal(requests.length, 1);
1093                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
1094             }).then(function () {
1095                 assert.equal(requests.length, 1);
1096                 done();
1097             }).catch(done);
1098         });
1099     });
1100 });