Add the support for perf try bots to sync-buildbot.js
[WebKit-https.git] / Websites / perf.webkit.org / unit-tests / buildbot-syncer-tests.js
1 'use strict';
2
3 let assert = require('assert');
4
5 require('../tools/js/v3-models.js');
6 let MockRemoteAPI = require('./resources/mock-remote-api.js').MockRemoteAPI;
7 let MockModels = require('./resources/mock-v3-models.js').MockModels;
8
9 let BuildbotBuildEntry = require('../tools/js/buildbot-syncer.js').BuildbotBuildEntry;
10 let BuildbotSyncer = require('../tools/js/buildbot-syncer.js').BuildbotSyncer;
11
12 function sampleiOSConfig()
13 {
14     return {
15         'slaveArgument': 'slavename',
16         'buildRequestArgument': 'build_request_id',
17         'repositoryGroups': {
18             'ios-svn-webkit': {
19                 'repositories': {'WebKit': {}, 'iOS': {}},
20                 'testProperties': {
21                     'desired_image': {'revision': 'iOS'},
22                     'opensource': {'revision': 'WebKit'},
23                 }
24             }
25         },
26         'types': {
27             'speedometer': {
28                 'test': ['Speedometer'],
29                 'properties': {'test_name': 'speedometer'}
30             },
31             'jetstream': {
32                 'test': ['JetStream'],
33                 'properties': {'test_name': 'jetstream'}
34             },
35             'dromaeo-dom': {
36                 'test': ['Dromaeo', 'DOM Core Tests'],
37                 'properties': {'tests': 'dromaeo-dom'}
38             },
39         },
40         'builders': {
41             'iPhone-bench': {
42                 'builder': 'ABTest-iPhone-RunBenchmark-Tests',
43                 'properties': {'forcescheduler': 'ABTest-iPhone-RunBenchmark-Tests-ForceScheduler'},
44                 'slaveList': ['ABTest-iPhone-0'],
45             },
46             'iPad-bench': {
47                 'builder': 'ABTest-iPad-RunBenchmark-Tests',
48                 'properties': {'forcescheduler': 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler'},
49                 'slaveList': ['ABTest-iPad-0', 'ABTest-iPad-1'],
50             },
51             'iOS-builder': {
52                 'builder': 'ABTest-iOS-Builder',
53                 'properties': {'forcescheduler': 'ABTest-Builder-ForceScheduler'},
54             },
55         },
56         'buildConfigurations': [
57             {'builders': ['iOS-builder'], 'platforms': ['iPhone', 'iPad']},
58         ],
59         'testConfigurations': [
60             {'builders': ['iPhone-bench'], 'types': ['speedometer', 'jetstream', 'dromaeo-dom'], 'platforms': ['iPhone']},
61             {'builders': ['iPad-bench'], 'types': ['speedometer', 'jetstream'], 'platforms': ['iPad']},
62         ]
63     };
64 }
65
66 function sampleiOSConfigWithExpansions()
67 {
68     return {
69         "triggerableName": "build-webkit-ios",
70         "buildRequestArgument": "build-request-id",
71         "repositoryGroups": { },
72         "types": {
73             "iphone-plt": {
74                 "test": ["PLT-iPhone"],
75                 "properties": {"test_name": "plt"}
76             },
77             "ipad-plt": {
78                 "test": ["PLT-iPad"],
79                 "properties": {"test_name": "plt"}
80             },
81             "speedometer": {
82                 "test": ["Speedometer"],
83                 "properties": {"tests": "speedometer"}
84             },
85         },
86         "builders": {
87             "iphone": {
88                 "builder": "iPhone AB Tests",
89                 "properties": {"forcescheduler": "force-iphone-ab-tests"},
90             },
91             "iphone-2": {
92                 "builder": "iPhone 2 AB Tests",
93                 "properties": {"forcescheduler": "force-iphone-2-ab-tests"},
94             },
95             "ipad": {
96                 "builder": "iPad AB Tests",
97                 "properties": {"forcescheduler": "force-ipad-ab-tests"},
98             },
99         },
100         "testConfigurations": [
101             {
102                 "builders": ["iphone", "iphone-2"],
103                 "platforms": ["iPhone", "iOS 10 iPhone"],
104                 "types": ["iphone-plt", "speedometer"],
105             },
106             {
107                 "builders": ["ipad"],
108                 "platforms": ["iPad"],
109                 "types": ["ipad-plt", "speedometer"],
110             },
111         ]
112     }
113 }
114
115 function smallConfiguration()
116 {
117     return {
118         'buildRequestArgument': 'id',
119         'repositoryGroups': {
120             'ios-svn-webkit': {
121                 'repositories': {'iOS': {}, 'WebKit': {}},
122                 'testProperties': {
123                     'os': {'revision': 'iOS'},
124                     'wk': {'revision': 'WebKit'}
125                 }
126             }
127         },
128         'types': {
129             'some-test': {
130                 'test': ['Some test'],
131             }
132         },
133         'builders': {
134             'some-builder': {
135                 'builder': 'some builder',
136             }
137         },
138         'testConfigurations': [{
139             'builders': ['some-builder'],
140             'platforms': ['Some platform'],
141             'types': ['some-test'],
142         }]
143     };
144 }
145
146 function smallPendingBuild()
147 {
148     return {
149         'builderName': 'some builder',
150         'builds': [],
151         'properties': [],
152         'source': {
153             'branch': '',
154             'changes': [],
155             'codebase': 'WebKit',
156             'hasPatch': false,
157             'project': '',
158             'repository': '',
159             'revision': ''
160         },
161     };
162 }
163
164 function smallInProgressBuild()
165 {
166     return {
167         'builderName': 'some builder',
168         'builds': [],
169         'properties': [],
170         'currentStep': { },
171         'eta': 123,
172         'number': 456,
173         'source': {
174             'branch': '',
175             'changes': [],
176             'codebase': 'WebKit',
177             'hasPatch': false,
178             'project': '',
179             'repository': '',
180             'revision': ''
181         },
182     };
183 }
184
185 function smallFinishedBuild()
186 {
187     return {
188         'builderName': 'some builder',
189         'builds': [],
190         'properties': [],
191         'currentStep': null,
192         'eta': null,
193         'number': 789,
194         'source': {
195             'branch': '',
196             'changes': [],
197             'codebase': 'WebKit',
198             'hasPatch': false,
199             'project': '',
200             'repository': '',
201             'revision': ''
202         },
203         'times': [0, 1],
204     };
205 }
206
207 function createSampleBuildRequest(platform, test)
208 {
209     assert(platform instanceof Platform);
210     assert(test instanceof Test);
211
212     const webkit197463 = CommitLog.ensureSingleton('111127', {'id': '111127', 'time': 1456955807334, 'repository': MockModels.webkit, 'revision': '197463'});
213     const shared111237 = CommitLog.ensureSingleton('111237', {'id': '111237', 'time': 1456931874000, 'repository': MockModels.sharedRepository, 'revision': '80229'});
214     const ios13A452 = CommitLog.ensureSingleton('88930', {'id': '88930', 'time': 0, 'repository': MockModels.ios, 'revision': '13A452'});
215
216     const commitSet = CommitSet.ensureSingleton('4197', {customRoots: [], revisionItems: [{commit: webkit197463}, {commit: shared111237}, {commit: ios13A452}]});
217
218     return BuildRequest.ensureSingleton('16733-' + platform.id(), {'triggerable': MockModels.triggerable,
219         repositoryGroup: MockModels.svnRepositoryGroup,
220         'commitSet': commitSet, 'status': 'pending', 'platform': platform, 'test': test});
221 }
222
223 function samplePendingBuild(buildRequestId, buildTime, slaveName)
224 {
225     return {
226         'builderName': 'ABTest-iPad-RunBenchmark-Tests',
227         'builds': [],
228         'properties': [
229             ['build_request_id', buildRequestId || '16733', 'Force Build Form'],
230             ['desired_image', '13A452', 'Force Build Form'],
231             ['owner', '<unknown>', 'Force Build Form'],
232             ['test_name', 'speedometer', 'Force Build Form'],
233             ['reason', 'force build','Force Build Form'],
234             ['slavename', slaveName, ''],
235             ['scheduler', 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler', 'Scheduler']
236         ],
237         'source': {
238             'branch': '',
239             'changes': [],
240             'codebase': 'compiler-rt',
241             'hasPatch': false,
242             'project': '',
243             'repository': '',
244             'revision': ''
245         },
246         'submittedAt': buildTime || 1458704983
247     };
248 }
249
250 function sampleInProgressBuild(slaveName)
251 {
252     return {
253         'blame': [],
254         'builderName': 'ABTest-iPad-RunBenchmark-Tests',
255         'currentStep': {
256             'eta': 0.26548067698460565,
257             'expectations': [['output', 845, 1315.0]],
258             'hidden': false,
259             'isFinished': false,
260             'isStarted': true,
261             'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/Some%20step/logs/stdio']],
262             'name': 'Some step',
263             'results': [null,[]],
264             'statistics': {},
265             'step_number': 1,
266             'text': [''],
267             'times': [1458718657.581628, null],
268             'urls': {}
269         },
270         'eta': 6497.991612434387,
271         'logs': [['stdio','https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/shell/logs/stdio']],
272         'number': 614,
273         'properties': [
274             ['build_request_id', '16733', 'Force Build Form'],
275             ['buildername', 'ABTest-iPad-RunBenchmark-Tests', 'Builder'],
276             ['buildnumber', 614, 'Build'],
277             ['desired_image', '13A452', 'Force Build Form'],
278             ['owner', '<unknown>', 'Force Build Form'],
279             ['reason', 'force build', 'Force Build Form'],
280             ['scheduler', 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler', 'Scheduler'],
281             ['slavename', slaveName || 'ABTest-iPad-0', 'BuildSlave'],
282         ],
283         'reason': 'A build was forced by \'<unknown>\': force build',
284         'results': null,
285         'slave': 'ABTest-iPad-0',
286         'sourceStamps': [{'branch': '', 'changes': [], 'codebase': 'compiler-rt', 'hasPatch': false, 'project': '', 'repository': '', 'revision': ''}],
287         'steps': [
288             {
289                 'eta': null,
290                 'expectations': [['output',2309,2309.0]],
291                 'hidden': false,
292                 'isFinished': true,
293                 'isStarted': true,
294                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/shell/logs/stdio']],
295                 'name': 'Finished step',
296                 'results': [0, []],
297                 'statistics': {},
298                 'step_number': 0,
299                 'text': [''],
300                 'times': [1458718655.419865, 1458718655.453633],
301                 'urls': {}
302             },
303             {
304                 'eta': 0.26548067698460565,
305                 'expectations': [['output', 845, 1315.0]],
306                 'hidden': false,
307                 'isFinished': false,
308                 'isStarted': true,
309                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/Some%20step/logs/stdio']],
310                 'name': 'Some step',
311                 'results': [null,[]],
312                 'statistics': {},
313                 'step_number': 1,
314                 'text': [''],
315                 'times': [1458718657.581628, null],
316                 'urls': {}
317             },
318             {
319                 'eta': null,
320                 'expectations': [['output', null, null]],
321                 'hidden': false,
322                 'isFinished': false,
323                 'isStarted': false,
324                 'logs': [],
325                 'name': 'Some other step',
326                 'results': [null, []],
327                 'statistics': {},
328                 'step_number': 2,
329                 'text': [],
330                 'times': [null, null],
331                 'urls': {}
332             },
333         ],
334         'text': [],
335         'times': [1458718655.415821, null]
336     };
337 }
338
339 function sampleFinishedBuild(buildRequestId, slaveName)
340 {
341     return {
342         'blame': [],
343         'builderName': 'ABTest-iPad-RunBenchmark-Tests',
344         'currentStep': null,
345         'eta': null,
346         'logs': [['stdio','https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755/steps/shell/logs/stdio']],
347         'number': 1755,
348         'properties': [
349             ['build_request_id', buildRequestId || '18935', 'Force Build Form'],
350             ['buildername', 'ABTest-iPad-RunBenchmark-Tests', 'Builder'],
351             ['buildnumber', 1755, 'Build'],
352             ['desired_image', '13A452', 'Force Build Form'],
353             ['owner', '<unknown>', 'Force Build Form'],
354             ['reason', 'force build', 'Force Build Form'],
355             ['scheduler', 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler', 'Scheduler'],
356             ['slavename', slaveName || 'ABTest-iPad-0', 'BuildSlave'],
357         ],
358         'reason': 'A build was forced by \'<unknown>\': force build',
359         'results': 2,
360         'slave': 'ABTest-iPad-0',
361         'sourceStamps': [{'branch': '', 'changes': [], 'codebase': 'compiler-rt', 'hasPatch': false, 'project': '', 'repository': '', 'revision': ''}],
362         'steps': [
363             {
364                 'eta': null,
365                 'expectations': [['output',2309,2309.0]],
366                 'hidden': false,
367                 'isFinished': true,
368                 'isStarted': true,
369                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/shell/logs/stdio']],
370                 'name': 'Finished step',
371                 'results': [0, []],
372                 'statistics': {},
373                 'step_number': 0,
374                 'text': [''],
375                 'times': [1458718655.419865, 1458718655.453633],
376                 'urls': {}
377             },
378             {
379                 'eta': null,
380                 'expectations': [['output', 845, 1315.0]],
381                 'hidden': false,
382                 'isFinished': true,
383                 'isStarted': true,
384                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/Some%20step/logs/stdio']],
385                 'name': 'Some step',
386                 'results': [null,[]],
387                 'statistics': {},
388                 'step_number': 1,
389                 'text': [''],
390                 'times': [1458718657.581628, null],
391                 'urls': {}
392             },
393             {
394                 'eta': null,
395                 'expectations': [['output', null, null]],
396                 'hidden': false,
397                 'isFinished': true,
398                 'isStarted': true,
399                 'logs': [],
400                 'name': 'Some other step',
401                 'results': [null, []],
402                 'statistics': {},
403                 'step_number': 2,
404                 'text': [],
405                 'times': [null, null],
406                 'urls': {}
407             },
408         ],
409         'text': [],
410         'times': [1458937478.25837, 1458946147.173785]
411     };
412 }
413
414 describe('BuildbotSyncer', () => {
415     MockModels.inject();
416     let requests = MockRemoteAPI.inject('http://build.webkit.org');
417
418     describe('_loadConfig', () => {
419
420         it('should create BuildbotSyncer objects for a configuration that specify all required options', () => {
421             assert.equal(BuildbotSyncer._loadConfig(MockRemoteAPI, smallConfiguration()).length, 1);
422         });
423
424         it('should throw when some required options are missing', () => {
425             assert.throws(() => {
426                 const config = smallConfiguration();
427                 delete config.builders;
428                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
429             }, /"some-builder" is not a valid builder in the configuration/);
430             assert.throws(() => {
431                 const config = smallConfiguration();
432                 delete config.types;
433                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
434             }, /"some-test" is not a valid type in the configuration/);
435             assert.throws(() => {
436                 const config = smallConfiguration();
437                 delete config.testConfigurations[0].builders;
438                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
439             }, /The test configuration 1 does not specify "builders" as an array/);
440             assert.throws(() => {
441                 const config = smallConfiguration();
442                 delete config.testConfigurations[0].platforms;
443                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
444             }, /The test configuration 1 does not specify "platforms" as an array/);
445             assert.throws(() => {
446                 const config = smallConfiguration();
447                 delete config.testConfigurations[0].types;
448                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
449             }, /The test configuration 0 does not specify "types" as an array/);
450             assert.throws(() => {
451                 const config = smallConfiguration();
452                 delete config.buildRequestArgument;
453                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
454             }, /buildRequestArgument must specify the name of the property used to store the build request ID/);
455         });
456
457         it('should throw when a test name is not an array of strings', () => {
458             assert.throws(() => {
459                 const config = smallConfiguration();
460                 config.testConfigurations[0].types = 'some test';
461                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
462             }, /The test configuration 0 does not specify "types" as an array/);
463             assert.throws(() => {
464                 const config = smallConfiguration();
465                 config.testConfigurations[0].types = [1];
466                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
467             }, /"1" is not a valid type in the configuration/);
468         });
469
470         it('should throw when properties is not an object', () => {
471             assert.throws(() => {
472                 const config = smallConfiguration();
473                 config.builders[Object.keys(config.builders)[0]].properties = 'hello';
474                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
475             }, /Build properties should be a dictionary/);
476             assert.throws(() => {
477                 const config = smallConfiguration();
478                 config.types[Object.keys(config.types)[0]].properties = 'hello';
479                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
480             }, /Build properties should be a dictionary/);
481         });
482
483         it('should throw when testProperties is specifed in a type or a builder', () => {
484             assert.throws(() => {
485                 const config = smallConfiguration();
486                 const firstType = Object.keys(config.types)[0];
487                 config.types[firstType].testProperties = {};
488                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
489             }, /Unrecognized parameter "testProperties"/);
490             assert.throws(() => {
491                 const config = smallConfiguration();
492                 const firstBuilder = Object.keys(config.builders)[0];
493                 config.builders[firstBuilder].testProperties = {};
494                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
495             }, /Unrecognized parameter "testProperties"/);
496         });
497
498         it('should throw when buildProperties is specifed in a type or a builder', () => {
499             assert.throws(() => {
500                 const config = smallConfiguration();
501                 const firstType = Object.keys(config.types)[0];
502                 config.types[firstType].buildProperties = {};
503                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
504             }, /Unrecognized parameter "buildProperties"/);
505             assert.throws(() => {
506                 const config = smallConfiguration();
507                 const firstBuilder = Object.keys(config.builders)[0];
508                 config.builders[firstBuilder].buildProperties = {};
509                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
510             }, /Unrecognized parameter "buildProperties"/);
511         });
512
513         it('should throw when properties for a type is malformed', () => {
514             const firstType = Object.keys(smallConfiguration().types)[0];
515             assert.throws(() => {
516                 const config = smallConfiguration();
517                 config.types[firstType].properties = 'hello';
518                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
519             }, /Build properties should be a dictionary/);
520             assert.throws(() => {
521                 const config = smallConfiguration();
522                 config.types[firstType].properties = {'some': {'otherKey': 'some root'}};
523                 BuildbotSyncer._loadConfig(RemoteAPI, config);
524             }, /Build properties "some" specifies a non-string value of type "object"/);
525             assert.throws(() => {
526                 const config = smallConfiguration();
527                 config.types[firstType].properties = {'some': {'otherKey': 'some root'}};
528                 BuildbotSyncer._loadConfig(RemoteAPI, config);
529             }, /Build properties "some" specifies a non-string value of type "object"/);
530             assert.throws(() => {
531                 const config = smallConfiguration();
532                 config.types[firstType].properties = {'some': {'revision': 'WebKit'}};
533                 BuildbotSyncer._loadConfig(RemoteAPI, config);
534             }, /Build properties "some" specifies a non-string value of type "object"/);
535             assert.throws(() => {
536                 const config = smallConfiguration();
537                 config.types[firstType].properties = {'some': 1};
538                 BuildbotSyncer._loadConfig(RemoteAPI, config);
539             }, / Build properties "some" specifies a non-string value of type "object"/);
540         });
541
542         it('should throw when properties for a builder is malformed', () => {
543             const firstBuilder = Object.keys(smallConfiguration().builders)[0];
544             assert.throws(() => {
545                 const config = smallConfiguration();
546                 config.builders[firstBuilder].properties = 'hello';
547                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
548             }, /Build properties should be a dictionary/);
549             assert.throws(() => {
550                 const config = smallConfiguration();
551                 config.builders[firstBuilder].properties = {'some': {'otherKey': 'some root'}};
552                 BuildbotSyncer._loadConfig(RemoteAPI, config);
553             }, /Build properties "some" specifies a non-string value of type "object"/);
554             assert.throws(() => {
555                 const config = smallConfiguration();
556                 config.builders[firstBuilder].properties = {'some': {'otherKey': 'some root'}};
557                 BuildbotSyncer._loadConfig(RemoteAPI, config);
558             }, /Build properties "some" specifies a non-string value of type "object"/);
559             assert.throws(() => {
560                 const config = smallConfiguration();
561                 config.builders[firstBuilder].properties = {'some': {'revision': 'WebKit'}};
562                 BuildbotSyncer._loadConfig(RemoteAPI, config);
563             }, /Build properties "some" specifies a non-string value of type "object"/);
564             assert.throws(() => {
565                 const config = smallConfiguration();
566                 config.builders[firstBuilder].properties = {'some': 1};
567                 BuildbotSyncer._loadConfig(RemoteAPI, config);
568             }, /Build properties "some" specifies a non-string value of type "object"/);
569         });
570
571         it('should create BuildbotSyncer objects for valid configurations', () => {
572             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
573             assert.equal(syncers.length, 3);
574             assert.ok(syncers[0] instanceof BuildbotSyncer);
575             assert.ok(syncers[1] instanceof BuildbotSyncer);
576             assert.ok(syncers[2] instanceof BuildbotSyncer);
577         });
578
579         it('should parse builder names correctly', () => {
580             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
581             assert.equal(syncers[0].builderName(), 'ABTest-iPhone-RunBenchmark-Tests');
582             assert.equal(syncers[1].builderName(), 'ABTest-iPad-RunBenchmark-Tests');
583             assert.equal(syncers[2].builderName(), 'ABTest-iOS-Builder');
584         });
585
586         it('should parse test configurations with build configurations correctly', () => {
587             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
588
589             let configurations = syncers[0].testConfigurations();
590             assert(syncers[0].isTester());
591             assert.equal(configurations.length, 3);
592             assert.equal(configurations[0].platform, MockModels.iphone);
593             assert.equal(configurations[0].test, MockModels.speedometer);
594             assert.equal(configurations[1].platform, MockModels.iphone);
595             assert.equal(configurations[1].test, MockModels.jetstream);
596             assert.equal(configurations[2].platform, MockModels.iphone);
597             assert.equal(configurations[2].test, MockModels.domcore);
598             assert.deepEqual(syncers[0].buildConfigurations(), []);
599
600             configurations = syncers[1].testConfigurations();
601             assert(syncers[1].isTester());
602             assert.equal(configurations.length, 2);
603             assert.equal(configurations[0].platform, MockModels.ipad);
604             assert.equal(configurations[0].test, MockModels.speedometer);
605             assert.equal(configurations[1].platform, MockModels.ipad);
606             assert.equal(configurations[1].test, MockModels.jetstream);
607             assert.deepEqual(syncers[1].buildConfigurations(), []);
608
609             assert(!syncers[2].isTester());
610             assert.deepEqual(syncers[2].testConfigurations(), []);
611             configurations = syncers[2].buildConfigurations();
612             assert.equal(configurations.length, 2);
613             assert.equal(configurations[0].platform, MockModels.iphone);
614             assert.equal(configurations[0].test, null);
615             assert.equal(configurations[1].platform, MockModels.ipad);
616             assert.equal(configurations[1].test, null);
617         });
618
619         it('should throw when a build configuration use the same builder as a test configuration', () => {
620             assert.throws(() => {
621                 const config = sampleiOSConfig();
622                 config.buildConfigurations[0].builders = config.testConfigurations[0].builders;
623                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
624             });
625         });
626
627         it('should parse test configurations with types and platforms expansions correctly', () => {
628             const syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfigWithExpansions());
629
630             assert.equal(syncers.length, 3);
631
632             let configurations = syncers[0].testConfigurations();
633             assert.equal(configurations.length, 4);
634             assert.equal(configurations[0].platform, MockModels.iphone);
635             assert.equal(configurations[0].test, MockModels.iPhonePLT);
636             assert.equal(configurations[1].platform, MockModels.iphone);
637             assert.equal(configurations[1].test, MockModels.speedometer);
638             assert.equal(configurations[2].platform, MockModels.iOS10iPhone);
639             assert.equal(configurations[2].test, MockModels.iPhonePLT);
640             assert.equal(configurations[3].platform, MockModels.iOS10iPhone);
641             assert.equal(configurations[3].test, MockModels.speedometer);
642             assert.deepEqual(syncers[0].buildConfigurations(), []);
643
644             configurations = syncers[1].testConfigurations();
645             assert.equal(configurations.length, 4);
646             assert.equal(configurations[0].platform, MockModels.iphone);
647             assert.equal(configurations[0].test, MockModels.iPhonePLT);
648             assert.equal(configurations[1].platform, MockModels.iphone);
649             assert.equal(configurations[1].test, MockModels.speedometer);
650             assert.equal(configurations[2].platform, MockModels.iOS10iPhone);
651             assert.equal(configurations[2].test, MockModels.iPhonePLT);
652             assert.equal(configurations[3].platform, MockModels.iOS10iPhone);
653             assert.equal(configurations[3].test, MockModels.speedometer);
654             assert.deepEqual(syncers[1].buildConfigurations(), []);
655
656             configurations = syncers[2].testConfigurations();
657             assert.equal(configurations.length, 2);
658             assert.equal(configurations[0].platform, MockModels.ipad);
659             assert.equal(configurations[0].test, MockModels.iPadPLT);
660             assert.equal(configurations[1].platform, MockModels.ipad);
661             assert.equal(configurations[1].test, MockModels.speedometer);
662             assert.deepEqual(syncers[2].buildConfigurations(), []);
663         });
664
665         it('should throw when repositoryGroups is not an object', () => {
666             assert.throws(() => {
667                 const config = smallConfiguration();
668                 config.repositoryGroups = 1;
669                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
670             }, /repositoryGroups must specify a dictionary from the name to its definition/);
671             assert.throws(() => {
672                 const config = smallConfiguration();
673                 config.repositoryGroups = 'hello';
674                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
675             }, /repositoryGroups must specify a dictionary from the name to its definition/);
676         });
677
678         it('should throw when a repository group does not specify a dictionary of repositories', () => {
679             assert.throws(() => {
680                 const config = smallConfiguration();
681                 config.repositoryGroups = {'some-group': {testProperties: {}}};
682                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
683             }, /Repository group "some-group" does not specify a dictionary of repositories/);
684             assert.throws(() => {
685                 const config = smallConfiguration();
686                 config.repositoryGroups = {'some-group': {repositories: 1}, testProperties: {}};
687                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
688             }, /Repository group "some-group" does not specify a dictionary of repositories/);
689         });
690
691         it('should throw when a repository group specifies an empty dictionary', () => {
692             assert.throws(() => {
693                 const config = smallConfiguration();
694                 config.repositoryGroups = {'some-group': {repositories: {}, testProperties: {}}};
695                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
696             }, /Repository group "some-group" does not specify any repository/);
697         });
698
699         it('should throw when a repository group specifies an invalid repository name', () => {
700             assert.throws(() => {
701                 const config = smallConfiguration();
702                 config.repositoryGroups = {'some-group': {repositories: {'InvalidRepositoryName': {}}}};
703                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
704             }, /"InvalidRepositoryName" is not a valid repository name/);
705         });
706
707         it('should throw when a repository group specifies a repository with a non-dictionary value', () => {
708             assert.throws(() => {
709                 const config = smallConfiguration();
710                 config.repositoryGroups = {'some-group': {repositories: {'WebKit': 1}}};
711                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
712             }, /"WebKit" specifies a non-dictionary value/);
713         });
714
715         it('should throw when the description of a repository group is not a string', () => {
716             assert.throws(() => {
717                 const config = smallConfiguration();
718                 config.repositoryGroups = {'some-group': {repositories: {'WebKit': {}}, description: 1}};
719                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
720             }, /Repository group "some-group" have an invalid description/);
721             assert.throws(() => {
722                 const config = smallConfiguration();
723                 config.repositoryGroups = {'some-group': {repositories: {'WebKit': {}}, description: [1, 2]}};
724                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
725             }, /Repository group "some-group" have an invalid description/);
726         });
727
728         it('should throw when a repository group does not specify a dictionary of properties', () => {
729             assert.throws(() => {
730                 const config = smallConfiguration();
731                 config.repositoryGroups = {'some-group': {repositories: {'WebKit': {}}, testProperties: 1}};
732                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
733             }, /Repository group "some-group" specifies the test configurations with an invalid type/);
734             assert.throws(() => {
735                 const config = smallConfiguration();
736                 config.repositoryGroups = {'some-group': {repositories: {'WebKit': {}}, testProperties: 'hello'}};
737                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
738             }, /Repository group "some-group" specifies the test configurations with an invalid type/);
739         });
740
741         it('should throw when a repository group refers to a non-existent repository in the properties dictionary', () => {
742             assert.throws(() => {
743                 const config = smallConfiguration();
744                 config.repositoryGroups = {'some-group': {repositories: {'WebKit': {}}, testProperties: {'wk': {revision: 'InvalidRepository'}}}};
745                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
746             }, /Repository group "some-group" an invalid repository "InvalidRepository"/);
747         });
748
749         it('should throw when a repository group refers to a repository which is not listed in the list of repositories', () => {
750             assert.throws(() => {
751                 const config = smallConfiguration();
752                 config.repositoryGroups = {'some-group': {repositories: {'WebKit': {}}, testProperties: {'os': {revision: 'iOS'}}}};
753                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
754             }, /Repository group "some-group" an invalid repository "iOS"/);
755             assert.throws(() => {
756                 const config = smallConfiguration();
757                 config.repositoryGroups = {'some-group': {
758                     repositories: {'WebKit': {acceptsPatch: true}},
759                     testProperties: {'wk': {revision: 'WebKit'}, 'install-roots': {'roots': {}}},
760                     buildProperties: {'os': {revision: 'iOS'}},
761                     acceptsRoots: true}};
762                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
763             }, /Repository group "some-group" an invalid repository "iOS"/);
764         });
765
766         it('should throw when a repository group refers to a repository in building a patch which does not accept a patch', () => {
767             assert.throws(() => {
768                 const config = smallConfiguration();
769                 config.repositoryGroups = {'some-group': {
770                     repositories: {'WebKit': {acceptsPatch: true}, 'iOS': {}},
771                     testProperties: {'wk': {revision: 'WebKit'}, 'ios': {revision: 'iOS'}, 'install-roots': {'roots': {}}},
772                     buildProperties: {'wk': {revision: 'WebKit'}, 'ios': {revision: 'iOS'}, 'wk-patch': {patch: 'iOS'}},
773                     acceptsRoots: true}};
774                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
775             }, /Repository group "some-group" specifies a patch for "iOS" but it does not accept a patch/);
776         });
777
778         it('should throw when a repository group specifies a patch without specifying a revision', () => {
779             assert.throws(() => {
780                 const config = smallConfiguration();
781                 config.repositoryGroups = {'some-group': {
782                     repositories: {'WebKit': {acceptsPatch: true}},
783                     testProperties: {'wk': {revision: 'WebKit'}, 'install-roots': {'roots': {}}},
784                     buildProperties: {'wk-patch': {patch: 'WebKit'}},
785                     acceptsRoots: true}};
786                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
787             }, /Repository group "some-group" specifies a patch for "WebKit" but does not specify a revision/);
788         });
789
790         it('should throw when a repository group does not use a listed repository', () => {
791             assert.throws(() => {
792                 const config = smallConfiguration();
793                 config.repositoryGroups = {'some-group': {'repositories': {'WebKit': {}}, testProperties: {}}};
794                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
795             }, /Repository group "some-group" does not use some of the repositories listed in testing/);
796             assert.throws(() => {
797                 const config = smallConfiguration();
798                 config.repositoryGroups = {'some-group': {
799                     repositories: {'WebKit': {acceptsPatch: true}},
800                     testProperties: {'wk': {revision: 'WebKit'}, 'install-roots': {'roots': {}}},
801                     buildProperties: {},
802                     acceptsRoots: true}};
803                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
804             }, /Repository group "some-group" does not use some of the repositories listed in building a patch/);
805         });
806
807         it('should throw when a repository group specifies non-boolean value to acceptsRoots', () => {
808             assert.throws(() => {
809                 const config = smallConfiguration();
810                 config.repositoryGroups = {'some-group': {'repositories': {'WebKit': {}}, 'testProperties': {'webkit': {'revision': 'WebKit'}}, acceptsRoots: 1}};
811                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
812             }, /Repository group "some-group" contains invalid acceptsRoots value:/);
813             assert.throws(() => {
814                 const config = smallConfiguration();
815                 config.repositoryGroups = {'some-group': {'repositories': {'WebKit': {}}, 'testProperties': {'webkit': {'revision': 'WebKit'}}, acceptsRoots: []}};
816                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
817             }, /Repository group "some-group" contains invalid acceptsRoots value:/);
818         });
819
820         it('should throw when a repository group specifies non-boolean value to acceptsPatch', () => {
821             assert.throws(() => {
822                 const config = smallConfiguration();
823                 config.repositoryGroups = {'some-group': {'repositories': {'WebKit': {acceptsPatch: 1}}, 'testProperties': {'webkit': {'revision': 'WebKit'}}}};
824                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
825             }, /"WebKit" contains invalid acceptsPatch value:/);
826             assert.throws(() => {
827                 const config = smallConfiguration();
828                 config.repositoryGroups = {'some-group': {'repositories': {'WebKit': {acceptsPatch: []}}, 'testProperties': {'webkit': {'revision': 'WebKit'}}}};
829                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
830             }, /"WebKit" contains invalid acceptsPatch value:/);
831         });
832
833         it('should throw when a repository group specifies a patch in testProperties', () => {
834             assert.throws(() => {
835                 const config = smallConfiguration();
836                 config.repositoryGroups = {'some-group': {'repositories': {'WebKit': {acceptsPatch: true}},
837                     'testProperties': {'webkit': {'revision': 'WebKit'}, 'webkit-patch': {'patch': 'WebKit'}}}};
838                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
839             }, /Repository group "some-group" specifies a patch for "WebKit" in the properties for testing/);
840         });
841
842         it('should throw when a repository group specifies roots in buildProperties', () => {
843             assert.throws(() => {
844                 const config = smallConfiguration();
845                 config.repositoryGroups = {'some-group': {
846                     repositories: {'WebKit': {acceptsPatch: true}},
847                     testProperties: {'webkit': {revision: 'WebKit'}, 'install-roots': {'roots': {}}},
848                     buildProperties: {'webkit': {revision: 'WebKit'}, 'patch': {patch: 'WebKit'}, 'install-roots': {roots: {}}},
849                     acceptsRoots: true}};
850                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
851             }, /Repository group "some-group" specifies roots in the properties for building/);
852         });
853
854         it('should throw when a repository group that does not accept roots specifies roots in testProperties', () => {
855             assert.throws(() => {
856                 const config = smallConfiguration();
857                 config.repositoryGroups = {'some-group': {
858                     repositories: {'WebKit': {}},
859                     testProperties: {'webkit': {'revision': 'WebKit'}, 'install-roots': {'roots': {}}}}};
860                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
861             }, /Repository group "some-group" specifies roots in a property but it does not accept roots/);
862         });
863
864         it('should throw when a repository group specifies buildProperties but does not accept roots', () => {
865             assert.throws(() => {
866                 const config = smallConfiguration();
867                 config.repositoryGroups = {'some-group': {
868                     repositories: {'WebKit': {acceptsPatch: true}},
869                     testProperties: {'webkit': {revision: 'WebKit'}},
870                     buildProperties: {'webkit': {revision: 'WebKit'}, 'webkit-patch': {patch: 'WebKit'}}}};
871                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
872             }, /Repository group "some-group" specifies the properties for building but does not accept roots in testing/);
873         });
874
875         it('should throw when a repository group specifies buildProperties but does not accept any patch', () => {
876             assert.throws(() => {
877                 const config = smallConfiguration();
878                 config.repositoryGroups = {'some-group': {
879                     repositories: {'WebKit': {}},
880                     testProperties: {'webkit': {'revision': 'WebKit'}, 'install-roots': {'roots': {}}},
881                     buildProperties: {'webkit': {'revision': 'WebKit'}},
882                     acceptsRoots: true}};
883                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
884             }, /Repository group "some-group" specifies the properties for building but does not accept any patches/);
885         });
886
887         it('should throw when a repository group accepts roots but does not specify roots in testProperties', () => {
888             assert.throws(() => {
889                 const config = smallConfiguration();
890                 config.repositoryGroups = {'some-group': {
891                     repositories: {'WebKit': {acceptsPatch: true}},
892                     testProperties: {'webkit': {revision: 'WebKit'}},
893                     buildProperties: {'webkit': {revision: 'WebKit'}, 'webkit-patch': {patch: 'WebKit'}},
894                     acceptsRoots: true}};
895                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
896             }, /Repository group "some-group" accepts roots but does not specify roots in testProperties/);
897         });
898     });
899
900     describe('_propertiesForBuildRequest', () => {
901         it('should include all properties specified in a given configuration', () => {
902             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
903             let properties = syncers[0]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
904             assert.deepEqual(Object.keys(properties).sort(), ['build_request_id', 'desired_image', 'forcescheduler', 'opensource', 'test_name']);
905         });
906
907         it('should preserve non-parametric property values', () => {
908             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
909             let properties = syncers[0]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
910             assert.equal(properties['test_name'], 'speedometer');
911             assert.equal(properties['forcescheduler'], 'ABTest-iPhone-RunBenchmark-Tests-ForceScheduler');
912
913             properties = syncers[1]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.ipad, MockModels.jetstream));
914             assert.equal(properties['test_name'], 'jetstream');
915             assert.equal(properties['forcescheduler'], 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler');
916         });
917
918         it('should resolve "root"', () => {
919             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
920             let properties = syncers[0]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
921             assert.equal(properties['desired_image'], '13A452');
922         });
923
924         it('should set the property for the build request id', () => {
925             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
926             let request = createSampleBuildRequest(MockModels.iphone, MockModels.speedometer);
927             let properties = syncers[0]._propertiesForBuildRequest(request);
928             assert.equal(properties['build_request_id'], request.id());
929         });
930     });
931
932     describe('pullBuildbot', () => {
933         it('should fetch pending builds from the right URL', () => {
934             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
935             assert.equal(syncer.builderName(), 'ABTest-iPad-RunBenchmark-Tests');
936             let expectedURL = '/json/builders/ABTest-iPad-RunBenchmark-Tests/pendingBuilds';
937             assert.equal(syncer.pathForPendingBuildsJSON(), expectedURL);
938             syncer.pullBuildbot();
939             assert.equal(requests.length, 1);
940             assert.equal(requests[0].url, expectedURL);
941         });
942
943         it('should fetch recent builds once pending builds have been fetched', () => {
944             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
945             assert.equal(syncer.builderName(), 'ABTest-iPad-RunBenchmark-Tests');
946
947             syncer.pullBuildbot(1);
948             assert.equal(requests.length, 1);
949             assert.equal(requests[0].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/pendingBuilds');
950             requests[0].resolve([]);
951             return MockRemoteAPI.waitForRequest().then(() => {
952                 assert.equal(requests.length, 2);
953                 assert.equal(requests[1].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/builds/?select=-1');
954             });
955         });
956
957         it('should fetch the right number of recent builds', () => {
958             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
959
960             syncer.pullBuildbot(3);
961             assert.equal(requests.length, 1);
962             assert.equal(requests[0].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/pendingBuilds');
963             requests[0].resolve([]);
964             return MockRemoteAPI.waitForRequest().then(() => {
965                 assert.equal(requests.length, 2);
966                 assert.equal(requests[1].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/builds/?select=-1&select=-2&select=-3');
967             });
968         });
969
970         it('should create BuildbotBuildEntry for pending builds', () => {
971             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
972             let promise = syncer.pullBuildbot();
973             requests[0].resolve([samplePendingBuild()]);
974             return promise.then((entries) => {
975                 assert.equal(entries.length, 1);
976                 let entry = entries[0];
977                 assert.ok(entry instanceof BuildbotBuildEntry);
978                 assert.ok(!entry.buildNumber());
979                 assert.ok(!entry.slaveName());
980                 assert.equal(entry.buildRequestId(), 16733);
981                 assert.ok(entry.isPending());
982                 assert.ok(!entry.isInProgress());
983                 assert.ok(!entry.hasFinished());
984                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
985             });
986         });
987
988         it('should create BuildbotBuildEntry for in-progress builds', () => {
989             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
990
991             let promise = syncer.pullBuildbot(1);
992             assert.equal(requests.length, 1);
993             requests[0].resolve([]);
994             return MockRemoteAPI.waitForRequest().then(() => {
995                 assert.equal(requests.length, 2);
996                 requests[1].resolve({[-1]: sampleInProgressBuild()});
997                 return promise;
998             }).then((entries) => {
999                 assert.equal(entries.length, 1);
1000                 let entry = entries[0];
1001                 assert.ok(entry instanceof BuildbotBuildEntry);
1002                 assert.equal(entry.buildNumber(), 614);
1003                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
1004                 assert.equal(entry.buildRequestId(), 16733);
1005                 assert.ok(!entry.isPending());
1006                 assert.ok(entry.isInProgress());
1007                 assert.ok(!entry.hasFinished());
1008                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
1009             });
1010         });
1011
1012         it('should create BuildbotBuildEntry for finished builds', () => {
1013             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1014
1015             let promise = syncer.pullBuildbot(1);
1016             assert.equal(requests.length, 1);
1017             requests[0].resolve([]);
1018             return MockRemoteAPI.waitForRequest().then(() => {
1019                 assert.equal(requests.length, 2);
1020                 requests[1].resolve({[-1]: sampleFinishedBuild()});
1021                 return promise;
1022             }).then((entries) => {
1023                 assert.deepEqual(entries.length, 1);
1024                 let entry = entries[0];
1025                 assert.ok(entry instanceof BuildbotBuildEntry);
1026                 assert.equal(entry.buildNumber(), 1755);
1027                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
1028                 assert.equal(entry.buildRequestId(), 18935);
1029                 assert.ok(!entry.isPending());
1030                 assert.ok(!entry.isInProgress());
1031                 assert.ok(entry.hasFinished());
1032                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
1033             });
1034         });
1035
1036         it('should create BuildbotBuildEntry for mixed pending, in-progress, finished, and missing builds', () => {
1037             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1038
1039             let promise = syncer.pullBuildbot(5);
1040             assert.equal(requests.length, 1);
1041
1042             requests[0].resolve([samplePendingBuild(123)]);
1043
1044             return MockRemoteAPI.waitForRequest().then(() => {
1045                 assert.equal(requests.length, 2);
1046                 requests[1].resolve({[-1]: sampleFinishedBuild(), [-2]: {'error': 'Not available'}, [-4]: sampleInProgressBuild()});
1047                 return promise;
1048             }).then((entries) => {
1049                 assert.deepEqual(entries.length, 3);
1050
1051                 let entry = entries[0];
1052                 assert.ok(entry instanceof BuildbotBuildEntry);
1053                 assert.equal(entry.buildNumber(), null);
1054                 assert.equal(entry.slaveName(), null);
1055                 assert.equal(entry.buildRequestId(), 123);
1056                 assert.ok(entry.isPending());
1057                 assert.ok(!entry.isInProgress());
1058                 assert.ok(!entry.hasFinished());
1059                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
1060
1061                 entry = entries[1];
1062                 assert.ok(entry instanceof BuildbotBuildEntry);
1063                 assert.equal(entry.buildNumber(), 614);
1064                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
1065                 assert.equal(entry.buildRequestId(), 16733);
1066                 assert.ok(!entry.isPending());
1067                 assert.ok(entry.isInProgress());
1068                 assert.ok(!entry.hasFinished());
1069                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
1070
1071                 entry = entries[2];
1072                 assert.ok(entry instanceof BuildbotBuildEntry);
1073                 assert.equal(entry.buildNumber(), 1755);
1074                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
1075                 assert.equal(entry.buildRequestId(), 18935);
1076                 assert.ok(!entry.isPending());
1077                 assert.ok(!entry.isInProgress());
1078                 assert.ok(entry.hasFinished());
1079                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
1080             });
1081         });
1082
1083         it('should sort BuildbotBuildEntry by order', () => {
1084             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1085
1086             let promise = syncer.pullBuildbot(5);
1087             assert.equal(requests.length, 1);
1088
1089             requests[0].resolve([samplePendingBuild(456, 2), samplePendingBuild(123, 1)]);
1090
1091             return MockRemoteAPI.waitForRequest().then(() => {
1092                 assert.equal(requests.length, 2);
1093                 requests[1].resolve({[-3]: sampleFinishedBuild(), [-1]: {'error': 'Not available'}, [-2]: sampleInProgressBuild()});
1094                 return promise;
1095             }).then((entries) => {
1096                 assert.deepEqual(entries.length, 4);
1097
1098                 let entry = entries[0];
1099                 assert.ok(entry instanceof BuildbotBuildEntry);
1100                 assert.equal(entry.buildNumber(), null);
1101                 assert.equal(entry.slaveName(), null);
1102                 assert.equal(entry.buildRequestId(), 123);
1103                 assert.ok(entry.isPending());
1104                 assert.ok(!entry.isInProgress());
1105                 assert.ok(!entry.hasFinished());
1106                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
1107
1108                 entry = entries[1];
1109                 assert.ok(entry instanceof BuildbotBuildEntry);
1110                 assert.equal(entry.buildNumber(), null);
1111                 assert.equal(entry.slaveName(), null);
1112                 assert.equal(entry.buildRequestId(), 456);
1113                 assert.ok(entry.isPending());
1114                 assert.ok(!entry.isInProgress());
1115                 assert.ok(!entry.hasFinished());
1116                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
1117
1118                 entry = entries[2];
1119                 assert.ok(entry instanceof BuildbotBuildEntry);
1120                 assert.equal(entry.buildNumber(), 614);
1121                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
1122                 assert.equal(entry.buildRequestId(), 16733);
1123                 assert.ok(!entry.isPending());
1124                 assert.ok(entry.isInProgress());
1125                 assert.ok(!entry.hasFinished());
1126                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
1127
1128                 entry = entries[3];
1129                 assert.ok(entry instanceof BuildbotBuildEntry);
1130                 assert.equal(entry.buildNumber(), 1755);
1131                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
1132                 assert.equal(entry.buildRequestId(), 18935);
1133                 assert.ok(!entry.isPending());
1134                 assert.ok(!entry.isInProgress());
1135                 assert.ok(entry.hasFinished());
1136                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
1137             });
1138         });
1139
1140         it('should override BuildbotBuildEntry for pending builds by in-progress builds', () => {
1141             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1142
1143             let promise = syncer.pullBuildbot(5);
1144             assert.equal(requests.length, 1);
1145
1146             requests[0].resolve([samplePendingBuild()]);
1147
1148             return MockRemoteAPI.waitForRequest().then(() => {
1149                 assert.equal(requests.length, 2);
1150                 requests[1].resolve({[-1]: sampleInProgressBuild()});
1151                 return promise;
1152             }).then((entries) => {
1153                 assert.equal(entries.length, 1);
1154
1155                 let entry = entries[0];
1156                 assert.ok(entry instanceof BuildbotBuildEntry);
1157                 assert.equal(entry.buildNumber(), 614);
1158                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
1159                 assert.equal(entry.buildRequestId(), 16733);
1160                 assert.ok(!entry.isPending());
1161                 assert.ok(entry.isInProgress());
1162                 assert.ok(!entry.hasFinished());
1163                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
1164             });
1165         });
1166
1167         it('should override BuildbotBuildEntry for pending builds by finished builds', () => {
1168             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1169
1170             let promise = syncer.pullBuildbot(5);
1171             assert.equal(requests.length, 1);
1172
1173             requests[0].resolve([samplePendingBuild()]);
1174
1175             return MockRemoteAPI.waitForRequest().then(() => {
1176                 assert.equal(requests.length, 2);
1177                 requests[1].resolve({[-1]: sampleFinishedBuild(16733)});
1178                 return promise;
1179             }).then((entries) => {
1180                 assert.equal(entries.length, 1);
1181
1182                 let entry = entries[0];
1183                 assert.ok(entry instanceof BuildbotBuildEntry);
1184                 assert.equal(entry.buildNumber(), 1755);
1185                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
1186                 assert.equal(entry.buildRequestId(), 16733);
1187                 assert.ok(!entry.isPending());
1188                 assert.ok(!entry.isInProgress());
1189                 assert.ok(entry.hasFinished());
1190                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
1191             });
1192         });
1193     });
1194
1195     describe('scheduleRequest', () => {
1196         it('should schedule a build request on a specified slave', () => {
1197             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[0];
1198
1199             const waitForRequest = MockRemoteAPI.waitForRequest();
1200             syncer.scheduleRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer), 'some-slave');
1201             return waitForRequest.then(() => {
1202                 assert.equal(requests.length, 1);
1203                 assert.equal(requests[0].url, '/builders/ABTest-iPhone-RunBenchmark-Tests/force');
1204                 assert.equal(requests[0].method, 'POST');
1205                 assert.deepEqual(requests[0].data, {
1206                     'build_request_id': '16733-' + MockModels.iphone.id(),
1207                     'desired_image': '13A452',
1208                     "opensource": "197463",
1209                     'forcescheduler': 'ABTest-iPhone-RunBenchmark-Tests-ForceScheduler',
1210                     'slavename': 'some-slave',
1211                     'test_name': 'speedometer'
1212                 });
1213             });
1214         });
1215     });
1216
1217     describe('scheduleRequestInGroupIfAvailable', () => {
1218
1219         function pullBuildbotWithAssertion(syncer, pendingBuilds, inProgressAndFinishedBuilds)
1220         {
1221             const promise = syncer.pullBuildbot(5);
1222             assert.equal(requests.length, 1);
1223             requests[0].resolve(pendingBuilds);
1224             return MockRemoteAPI.waitForRequest().then(() => {
1225                 assert.equal(requests.length, 2);
1226                 requests[1].resolve(inProgressAndFinishedBuilds);
1227                 requests.length = 0;
1228                 return promise;
1229             });
1230         }
1231
1232         it('should schedule a build if builder has no builds if slaveList is not specified', () => {
1233             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, smallConfiguration())[0];
1234
1235             return pullBuildbotWithAssertion(syncer, [], {}).then(() => {
1236                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
1237                 assert.equal(requests.length, 1);
1238                 assert.equal(requests[0].url, '/builders/some%20builder/force');
1239                 assert.equal(requests[0].method, 'POST');
1240                 assert.deepEqual(requests[0].data, {id: '16733-' + MockModels.somePlatform.id(), 'os': '13A452', 'wk': '197463'});
1241             });
1242         });
1243
1244         it('should schedule a build if builder only has finished builds if slaveList is not specified', () => {
1245             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, smallConfiguration())[0];
1246
1247             return pullBuildbotWithAssertion(syncer, [], {[-1]: smallFinishedBuild()}).then(() => {
1248                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
1249                 assert.equal(requests.length, 1);
1250                 assert.equal(requests[0].url, '/builders/some%20builder/force');
1251                 assert.equal(requests[0].method, 'POST');
1252                 assert.deepEqual(requests[0].data, {id: '16733-' + MockModels.somePlatform.id(), 'os': '13A452', 'wk': '197463'});
1253             });
1254         });
1255
1256         it('should not schedule a build if builder has a pending build if slaveList is not specified', () => {
1257             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, smallConfiguration())[0];
1258
1259             return pullBuildbotWithAssertion(syncer, [smallPendingBuild()], {}).then(() => {
1260                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
1261                 assert.equal(requests.length, 0);
1262             });
1263         });
1264
1265         it('should schedule a build if builder does not have pending or completed builds on the matching slave', () => {
1266             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[0];
1267
1268             return pullBuildbotWithAssertion(syncer, [], {}).then(() => {
1269                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
1270                 assert.equal(requests.length, 1);
1271                 assert.equal(requests[0].url, '/builders/ABTest-iPhone-RunBenchmark-Tests/force');
1272                 assert.equal(requests[0].method, 'POST');
1273             });
1274         });
1275
1276         it('should schedule a build if builder only has finished builds on the matching slave', () => {
1277             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1278
1279             pullBuildbotWithAssertion(syncer, [], {[-1]: sampleFinishedBuild()}).then(() => {
1280                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1281                 assert.equal(requests.length, 1);
1282                 assert.equal(requests[0].url, '/builders/ABTest-iPad-RunBenchmark-Tests/force');
1283                 assert.equal(requests[0].method, 'POST');
1284             });
1285         });
1286
1287         it('should not schedule a build if builder has a pending build on the maching slave', () => {
1288             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1289
1290             pullBuildbotWithAssertion(syncer, [samplePendingBuild()], {}).then(() => {
1291                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1292                 assert.equal(requests.length, 0);
1293             });
1294         });
1295
1296         it('should schedule a build if builder only has a pending build on a non-maching slave', () => {
1297             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1298
1299             return pullBuildbotWithAssertion(syncer, [samplePendingBuild(1, 1, 'another-slave')], {}).then(() => {
1300                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1301                 assert.equal(requests.length, 1);
1302             });
1303         });
1304
1305         it('should schedule a build if builder only has an in-progress build on the matching slave', () => {
1306             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1307
1308             return pullBuildbotWithAssertion(syncer, [], {[-1]: sampleInProgressBuild()}).then(() => {
1309                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1310                 assert.equal(requests.length, 1);
1311             });
1312         });
1313
1314         it('should schedule a build if builder has an in-progress build on another slave', () => {
1315             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1316
1317             return pullBuildbotWithAssertion(syncer, [], {[-1]: sampleInProgressBuild('other-slave')}).then(() => {
1318                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1319                 assert.equal(requests.length, 1);
1320             });
1321         });
1322
1323         it('should not schedule a build if the request does not match any configuration', () => {
1324             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[0];
1325
1326             return pullBuildbotWithAssertion(syncer, [], {}).then(() => {
1327                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1328                 assert.equal(requests.length, 0);
1329             });
1330         });
1331
1332         it('should not schedule a build if a new request had been submitted to the same slave', (done) => {
1333             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1334
1335             pullBuildbotWithAssertion(syncer, [], {}).then(() => {
1336                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-0');
1337                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-1');
1338             }).then(() => {
1339                 assert.equal(requests.length, 2);
1340                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1341             }).then(() => {
1342                 assert.equal(requests.length, 2);
1343                 done();
1344             }).catch(done);
1345         });
1346
1347         it('should schedule a build if a new request had been submitted to another slave', () => {
1348             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1349
1350             return pullBuildbotWithAssertion(syncer, [], {}).then(() => {
1351                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-0');
1352                 assert.equal(requests.length, 1);
1353                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-1');
1354                 assert.equal(requests.length, 2);
1355             });
1356         });
1357
1358         it('should not schedule a build if a new request had been submitted to the same builder without slaveList', () => {
1359             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, smallConfiguration())[0];
1360
1361             return pullBuildbotWithAssertion(syncer, [], {}).then(() => {
1362                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest), null);
1363                 assert.equal(requests.length, 1);
1364                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
1365                 assert.equal(requests.length, 1);
1366             });
1367         });
1368     });
1369 });