Add the UI for scheduling a A/B testing with a custom root
[WebKit-https.git] / Websites / perf.webkit.org / unit-tests / buildbot-syncer-tests.js
1 'use strict';
2
3 let assert = require('assert');
4
5 require('../tools/js/v3-models.js');
6 let MockRemoteAPI = require('./resources/mock-remote-api.js').MockRemoteAPI;
7 let MockModels = require('./resources/mock-v3-models.js').MockModels;
8
9 let BuildbotBuildEntry = require('../tools/js/buildbot-syncer.js').BuildbotBuildEntry;
10 let BuildbotSyncer = require('../tools/js/buildbot-syncer.js').BuildbotSyncer;
11
12 function sampleiOSConfig()
13 {
14     return {
15         'slaveArgument': 'slavename',
16         'buildRequestArgument': 'build_request_id',
17         'repositoryGroups': {
18             'ios-svn-webkit': {
19                 'repositories': ['WebKit', 'iOS'],
20                 'properties': {
21                     'desired_image': '<iOS>',
22                     'opensource': '<WebKit>',
23                 }
24             }
25         },
26         'types': {
27             'speedometer': {
28                 'test': ['Speedometer'],
29                 'arguments': {'test_name': 'speedometer'}
30             },
31             'jetstream': {
32                 'test': ['JetStream'],
33                 'arguments': {'test_name': 'jetstream'}
34             },
35             'dromaeo-dom': {
36                 'test': ['Dromaeo', 'DOM Core Tests'],
37                 'arguments': {'tests': 'dromaeo-dom'}
38             },
39         },
40         'builders': {
41             'iPhone-bench': {
42                 'builder': 'ABTest-iPhone-RunBenchmark-Tests',
43                 'arguments': { 'forcescheduler': 'ABTest-iPhone-RunBenchmark-Tests-ForceScheduler' },
44                 'slaveList': ['ABTest-iPhone-0'],
45             },
46             'iPad-bench': {
47                 'builder': 'ABTest-iPad-RunBenchmark-Tests',
48                 'arguments': { 'forcescheduler': 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler' },
49                 'slaveList': ['ABTest-iPad-0', 'ABTest-iPad-1'],
50             }
51         },
52         'configurations': [
53             {'type': 'speedometer', 'builder': 'iPhone-bench', 'platform': 'iPhone'},
54             {'type': 'jetstream', 'builder': 'iPhone-bench', 'platform': 'iPhone'},
55             {'type': 'dromaeo-dom', 'builder': 'iPhone-bench', 'platform': 'iPhone'},
56
57             {'type': 'speedometer', 'builder': 'iPad-bench', 'platform': 'iPad'},
58             {'type': 'jetstream', 'builder': 'iPad-bench', 'platform': 'iPad'},
59         ]
60     };
61 }
62
63 function sampleiOSConfigWithExpansions()
64 {
65     return {
66         "triggerableName": "build-webkit-ios",
67         "buildRequestArgument": "build-request-id",
68         "repositoryGroups": { },
69         "types": {
70             "iphone-plt": {
71                 "test": ["PLT-iPhone"],
72                 "arguments": {"test_name": "plt"}
73             },
74             "ipad-plt": {
75                 "test": ["PLT-iPad"],
76                 "arguments": {"test_name": "plt"}
77             },
78             "speedometer": {
79                 "test": ["Speedometer"],
80                 "arguments": {"tests": "speedometer"}
81             },
82         },
83         "builders": {
84             "iphone": {
85                 "builder": "iPhone AB Tests",
86                 "arguments": {"forcescheduler": "force-iphone-ab-tests"}
87             },
88             "ipad": {
89                 "builder": "iPad AB Tests",
90                 "arguments": {"forcescheduler": "force-ipad-ab-tests"}
91             },
92         },
93         "configurations": [
94             {
95                 "builder": "iphone",
96                 "platforms": ["iPhone", "iOS 10 iPhone"],
97                 "types": ["iphone-plt", "speedometer"],
98             },
99             {
100                 "builder": "ipad",
101                 "platforms": ["iPad"],
102                 "types": ["ipad-plt", "speedometer"],
103             },
104         ]
105     }
106 }
107
108 function smallConfiguration()
109 {
110     return {
111         'buildRequestArgument': 'id',
112         'repositoryGroups': {
113             'ios-svn-webkit': {
114                 'repositories': ['iOS', 'WebKit'],
115                 'properties': {
116                     'os': '<iOS>',
117                     'wk': '<WebKit>'
118                 }
119             }
120         },
121         'configurations': [{
122             'builder': 'some builder',
123             'platform': 'Some platform',
124             'test': ['Some test']
125         }]
126     };
127 }
128
129 function smallPendingBuild()
130 {
131     return {
132         'builderName': 'some builder',
133         'builds': [],
134         'properties': [],
135         'source': {
136             'branch': '',
137             'changes': [],
138             'codebase': 'WebKit',
139             'hasPatch': false,
140             'project': '',
141             'repository': '',
142             'revision': ''
143         },
144     };
145 }
146
147 function smallInProgressBuild()
148 {
149     return {
150         'builderName': 'some builder',
151         'builds': [],
152         'properties': [],
153         'currentStep': { },
154         'eta': 123,
155         'number': 456,
156         'source': {
157             'branch': '',
158             'changes': [],
159             'codebase': 'WebKit',
160             'hasPatch': false,
161             'project': '',
162             'repository': '',
163             'revision': ''
164         },
165     };
166 }
167
168 function smallFinishedBuild()
169 {
170     return {
171         'builderName': 'some builder',
172         'builds': [],
173         'properties': [],
174         'currentStep': null,
175         'eta': null,
176         'number': 789,
177         'source': {
178             'branch': '',
179             'changes': [],
180             'codebase': 'WebKit',
181             'hasPatch': false,
182             'project': '',
183             'repository': '',
184             'revision': ''
185         },
186         'times': [0, 1],
187     };
188 }
189
190 function createSampleBuildRequest(platform, test)
191 {
192     assert(platform instanceof Platform);
193     assert(test instanceof Test);
194
195     let commitSet = CommitSet.ensureSingleton('4197', {customRoots: [], commits: [
196         {'id': '111127', 'time': 1456955807334, 'repository': MockModels.webkit, 'revision': '197463'},
197         {'id': '111237', 'time': 1456931874000, 'repository': MockModels.sharedRepository, 'revision': '80229'},
198         {'id': '88930', 'time': 0, 'repository': MockModels.ios, 'revision': '13A452'},
199     ]});
200
201     return BuildRequest.ensureSingleton('16733-' + platform.id(), {'triggerable': MockModels.triggerable,
202         repositoryGroup: MockModels.svnRepositoryGroup,
203         'commitSet': commitSet, 'status': 'pending', 'platform': platform, 'test': test});
204 }
205
206 function samplePendingBuild(buildRequestId, buildTime, slaveName)
207 {
208     return {
209         'builderName': 'ABTest-iPad-RunBenchmark-Tests',
210         'builds': [],
211         'properties': [
212             ['build_request_id', buildRequestId || '16733', 'Force Build Form'],
213             ['desired_image', '13A452', 'Force Build Form'],
214             ['owner', '<unknown>', 'Force Build Form'],
215             ['test_name', 'speedometer', 'Force Build Form'],
216             ['reason', 'force build','Force Build Form'],
217             ['slavename', slaveName, ''],
218             ['scheduler', 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler', 'Scheduler']
219         ],
220         'source': {
221             'branch': '',
222             'changes': [],
223             'codebase': 'compiler-rt',
224             'hasPatch': false,
225             'project': '',
226             'repository': '',
227             'revision': ''
228         },
229         'submittedAt': buildTime || 1458704983
230     };
231 }
232
233 function sampleInProgressBuild(slaveName)
234 {
235     return {
236         'blame': [],
237         'builderName': 'ABTest-iPad-RunBenchmark-Tests',
238         'currentStep': {
239             'eta': 0.26548067698460565,
240             'expectations': [['output', 845, 1315.0]],
241             'hidden': false,
242             'isFinished': false,
243             'isStarted': true,
244             'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/Some%20step/logs/stdio']],
245             'name': 'Some step',
246             'results': [null,[]],
247             'statistics': {},
248             'step_number': 1,
249             'text': [''],
250             'times': [1458718657.581628, null],
251             'urls': {}
252         },
253         'eta': 6497.991612434387,
254         'logs': [['stdio','https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/shell/logs/stdio']],
255         'number': 614,
256         'properties': [
257             ['build_request_id', '16733', 'Force Build Form'],
258             ['buildername', 'ABTest-iPad-RunBenchmark-Tests', 'Builder'],
259             ['buildnumber', 614, 'Build'],
260             ['desired_image', '13A452', 'Force Build Form'],
261             ['owner', '<unknown>', 'Force Build Form'],
262             ['reason', 'force build', 'Force Build Form'],
263             ['scheduler', 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler', 'Scheduler'],
264             ['slavename', slaveName || 'ABTest-iPad-0', 'BuildSlave'],
265         ],
266         'reason': 'A build was forced by \'<unknown>\': force build',
267         'results': null,
268         'slave': 'ABTest-iPad-0',
269         'sourceStamps': [{'branch': '', 'changes': [], 'codebase': 'compiler-rt', 'hasPatch': false, 'project': '', 'repository': '', 'revision': ''}],
270         'steps': [
271             {
272                 'eta': null,
273                 'expectations': [['output',2309,2309.0]],
274                 'hidden': false,
275                 'isFinished': true,
276                 'isStarted': true,
277                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/shell/logs/stdio']],
278                 'name': 'Finished step',
279                 'results': [0, []],
280                 'statistics': {},
281                 'step_number': 0,
282                 'text': [''],
283                 'times': [1458718655.419865, 1458718655.453633],
284                 'urls': {}
285             },
286             {
287                 'eta': 0.26548067698460565,
288                 'expectations': [['output', 845, 1315.0]],
289                 'hidden': false,
290                 'isFinished': false,
291                 'isStarted': true,
292                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/Some%20step/logs/stdio']],
293                 'name': 'Some step',
294                 'results': [null,[]],
295                 'statistics': {},
296                 'step_number': 1,
297                 'text': [''],
298                 'times': [1458718657.581628, null],
299                 'urls': {}
300             },
301             {
302                 'eta': null,
303                 'expectations': [['output', null, null]],
304                 'hidden': false,
305                 'isFinished': false,
306                 'isStarted': false,
307                 'logs': [],
308                 'name': 'Some other step',
309                 'results': [null, []],
310                 'statistics': {},
311                 'step_number': 2,
312                 'text': [],
313                 'times': [null, null],
314                 'urls': {}
315             },
316         ],
317         'text': [],
318         'times': [1458718655.415821, null]
319     };
320 }
321
322 function sampleFinishedBuild(buildRequestId, slaveName)
323 {
324     return {
325         'blame': [],
326         'builderName': 'ABTest-iPad-RunBenchmark-Tests',
327         'currentStep': null,
328         'eta': null,
329         'logs': [['stdio','https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755/steps/shell/logs/stdio']],
330         'number': 1755,
331         'properties': [
332             ['build_request_id', buildRequestId || '18935', 'Force Build Form'],
333             ['buildername', 'ABTest-iPad-RunBenchmark-Tests', 'Builder'],
334             ['buildnumber', 1755, 'Build'],
335             ['desired_image', '13A452', 'Force Build Form'],
336             ['owner', '<unknown>', 'Force Build Form'],
337             ['reason', 'force build', 'Force Build Form'],
338             ['scheduler', 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler', 'Scheduler'],
339             ['slavename', slaveName || 'ABTest-iPad-0', 'BuildSlave'],
340         ],
341         'reason': 'A build was forced by \'<unknown>\': force build',
342         'results': 2,
343         'slave': 'ABTest-iPad-0',
344         'sourceStamps': [{'branch': '', 'changes': [], 'codebase': 'compiler-rt', 'hasPatch': false, 'project': '', 'repository': '', 'revision': ''}],
345         'steps': [
346             {
347                 'eta': null,
348                 'expectations': [['output',2309,2309.0]],
349                 'hidden': false,
350                 'isFinished': true,
351                 'isStarted': true,
352                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/shell/logs/stdio']],
353                 'name': 'Finished step',
354                 'results': [0, []],
355                 'statistics': {},
356                 'step_number': 0,
357                 'text': [''],
358                 'times': [1458718655.419865, 1458718655.453633],
359                 'urls': {}
360             },
361             {
362                 'eta': null,
363                 'expectations': [['output', 845, 1315.0]],
364                 'hidden': false,
365                 'isFinished': true,
366                 'isStarted': true,
367                 'logs': [['stdio', 'https://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614/steps/Some%20step/logs/stdio']],
368                 'name': 'Some step',
369                 'results': [null,[]],
370                 'statistics': {},
371                 'step_number': 1,
372                 'text': [''],
373                 'times': [1458718657.581628, null],
374                 'urls': {}
375             },
376             {
377                 'eta': null,
378                 'expectations': [['output', null, null]],
379                 'hidden': false,
380                 'isFinished': true,
381                 'isStarted': true,
382                 'logs': [],
383                 'name': 'Some other step',
384                 'results': [null, []],
385                 'statistics': {},
386                 'step_number': 2,
387                 'text': [],
388                 'times': [null, null],
389                 'urls': {}
390             },
391         ],
392         'text': [],
393         'times': [1458937478.25837, 1458946147.173785]
394     };
395 }
396
397 describe('BuildbotSyncer', () => {
398     MockModels.inject();
399     let requests = MockRemoteAPI.inject('http://build.webkit.org');
400
401     describe('_loadConfig', () => {
402
403         it('should create BuildbotSyncer objects for a configuration that specify all required options', () => {
404             assert.equal(BuildbotSyncer._loadConfig(MockRemoteAPI, smallConfiguration()).length, 1);
405         });
406
407         it('should throw when some required options are missing', () => {
408             assert.throws(() => {
409                 const config = smallConfiguration();
410                 delete config.configurations[0].builder;
411                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
412             }, 'builder should be a required option');
413             assert.throws(() => {
414                 const config = smallConfiguration();
415                 delete config.configurations[0].platform;
416                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
417             }, 'platform should be a required option');
418             assert.throws(() => {
419                 const config = smallConfiguration();
420                 delete config.configurations[0].test;
421                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
422             }, 'test should be a required option');
423             assert.throws(() => {
424                 const config = smallConfiguration();
425                 delete config.buildRequestArgument;
426                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
427             }, 'buildRequestArgument should be required');
428         });
429
430         it('should throw when a test name is not an array of strings', () => {
431             assert.throws(() => {
432                 const config = smallConfiguration();
433                 config.configurations[0].test = 'some test';
434                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
435             });
436             assert.throws(() => {
437                 const config = smallConfiguration();
438                 config.configurations[0].test = [1];
439                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
440             });
441         });
442
443         it('should throw when arguments is not an object', () => {
444             assert.throws(() => {
445                 const config = smallConfiguration();
446                 config.configurations[0].arguments = 'hello';
447                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
448             });
449         });
450
451         it('should throw when arguments\'s values are malformed', () => {
452             assert.throws(() => {
453                 const config = smallConfiguration();
454                 config.configurations[0].arguments = {'some': {'otherKey': 'some root'}};
455                 BuildbotSyncer._loadConfig(RemoteAPI, config);
456             });
457             assert.throws(() => {
458                 const config = smallConfiguration();
459                 config.configurations[0].arguments = {'some': {'root': ['a', 'b']}};
460                 BuildbotSyncer._loadConfig(RemoteAPI, config);
461             });
462             assert.throws(() => {
463                 const config = smallConfiguration();
464                 config.configurations[0].arguments = {'some': {'root': 1}};
465                 BuildbotSyncer._loadConfig(RemoteAPI, config);
466             });
467         });
468
469         it('should create BuildbotSyncer objects for valid configurations', () => {
470             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
471             assert.equal(syncers.length, 2);
472             assert.ok(syncers[0] instanceof BuildbotSyncer);
473             assert.ok(syncers[1] instanceof BuildbotSyncer);
474         });
475
476         it('should parse builder names correctly', () => {
477             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
478             assert.equal(syncers[0].builderName(), 'ABTest-iPhone-RunBenchmark-Tests');
479             assert.equal(syncers[1].builderName(), 'ABTest-iPad-RunBenchmark-Tests');
480         });
481
482         it('should parse test configurations correctly', () => {
483             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
484
485             let configurations = syncers[0].testConfigurations();
486             assert.equal(configurations.length, 3);
487             assert.equal(configurations[0].platform, MockModels.iphone);
488             assert.equal(configurations[0].test, MockModels.speedometer);
489             assert.equal(configurations[1].platform, MockModels.iphone);
490             assert.equal(configurations[1].test, MockModels.jetstream);
491             assert.equal(configurations[2].platform, MockModels.iphone);
492             assert.equal(configurations[2].test, MockModels.domcore);
493
494             configurations = syncers[1].testConfigurations();
495             assert.equal(configurations.length, 2);
496             assert.equal(configurations[0].platform, MockModels.ipad);
497             assert.equal(configurations[0].test, MockModels.speedometer);
498             assert.equal(configurations[1].platform, MockModels.ipad);
499             assert.equal(configurations[1].test, MockModels.jetstream);
500         });
501
502         it('should parse test configurations with types and platforms expansions correctly', () => {
503             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfigWithExpansions());
504
505             assert.equal(syncers.length, 2);
506
507             let configurations = syncers[0].testConfigurations();
508             assert.equal(configurations.length, 4);
509             assert.equal(configurations[0].platform, MockModels.iphone);
510             assert.equal(configurations[0].test, MockModels.iPhonePLT);
511             assert.equal(configurations[1].platform, MockModels.iOS10iPhone);
512             assert.equal(configurations[1].test, MockModels.iPhonePLT);
513             assert.equal(configurations[2].platform, MockModels.iphone);
514             assert.equal(configurations[2].test, MockModels.speedometer);
515             assert.equal(configurations[3].platform, MockModels.iOS10iPhone);
516             assert.equal(configurations[3].test, MockModels.speedometer);
517
518             configurations = syncers[1].testConfigurations();
519             assert.equal(configurations.length, 2);
520             assert.equal(configurations[0].platform, MockModels.ipad);
521             assert.equal(configurations[0].test, MockModels.iPadPLT);
522             assert.equal(configurations[1].platform, MockModels.ipad);
523             assert.equal(configurations[1].test, MockModels.speedometer);
524         });
525
526         it('should throw when repositoryGroups is not an object', () => {
527             assert.throws(() => {
528                 const config = smallConfiguration();
529                 config.repositoryGroups = 1;
530                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
531             });
532             assert.throws(() => {
533                 const config = smallConfiguration();
534                 config.repositoryGroups = 'hello';
535                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
536             });
537         });
538
539         it('should throw when a repository group does not specify a list of repository', () => {
540             assert.throws(() => {
541                 const config = smallConfiguration();
542                 config.repositoryGroups = {'some-group': {}};
543                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
544             });
545             assert.throws(() => {
546                 const config = smallConfiguration();
547                 config.repositoryGroups = {'some-group': {'repositories': 1}};
548                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
549             });
550         });
551
552         it('should throw when a repository group specifies an empty list of repository', () => {
553             assert.throws(() => {
554                 const config = smallConfiguration();
555                 config.repositoryGroups = {'some-group': {'repositories': []}};
556                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
557             });
558         });
559
560         it('should throw when a repository group specifies a valid repository', () => {
561             assert.throws(() => {
562                 const config = smallConfiguration();
563                 config.repositoryGroups = {'some-group': {'repositories': ['InvalidRepositoryName']}};
564                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
565             });
566         });
567
568         it('should throw when the description of a repository group is not a string', () => {
569             assert.throws(() => {
570                 const config = smallConfiguration();
571                 config.repositoryGroups = {'some-group': {'repositories': ['WebKit'], 'description': 1}};
572                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
573             });
574             assert.throws(() => {
575                 const config = smallConfiguration();
576                 config.repositoryGroups = {'some-group': {'repositories': ['WebKit'], 'description': [1, 2]}};
577                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
578             });
579         });
580
581         it('should throw when a repository group does not specify a dictionary of properties', () => {
582             assert.throws(() => {
583                 const config = smallConfiguration();
584                 config.repositoryGroups = {'some-group': {'repositories': ['WebKit'], properties: 1}};
585                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
586             });
587             assert.throws(() => {
588                 const config = smallConfiguration();
589                 config.repositoryGroups = {'some-group': {'repositories': ['WebKit'], properties: 'hello'}};
590                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
591             });
592         });
593
594         it('should throw when a repository group refers to a non-existent repository in the properties dictionary', () => {
595             assert.throws(() => {
596                 const config = smallConfiguration();
597                 config.repositoryGroups = {'some-group': {'repositories': ['WebKit'], properties: {'wk': '<InvalidRepository>'}}};
598                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
599             });
600         });
601
602         it('should throw when a repository group refers to a repository in the properties dictionary which is not listed in the list of repositories', () => {
603             assert.throws(() => {
604                 const config = smallConfiguration();
605                 config.repositoryGroups = {'some-group': {'repositories': ['WebKit'], properties: {'os': '<iOS>'}}};
606                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
607             });
608         });
609
610         it('should throw when a repository group does not use a lited repository', () => {
611             assert.throws(() => {
612                 const config = smallConfiguration();
613                 config.repositoryGroups = {'some-group': {'repositories': ['WebKit'], properties: {}}};
614                 BuildbotSyncer._loadConfig(MockRemoteAPI, config);
615             });
616         });
617     });
618
619     describe('_propertiesForBuildRequest', () => {
620         it('should include all properties specified in a given configuration', () => {
621             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
622             let properties = syncers[0]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
623             assert.deepEqual(Object.keys(properties).sort(), ['build_request_id', 'desired_image', 'forcescheduler', 'opensource', 'test_name']);
624         });
625
626         it('should preserve non-parametric property values', () => {
627             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
628             let properties = syncers[0]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
629             assert.equal(properties['test_name'], 'speedometer');
630             assert.equal(properties['forcescheduler'], 'ABTest-iPhone-RunBenchmark-Tests-ForceScheduler');
631
632             properties = syncers[1]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.ipad, MockModels.jetstream));
633             assert.equal(properties['test_name'], 'jetstream');
634             assert.equal(properties['forcescheduler'], 'ABTest-iPad-RunBenchmark-Tests-ForceScheduler');
635         });
636
637         it('should resolve "root"', () => {
638             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
639             let properties = syncers[0]._propertiesForBuildRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
640             assert.equal(properties['desired_image'], '13A452');
641         });
642
643         it('should set the property for the build request id', () => {
644             let syncers = BuildbotSyncer._loadConfig(RemoteAPI, sampleiOSConfig());
645             let request = createSampleBuildRequest(MockModels.iphone, MockModels.speedometer);
646             let properties = syncers[0]._propertiesForBuildRequest(request);
647             assert.equal(properties['build_request_id'], request.id());
648         });
649     });
650
651     describe('pullBuildbot', () => {
652         it('should fetch pending builds from the right URL', () => {
653             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
654             assert.equal(syncer.builderName(), 'ABTest-iPad-RunBenchmark-Tests');
655             let expectedURL = '/json/builders/ABTest-iPad-RunBenchmark-Tests/pendingBuilds';
656             assert.equal(syncer.pathForPendingBuildsJSON(), expectedURL);
657             syncer.pullBuildbot();
658             assert.equal(requests.length, 1);
659             assert.equal(requests[0].url, expectedURL);
660         });
661
662         it('should fetch recent builds once pending builds have been fetched', () => {
663             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
664             assert.equal(syncer.builderName(), 'ABTest-iPad-RunBenchmark-Tests');
665
666             syncer.pullBuildbot(1);
667             assert.equal(requests.length, 1);
668             assert.equal(requests[0].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/pendingBuilds');
669             requests[0].resolve([]);
670             return MockRemoteAPI.waitForRequest().then(() => {
671                 assert.equal(requests.length, 2);
672                 assert.equal(requests[1].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/builds/?select=-1');
673             });
674         });
675
676         it('should fetch the right number of recent builds', () => {
677             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
678
679             syncer.pullBuildbot(3);
680             assert.equal(requests.length, 1);
681             assert.equal(requests[0].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/pendingBuilds');
682             requests[0].resolve([]);
683             return MockRemoteAPI.waitForRequest().then(() => {
684                 assert.equal(requests.length, 2);
685                 assert.equal(requests[1].url, '/json/builders/ABTest-iPad-RunBenchmark-Tests/builds/?select=-1&select=-2&select=-3');
686             });
687         });
688
689         it('should create BuildbotBuildEntry for pending builds', () => {
690             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
691             let promise = syncer.pullBuildbot();
692             requests[0].resolve([samplePendingBuild()]);
693             return promise.then((entries) => {
694                 assert.equal(entries.length, 1);
695                 let entry = entries[0];
696                 assert.ok(entry instanceof BuildbotBuildEntry);
697                 assert.ok(!entry.buildNumber());
698                 assert.ok(!entry.slaveName());
699                 assert.equal(entry.buildRequestId(), 16733);
700                 assert.ok(entry.isPending());
701                 assert.ok(!entry.isInProgress());
702                 assert.ok(!entry.hasFinished());
703                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
704             });
705         });
706
707         it('should create BuildbotBuildEntry for in-progress builds', () => {
708             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
709
710             let promise = syncer.pullBuildbot(1);
711             assert.equal(requests.length, 1);
712             requests[0].resolve([]);
713             return MockRemoteAPI.waitForRequest().then(() => {
714                 assert.equal(requests.length, 2);
715                 requests[1].resolve({[-1]: sampleInProgressBuild()});
716                 return promise;
717             }).then((entries) => {
718                 assert.equal(entries.length, 1);
719                 let entry = entries[0];
720                 assert.ok(entry instanceof BuildbotBuildEntry);
721                 assert.equal(entry.buildNumber(), 614);
722                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
723                 assert.equal(entry.buildRequestId(), 16733);
724                 assert.ok(!entry.isPending());
725                 assert.ok(entry.isInProgress());
726                 assert.ok(!entry.hasFinished());
727                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
728             });
729         });
730
731         it('should create BuildbotBuildEntry for finished builds', () => {
732             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
733
734             let promise = syncer.pullBuildbot(1);
735             assert.equal(requests.length, 1);
736             requests[0].resolve([]);
737             return MockRemoteAPI.waitForRequest().then(() => {
738                 assert.equal(requests.length, 2);
739                 requests[1].resolve({[-1]: sampleFinishedBuild()});
740                 return promise;
741             }).then((entries) => {
742                 assert.deepEqual(entries.length, 1);
743                 let entry = entries[0];
744                 assert.ok(entry instanceof BuildbotBuildEntry);
745                 assert.equal(entry.buildNumber(), 1755);
746                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
747                 assert.equal(entry.buildRequestId(), 18935);
748                 assert.ok(!entry.isPending());
749                 assert.ok(!entry.isInProgress());
750                 assert.ok(entry.hasFinished());
751                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
752             });
753         });
754
755         it('should create BuildbotBuildEntry for mixed pending, in-progress, finished, and missing builds', () => {
756             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
757
758             let promise = syncer.pullBuildbot(5);
759             assert.equal(requests.length, 1);
760
761             requests[0].resolve([samplePendingBuild(123)]);
762
763             return MockRemoteAPI.waitForRequest().then(() => {
764                 assert.equal(requests.length, 2);
765                 requests[1].resolve({[-1]: sampleFinishedBuild(), [-2]: {'error': 'Not available'}, [-4]: sampleInProgressBuild()});
766                 return promise;
767             }).then((entries) => {
768                 assert.deepEqual(entries.length, 3);
769
770                 let entry = entries[0];
771                 assert.ok(entry instanceof BuildbotBuildEntry);
772                 assert.equal(entry.buildNumber(), null);
773                 assert.equal(entry.slaveName(), null);
774                 assert.equal(entry.buildRequestId(), 123);
775                 assert.ok(entry.isPending());
776                 assert.ok(!entry.isInProgress());
777                 assert.ok(!entry.hasFinished());
778                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
779
780                 entry = entries[1];
781                 assert.ok(entry instanceof BuildbotBuildEntry);
782                 assert.equal(entry.buildNumber(), 614);
783                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
784                 assert.equal(entry.buildRequestId(), 16733);
785                 assert.ok(!entry.isPending());
786                 assert.ok(entry.isInProgress());
787                 assert.ok(!entry.hasFinished());
788                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
789
790                 entry = entries[2];
791                 assert.ok(entry instanceof BuildbotBuildEntry);
792                 assert.equal(entry.buildNumber(), 1755);
793                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
794                 assert.equal(entry.buildRequestId(), 18935);
795                 assert.ok(!entry.isPending());
796                 assert.ok(!entry.isInProgress());
797                 assert.ok(entry.hasFinished());
798                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
799             });
800         });
801
802         it('should sort BuildbotBuildEntry by order', () => {
803             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
804
805             let promise = syncer.pullBuildbot(5);
806             assert.equal(requests.length, 1);
807
808             requests[0].resolve([samplePendingBuild(456, 2), samplePendingBuild(123, 1)]);
809
810             return MockRemoteAPI.waitForRequest().then(() => {
811                 assert.equal(requests.length, 2);
812                 requests[1].resolve({[-3]: sampleFinishedBuild(), [-1]: {'error': 'Not available'}, [-2]: sampleInProgressBuild()});
813                 return promise;
814             }).then((entries) => {
815                 assert.deepEqual(entries.length, 4);
816
817                 let entry = entries[0];
818                 assert.ok(entry instanceof BuildbotBuildEntry);
819                 assert.equal(entry.buildNumber(), null);
820                 assert.equal(entry.slaveName(), null);
821                 assert.equal(entry.buildRequestId(), 123);
822                 assert.ok(entry.isPending());
823                 assert.ok(!entry.isInProgress());
824                 assert.ok(!entry.hasFinished());
825                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
826
827                 entry = entries[1];
828                 assert.ok(entry instanceof BuildbotBuildEntry);
829                 assert.equal(entry.buildNumber(), null);
830                 assert.equal(entry.slaveName(), null);
831                 assert.equal(entry.buildRequestId(), 456);
832                 assert.ok(entry.isPending());
833                 assert.ok(!entry.isInProgress());
834                 assert.ok(!entry.hasFinished());
835                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/');
836
837                 entry = entries[2];
838                 assert.ok(entry instanceof BuildbotBuildEntry);
839                 assert.equal(entry.buildNumber(), 614);
840                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
841                 assert.equal(entry.buildRequestId(), 16733);
842                 assert.ok(!entry.isPending());
843                 assert.ok(entry.isInProgress());
844                 assert.ok(!entry.hasFinished());
845                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
846
847                 entry = entries[3];
848                 assert.ok(entry instanceof BuildbotBuildEntry);
849                 assert.equal(entry.buildNumber(), 1755);
850                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
851                 assert.equal(entry.buildRequestId(), 18935);
852                 assert.ok(!entry.isPending());
853                 assert.ok(!entry.isInProgress());
854                 assert.ok(entry.hasFinished());
855                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
856             });
857         });
858
859         it('should override BuildbotBuildEntry for pending builds by in-progress builds', () => {
860             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
861
862             let promise = syncer.pullBuildbot(5);
863             assert.equal(requests.length, 1);
864
865             requests[0].resolve([samplePendingBuild()]);
866
867             return MockRemoteAPI.waitForRequest().then(() => {
868                 assert.equal(requests.length, 2);
869                 requests[1].resolve({[-1]: sampleInProgressBuild()});
870                 return promise;
871             }).then((entries) => {
872                 assert.equal(entries.length, 1);
873
874                 let entry = entries[0];
875                 assert.ok(entry instanceof BuildbotBuildEntry);
876                 assert.equal(entry.buildNumber(), 614);
877                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
878                 assert.equal(entry.buildRequestId(), 16733);
879                 assert.ok(!entry.isPending());
880                 assert.ok(entry.isInProgress());
881                 assert.ok(!entry.hasFinished());
882                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/614');
883             });
884         });
885
886         it('should override BuildbotBuildEntry for pending builds by finished builds', () => {
887             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
888
889             let promise = syncer.pullBuildbot(5);
890             assert.equal(requests.length, 1);
891
892             requests[0].resolve([samplePendingBuild()]);
893
894             return MockRemoteAPI.waitForRequest().then(() => {
895                 assert.equal(requests.length, 2);
896                 requests[1].resolve({[-1]: sampleFinishedBuild(16733)});
897                 return promise;
898             }).then((entries) => {
899                 assert.equal(entries.length, 1);
900
901                 let entry = entries[0];
902                 assert.ok(entry instanceof BuildbotBuildEntry);
903                 assert.equal(entry.buildNumber(), 1755);
904                 assert.equal(entry.slaveName(), 'ABTest-iPad-0');
905                 assert.equal(entry.buildRequestId(), 16733);
906                 assert.ok(!entry.isPending());
907                 assert.ok(!entry.isInProgress());
908                 assert.ok(entry.hasFinished());
909                 assert.equal(entry.url(), 'http://build.webkit.org/builders/ABTest-iPad-RunBenchmark-Tests/builds/1755');
910             });
911         });
912     });
913
914     describe('scheduleRequest', () => {
915         it('should schedule a build request on a specified slave', () => {
916             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[0];
917
918             const waitForRequest = MockRemoteAPI.waitForRequest();
919             syncer.scheduleRequest(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer), 'some-slave');
920             return waitForRequest.then(() => {
921                 assert.equal(requests.length, 1);
922                 assert.equal(requests[0].url, '/builders/ABTest-iPhone-RunBenchmark-Tests/force');
923                 assert.equal(requests[0].method, 'POST');
924                 assert.deepEqual(requests[0].data, {
925                     'build_request_id': '16733-' + MockModels.iphone.id(),
926                     'desired_image': '13A452',
927                     "opensource": "197463",
928                     'forcescheduler': 'ABTest-iPhone-RunBenchmark-Tests-ForceScheduler',
929                     'slavename': 'some-slave',
930                     'test_name': 'speedometer'
931                 });
932             });
933         });
934     });
935
936     describe('scheduleRequestInGroupIfAvailable', () => {
937
938         function pullBuildbotWithAssertion(syncer, pendingBuilds, inProgressAndFinishedBuilds)
939         {
940             const promise = syncer.pullBuildbot(5);
941             assert.equal(requests.length, 1);
942             requests[0].resolve(pendingBuilds);
943             return MockRemoteAPI.waitForRequest().then(() => {
944                 assert.equal(requests.length, 2);
945                 requests[1].resolve(inProgressAndFinishedBuilds);
946                 requests.length = 0;
947                 return promise;
948             });
949         }
950
951         it('should schedule a build if builder has no builds if slaveList is not specified', () => {
952             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, smallConfiguration())[0];
953
954             return pullBuildbotWithAssertion(syncer, [], {}).then(() => {
955                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
956                 assert.equal(requests.length, 1);
957                 assert.equal(requests[0].url, '/builders/some%20builder/force');
958                 assert.equal(requests[0].method, 'POST');
959                 assert.deepEqual(requests[0].data, {id: '16733-' + MockModels.somePlatform.id(), 'os': '13A452', 'wk': '197463'});
960             });
961         });
962
963         it('should schedule a build if builder only has finished builds if slaveList is not specified', () => {
964             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, smallConfiguration())[0];
965
966             return pullBuildbotWithAssertion(syncer, [], {[-1]: smallFinishedBuild()}).then(() => {
967                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
968                 assert.equal(requests.length, 1);
969                 assert.equal(requests[0].url, '/builders/some%20builder/force');
970                 assert.equal(requests[0].method, 'POST');
971                 assert.deepEqual(requests[0].data, {id: '16733-' + MockModels.somePlatform.id(), 'os': '13A452', 'wk': '197463'});
972             });
973         });
974
975         it('should not schedule a build if builder has a pending build if slaveList is not specified', () => {
976             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, smallConfiguration())[0];
977
978             return pullBuildbotWithAssertion(syncer, [smallPendingBuild()], {}).then(() => {
979                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
980                 assert.equal(requests.length, 0);
981             });
982         });
983
984         it('should schedule a build if builder does not have pending or completed builds on the matching slave', () => {
985             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[0];
986
987             return pullBuildbotWithAssertion(syncer, [], {}).then(() => {
988                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.iphone, MockModels.speedometer));
989                 assert.equal(requests.length, 1);
990                 assert.equal(requests[0].url, '/builders/ABTest-iPhone-RunBenchmark-Tests/force');
991                 assert.equal(requests[0].method, 'POST');
992             });
993         });
994
995         it('should schedule a build if builder only has finished builds on the matching slave', () => {
996             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
997
998             pullBuildbotWithAssertion(syncer, [], {[-1]: sampleFinishedBuild()}).then(() => {
999                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1000                 assert.equal(requests.length, 1);
1001                 assert.equal(requests[0].url, '/builders/ABTest-iPad-RunBenchmark-Tests/force');
1002                 assert.equal(requests[0].method, 'POST');
1003             });
1004         });
1005
1006         it('should not schedule a build if builder has a pending build on the maching slave', () => {
1007             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1008
1009             pullBuildbotWithAssertion(syncer, [samplePendingBuild()], {}).then(() => {
1010                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1011                 assert.equal(requests.length, 0);
1012             });
1013         });
1014
1015         it('should schedule a build if builder only has a pending build on a non-maching slave', () => {
1016             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1017
1018             return pullBuildbotWithAssertion(syncer, [samplePendingBuild(1, 1, 'another-slave')], {}).then(() => {
1019                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1020                 assert.equal(requests.length, 1);
1021             });
1022         });
1023
1024         it('should schedule a build if builder only has an in-progress build on the matching slave', () => {
1025             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1026
1027             return pullBuildbotWithAssertion(syncer, [], {[-1]: sampleInProgressBuild()}).then(() => {
1028                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1029                 assert.equal(requests.length, 1);
1030             });
1031         });
1032
1033         it('should schedule a build if builder has an in-progress build on another slave', () => {
1034             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1035
1036             return pullBuildbotWithAssertion(syncer, [], {[-1]: sampleInProgressBuild('other-slave')}).then(() => {
1037                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1038                 assert.equal(requests.length, 1);
1039             });
1040         });
1041
1042         it('should not schedule a build if the request does not match any configuration', () => {
1043             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[0];
1044
1045             return pullBuildbotWithAssertion(syncer, [], {}).then(() => {
1046                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1047                 assert.equal(requests.length, 0);
1048             });
1049         });
1050
1051         it('should not schedule a build if a new request had been submitted to the same slave', (done) => {
1052             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1053
1054             pullBuildbotWithAssertion(syncer, [], {}).then(() => {
1055                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-0');
1056                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-1');
1057             }).then(() => {
1058                 assert.equal(requests.length, 2);
1059                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer));
1060             }).then(() => {
1061                 assert.equal(requests.length, 2);
1062                 done();
1063             }).catch(done);
1064         });
1065
1066         it('should schedule a build if a new request had been submitted to another slave', () => {
1067             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, sampleiOSConfig())[1];
1068
1069             return pullBuildbotWithAssertion(syncer, [], {}).then(() => {
1070                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-0');
1071                 assert.equal(requests.length, 1);
1072                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.ipad, MockModels.speedometer), 'ABTest-iPad-1');
1073                 assert.equal(requests.length, 2);
1074             });
1075         });
1076
1077         it('should not schedule a build if a new request had been submitted to the same builder without slaveList', () => {
1078             let syncer = BuildbotSyncer._loadConfig(MockRemoteAPI, smallConfiguration())[0];
1079
1080             return pullBuildbotWithAssertion(syncer, [], {}).then(() => {
1081                 syncer.scheduleRequest(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest), null);
1082                 assert.equal(requests.length, 1);
1083                 syncer.scheduleRequestInGroupIfAvailable(createSampleBuildRequest(MockModels.somePlatform, MockModels.someTest));
1084                 assert.equal(requests.length, 1);
1085             });
1086         });
1087     });
1088 });