Retrying an A/B testing does not set the repetition count in some cases
[WebKit.git] / Websites / perf.webkit.org / browser-tests / index.html
1 <!DOCTYPE html>
2 <html>
3 <head>
4 <meta charset="utf-8">
5 <title>In-Browser Tests for Performance Dashboard</title>
6 <link rel="stylesheet" href="../node_modules/mocha/mocha.css">
7 <script src="../node_modules/mocha/mocha.js"></script>
8 <script src="https://cdnjs.cloudflare.com/ajax/libs/expect.js/0.2.0/expect.min.js"></script>
9 <script>
10
11 mocha.setup('bdd');
12
13 </script>
14 <script src="../unit-tests/resources/mock-remote-api.js"></script>
15 </head>
16 <body>
17 <div id="mocha"></div>
18 <script src="component-base-tests.js"></script>
19 <script src="close-button-tests.js"></script>
20 <script src="editable-text-tests.js"></script>
21 <script src="time-series-chart-tests.js"></script>
22 <script src="interactive-time-series-chart-tests.js"></script>
23 <script src="chart-status-evaluator-tests.js"></script>
24 <script src="chart-revision-range-tests.js"></script>
25 <script src="commit-log-viewer-tests.js"></script>
26 <script src="test-group-form-tests.js"></script>
27 <script>
28
29 afterEach(() => {
30     BrowsingContext.cleanup();
31 });
32
33 class BrowsingContext {
34
35     constructor()
36     {
37         let iframe = document.createElement('iframe');
38         document.body.appendChild(iframe);
39         iframe.style.position = 'absolute';
40         iframe.style.left = '0px';
41         iframe.style.top = '0px';
42         BrowsingContext._iframes.push(iframe);
43
44         // Expedite calls to callbacks to make tests go faster.
45         iframe.contentWindow.requestAnimationFrame = (callback) => setTimeout(callback, 0);
46
47         this.iframe = iframe;
48         this.symbols = {};
49         this.global = this.iframe.contentWindow;
50         this.document = this.iframe.contentDocument;
51         this._didLoadMockRemote = false;
52     }
53
54     importScripts(pathList, ...symbolList)
55     {
56         const doc = this.iframe.contentDocument;
57         const global = this.iframe.contentWindow;
58
59         pathList = pathList.map((path) => `../public/v3/${path}`);
60         if (!this._didLoadMockRemote) {
61             this._didLoadMockRemote = true;
62             pathList.unshift('../unit-tests/resources/mock-remote-api.js');
63         }
64
65         return Promise.all(pathList.map((path) => {
66             return new Promise((resolve, reject) => {
67                 let script = doc.createElement('script');
68                 script.addEventListener('load', resolve);
69                 script.addEventListener('error', reject);
70                 script.src = path;
71                 script.async = false;
72                 doc.body.appendChild(script);
73             });
74         })).then(() => {
75             const script = doc.createElement('script');
76             script.textContent = `window.importedSymbols = [${symbolList.join(', ')}];`;
77             global.RemoteAPI = global.MockRemoteAPI;
78             doc.body.appendChild(script);
79
80             const importedSymbols = global.importedSymbols;
81             for (let i = 0; i < symbolList.length; i++)
82                 this.symbols[symbolList[i]] = importedSymbols[i];
83
84             return symbolList.length == 1 ? importedSymbols[0] : importedSymbols;
85         });
86     }
87
88     importScript(path, ...symbols)
89     {
90         return this.importScripts([path], ...symbols);
91     }
92
93     static cleanup()
94     {
95         BrowsingContext._iframes.forEach((iframe) => { iframe.remove(); });
96         BrowsingContext._iframes = [];
97     }
98 }
99 BrowsingContext._iframes = [];
100
101 function waitForComponentsToRender(context)
102 {
103     if (!context._dummyComponent) {
104         const ComponentBase = context.symbols.ComponentBase;
105         context._dummyComponent = class SomeComponent extends ComponentBase {
106             constructor(resolve)
107             {
108                 super();
109                 this._resolve = resolve;
110             }
111             render() { setTimeout(this._resolve, 0); }
112         }
113         ComponentBase.defineElement('dummy-component', context._dummyComponent);
114     }
115     return new Promise((resolve) => {
116         const instance = new context._dummyComponent(resolve);
117         context.document.body.appendChild(instance.element());
118         setTimeout(() => {
119             instance.enqueueToRender();
120         }, 0);
121     });
122 }
123
124 function wait(milliseconds)
125 {
126     return new Promise((resolve) => {
127         setTimeout(resolve, milliseconds);
128     });
129 }
130
131 function canvasImageData(canvas)
132 {
133     return canvas.getContext('2d').getImageData(0, 0, canvas.width, canvas.height);
134 }
135
136 function canvasRefTest(canvas1, canvas2, shouldMatch)
137 {
138     expect(canvas1.offsetWidth).to.be(canvas2.offsetWidth);
139     expect(canvas2.offsetHeight).to.be(canvas2.offsetHeight);
140     const data1 = canvasImageData(canvas1).data;
141     const data2 = canvasImageData(canvas2).data;
142     expect(data1.length).to.be.a('number');
143     expect(data1.length).to.be(data2.length);
144
145     let match = true;
146     for (let i = 0; i < data1.length; i++) {
147         if (data1[i] != data2[i]) {
148             match = false;
149             break;
150         }
151     }
152
153     if (match == shouldMatch)
154         return;
155
156     [canvas1, canvas2].forEach((canvas) => {
157         let image = document.createElement('img');
158         image.src = canvas.toDataURL();
159         image.style.display = 'block';
160         document.body.appendChild(image);
161     });
162
163     throw new Error(shouldMatch ? 'Canvas contents were different' : 'Canvas contents were identical');
164 }
165
166 const CanvasTest = {
167     fillCanvasBeforeRedrawCheck(canvas)
168     {
169         const canvasContext = canvas.getContext('2d');
170         canvasContext.fillStyle = 'white';
171         canvasContext.fillRect(0, 0, canvas.width, canvas.height);
172     },
173
174     hasCanvasBeenRedrawn(canvas)
175     {
176         return canvasImageData(canvas).data.some((value) => value != 255);
177     },
178
179     canvasImageData(canvas) { return canvasImageData(canvas); },
180
181     canvasContainsColor(canvas, color, rect = {})
182     {
183         const content = canvas.getContext('2d').getImageData(rect.x || 0, rect.y || 0, rect.width || canvas.width, rect.height || canvas.height);
184         let found = false;
185         const data = content.data;
186         for (let startOfPixel = 0; startOfPixel < data.length; startOfPixel += 4) {
187             let r = data[startOfPixel];
188             let g = data[startOfPixel + 1];
189             let b = data[startOfPixel + 2];
190             let a = data[startOfPixel + 3];
191             if (r == color.r && g == color.g && b == color.b && (color.a == undefined || a == color.a))
192                 return true;
193         }
194         return false;
195     },
196
197     expectCanvasesMatch(canvas1, canvas2) { return canvasRefTest(canvas1, canvas2, true); },
198     expectCanvasesMismatch(canvas1, canvas2) { return canvasRefTest(canvas1, canvas2, false); },
199 }
200
201 const dayInMilliseconds = 24 * 3600 * 1000;
202
203 function posixTime(string) { return +new Date(string); }
204
205 const ChartTest = {
206     importChartScripts(context)
207     {
208         return context.importScripts([
209             '../shared/statistics.js',
210             'instrumentation.js',
211             'models/data-model.js',
212             'models/time-series.js',
213             'models/measurement-set.js',
214             'models/measurement-cluster.js',
215             'models/measurement-adaptor.js',
216             'models/repository.js',
217             'models/platform.js',
218             'models/test.js',
219             'models/metric.js',
220             'models/commit-set.js',
221             'models/commit-log.js',
222             'components/base.js',
223             'components/time-series-chart.js',
224             'components/interactive-time-series-chart.js'],
225             'ComponentBase', 'TimeSeriesChart', 'InteractiveTimeSeriesChart',
226             'Platform', 'Metric', 'Test', 'Repository', 'MeasurementSet', 'MockRemoteAPI').then(() => {
227                 return context.symbols.TimeSeriesChart;
228             })
229     },
230
231     posixTime: posixTime,
232
233     get sampleCluster() { return this.makeSampleCluster(); },
234
235     makeModelObjectsForSampleCluster(context)
236     {
237         const test = context.symbols.Test.ensureSingleton(2, {name: 'Test'});
238         const metric = context.symbols.Metric.ensureSingleton(1, {name: 'Time', test})
239         const platform = context.symbols.Platform.ensureSingleton(1,
240             {name: 'SomePlatform', metrics: [metric], lastModifiedByMetric: [posixTime('2016-01-18T00:00:00Z')]});
241         metric.addPlatform(platform);
242         context.symbols.Repository.ensureSingleton(1, {name: 'SomeApp'});
243         context.symbols.Repository.ensureSingleton(2, {name: 'macOS'});
244     },
245
246     makeSampleCluster(options = {})
247     {
248         const baselineStart = options.baselineIsSmaller ? 30 : 130;
249         const targetStart = options.targetIsBigger ? 120 : 90;
250         return {
251         "clusterStart": posixTime('2016-01-01T00:00:00Z'),
252         "clusterSize": 7 * dayInMilliseconds,
253         "startTime": posixTime('2016-01-01T00:00:00Z'),
254         "endTime": posixTime('2016-01-08T00:00:00Z'),
255         "lastModified": posixTime('2016-01-18T00:00:00Z'),
256         "clusterCount": 1,
257         "status": "OK",
258         "formatMap": [
259             "id", "mean", "iterationCount", "sum", "squareSum", "markedOutlier",
260             "revisions",
261             "commitTime", "build", "buildTime", "buildNumber", "builder"
262         ],
263         "configurations": {
264             "current": [
265                 [
266                     1000, 100, 1, 100, 100 * 100, false,
267                     [ [2000, 1, "4000", posixTime('2016-01-05T17:35:00Z')], [3000, 2, "15B42", 0] ],
268                     posixTime('2016-01-05T17:35:00Z'), 5000, posixTime('2016-01-05T19:23:00Z'), "10", 7
269                 ],
270                 [
271                     1001, 131, 1, 131, 131 * 131, true,
272                     [ [2001, 1, "4001", posixTime('2016-01-05T18:43:01Z')], [3000, 2, "15B42", 0] ],
273                     posixTime('2016-01-05T18:43:01Z'), 5001, posixTime('2016-01-05T20:58:01Z'), "11", 7
274                 ],
275                 [
276                     1002, 122, 1, 122, 122 * 122, false,
277                     [ [2002, 1, "4002", posixTime('2016-01-05T20:01:02Z')], [3000, 2, "15B42", 0] ],
278                     posixTime('2016-01-05T20:01:02Z'), 5002, posixTime('2016-01-05T22:37:02Z'), "12", 7
279                 ],
280                 [
281                     1003, 113, 1, 113, 113 * 113, false,
282                     [ [2003, 1, "4003", posixTime('2016-01-05T23:19:03Z')], [3000, 2, "15B42", 0] ],
283                     posixTime('2016-01-05T23:19:03Z'), 5003, posixTime('2016-01-06T23:19:03Z'), "13", 7
284                 ],
285                 [
286                     1004, 124, 1, 124, 124 * 124, false,
287                     [ [2004, 1, "4004", posixTime('2016-01-06T01:52:04Z')], [3001, 2, "15C50", 0] ],
288                     posixTime('2016-01-06T01:52:04Z'), 5004, posixTime('2016-01-06T02:42:04Z'), "14", 7
289                 ],
290                 [
291                     1005, 115, 1, 115, 115 * 115, true,
292                     [ [2005, 1, "4005", posixTime('2016-01-06T03:22:05Z')], [3001, 2, "15C50", 0] ],
293                     posixTime('2016-01-06T03:22:05Z'), 5005, posixTime('2016-01-06T06:01:05Z'), "15", 7
294                 ],
295                 [
296                     1006, 116, 1, 116, 116 * 116, false,
297                     [ [2006, 1, "4006", posixTime('2016-01-06T05:59:06Z')], [3001, 2, "15C50", 0] ],
298                     posixTime('2016-01-06T05:59:06Z'), 5006, posixTime('2016-01-06T08:34:06Z'), "16", 7
299                 ]
300             ],
301             "baseline": [
302                 [
303                     7000, baselineStart, 1, baselineStart, baselineStart * baselineStart, false,
304                     [ ],
305                     posixTime('2016-01-05T12:00:30Z'), 5030, posixTime('2016-01-05T12:00:30Z'), "30", 7
306                 ],
307                 [
308                     7001, baselineStart + 1, 1, baselineStart + 1, Math.pow(baselineStart + 1, 2), false,
309                     [ ],
310                     posixTime('2016-01-06T00:00:31Z'), 5031, posixTime('2016-01-06T00:00:31Z'), "31", 7
311                 ],
312             ],
313             "target": [
314                 [
315                     8000, targetStart, 1, targetStart, targetStart * targetStart, false,
316                     [ ],
317                     posixTime('2016-01-05T12:00:30Z'), 5030, posixTime('2016-01-05T12:00:30Z'), "90", 7
318                 ],
319                 [
320                     8001, targetStart + 1, 1, targetStart + 1, Math.pow(targetStart + 1, 2), false,
321                     [ ],
322                     posixTime('2016-01-06T00:00:31Z'), 5031, posixTime('2016-01-06T00:00:31Z'), "91", 7
323                 ],
324             ]
325         }};
326     },
327
328     createChartWithSampleCluster(context, sourceList = null, chartOptions = {}, className = 'TimeSeriesChart')
329     {
330         const TimeSeriesChart = context.symbols[className];
331         const MeasurementSet = context.symbols.MeasurementSet;
332
333         if (sourceList == null)
334             sourceList = [{type: 'current'}];
335
336         const sampleCluster = MeasurementSet.findSet(1, 1, 0);
337         for (let source of sourceList) {
338             if (!source.type)
339                 source.type = 'current';
340             source.measurementSet = sampleCluster;
341         }
342
343         const chart = new TimeSeriesChart(sourceList, chartOptions);
344         const element = chart.element();
345         element.style.width = chartOptions.width || '300px';
346         element.style.height = chartOptions.height || '100px';
347         context.document.body.appendChild(element);
348
349         return chart;
350     },
351
352     createInteractiveChartWithSampleCluster(context, sourceList = null, chartOptions = {})
353     {
354         if (sourceList == null)
355             sourceList = [{type: 'current', interactive: true}];
356         return this.createChartWithSampleCluster(context, sourceList, chartOptions, 'InteractiveTimeSeriesChart');
357     },
358
359     respondWithSampleCluster(request, options)
360     {
361         expect(request.url).to.be('../data/measurement-set-1-1.json');
362         expect(request.method).to.be('GET');
363         request.resolve(this.makeSampleCluster(options));
364     },
365 };
366
367 mocha.checkLeaks();
368 mocha.globals(['expect', 'BrowsingContext', 'CanvasTest', 'ChartTest', 'wait', 'waitForComponentsToRender']);
369 mocha.run();
370
371 </script>
372 </body>
373 </html>