DoYouEvenBench: Extract tests and runner code from benchmark.js/html
authorrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 7 Jan 2014 23:22:22 +0000 (23:22 +0000)
committerrniwa@webkit.org <rniwa@webkit.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 7 Jan 2014 23:22:22 +0000 (23:22 +0000)
https://bugs.webkit.org/show_bug.cgi?id=126596

Reviewed by Stephanie Lewis.

Extracted benchmark-runner.js and tests.js out of benchmark.js and benchmark.html.

Added a "client" interface to BenchmarkRunner so that benchmark.html could register necessary hooks to
update its UI. Also made BenchmarkRunner store a tree of results so that the serialization of test names
could be isolated from BenchmarkRunner itself in the future.

* DoYouEvenBench/benchmark.html:
Moved the code to instantiate and update UI here from benchmark.js. The test code was moved out of this
file into resources/tests.js.

* DoYouEvenBench/resources/benchmark-runner.js: Renamed from PerformanceTests/DoYouEvenBench/benchmark.js.
(SimplePromise): Moved from benchmark.js
(SimplePromise.prototype.then): Ditto.
(SimplePromise.prototype.resolve): Ditto.
(BenchmarkTestStep): Added. Wraps each test step.
(BenchmarkRunner.suite): Moved from benchmark.js.
(BenchmarkRunner.setClient): Added.
(BenchmarkRunner.waitForElement): Moved.
(BenchmarkRunner._removeFrame): Ditto.
(BenchmarkRunner._appendFrame): Ditto. Set the width and the height of the iframe as they're more than
presentational as they affect performance.
(BenchmarkRunner._waitAndWarmUp): Ditto.
(BenchmarkRunner._runTest): Ditto.
(BenchmarkRunner._testName): Ditto.
(BenchmarkState): Ditto.
(BenchmarkState.prototype.currentSuite): Ditto.
(BenchmarkState.prototype.currentTest): Ditto.
(BenchmarkState.prototype.next): Ditto.
(BenchmarkState.prototype.isFirstTest): Ditto.
(BenchmarkState.prototype.prepareCurrentSuite): Ditto.
(BenchmarkRunner.step): Ditto.
(BenchmarkRunner._runTestAndRecordResults): Ditto. Note the code to update the UI has been move to
benchmark.html. Also moved the code to accumulate the totals here from _finalize.
(BenchmarkRunner._finalize): Moved.

* DoYouEvenBench/resources/tests.js: Copied from PerformanceTests/DoYouEvenBench/benchmark.html.
Uses BenchmarkTestStep instead of an array for each test step.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@161458 268f45cc-cd09-0410-ab3c-d52691b4dbfc

PerformanceTests/ChangeLog
PerformanceTests/DoYouEvenBench/benchmark.html
PerformanceTests/DoYouEvenBench/resources/benchmark-runner.js [moved from PerformanceTests/DoYouEvenBench/benchmark.js with 55% similarity]
PerformanceTests/DoYouEvenBench/resources/tests.js [new file with mode: 0644]

index 011b83c..a3482f9 100644 (file)
@@ -1,3 +1,48 @@
+2014-01-07  Ryosuke Niwa  <rniwa@webkit.org>
+
+        DoYouEvenBench: Extract tests and runner code from benchmark.js/html
+        https://bugs.webkit.org/show_bug.cgi?id=126596
+
+        Reviewed by Stephanie Lewis.
+        
+        Extracted benchmark-runner.js and tests.js out of benchmark.js and benchmark.html.
+
+        Added a "client" interface to BenchmarkRunner so that benchmark.html could register necessary hooks to
+        update its UI. Also made BenchmarkRunner store a tree of results so that the serialization of test names
+        could be isolated from BenchmarkRunner itself in the future.
+
+        * DoYouEvenBench/benchmark.html:
+        Moved the code to instantiate and update UI here from benchmark.js. The test code was moved out of this
+        file into resources/tests.js.
+
+        * DoYouEvenBench/resources/benchmark-runner.js: Renamed from PerformanceTests/DoYouEvenBench/benchmark.js.
+        (SimplePromise): Moved from benchmark.js
+        (SimplePromise.prototype.then): Ditto.
+        (SimplePromise.prototype.resolve): Ditto.
+        (BenchmarkTestStep): Added. Wraps each test step.
+        (BenchmarkRunner.suite): Moved from benchmark.js.
+        (BenchmarkRunner.setClient): Added.
+        (BenchmarkRunner.waitForElement): Moved.
+        (BenchmarkRunner._removeFrame): Ditto.
+        (BenchmarkRunner._appendFrame): Ditto. Set the width and the height of the iframe as they're more than
+        presentational as they affect performance.
+        (BenchmarkRunner._waitAndWarmUp): Ditto.
+        (BenchmarkRunner._runTest): Ditto.
+        (BenchmarkRunner._testName): Ditto.
+        (BenchmarkState): Ditto.
+        (BenchmarkState.prototype.currentSuite): Ditto.
+        (BenchmarkState.prototype.currentTest): Ditto.
+        (BenchmarkState.prototype.next): Ditto.
+        (BenchmarkState.prototype.isFirstTest): Ditto.
+        (BenchmarkState.prototype.prepareCurrentSuite): Ditto.
+        (BenchmarkRunner.step): Ditto.
+        (BenchmarkRunner._runTestAndRecordResults): Ditto. Note the code to update the UI has been move to
+        benchmark.html. Also moved the code to accumulate the totals here from _finalize.
+        (BenchmarkRunner._finalize): Moved. 
+
+        * DoYouEvenBench/resources/tests.js: Copied from PerformanceTests/DoYouEvenBench/benchmark.html.
+        Uses BenchmarkTestStep instead of an array for each test step.
+
 2014-01-02  Myles C. Maxfield  <mmaxfield@apple.com>
 
         Allow ImageBuffer to re-use IOSurfaces
index bb55289..a5bdb56 100644 (file)
 <html>
 <head>
 <title>DoYouEvenBench</title>
-<script src="benchmark.js"></script>
+<script src="resources/benchmark-runner.js" defer></script>
+<script src="resources/tests.js" defer></script>
+<style>
+iframe { border: 1px solid black; }
+ol { list-style: none; margin: 0; padding: 0; }
+ol ol { margin-left: 2em; list-position: outside; }
+.running { text-decoration: underline; }
+.ran { color: grey; }
+nav { position: absolute; right: 10px; }
+</style>
+</head>
+<body>
 <script>
 
-var numberOfItemsToAdd = 100;
-
-BenchmarkRunner.suite({
-    name: 'VanillaJS/TodoMVC',
-    url: 'todomvc/vanilla-examples/vanillajs/index.html',
-    prepare: function (contentWindow, contentDocument) {
-        return BenchmarkRunner.waitForElement('#new-todo').then(function (element) {
-            element.focus();
-            return element;
-        });
-    },
-    tests: [
-        ['Adding' + numberOfItemsToAdd + 'Items', function (newTodo, contentWindow, contentDocument) {
-            var todoController = contentWindow.todo.controller;
-            for (var i = 0; i < numberOfItemsToAdd; i++) {
-                newTodo.value = 'Something to do ' + i;
-                todoController.addItem({keyCode: todoController.ENTER_KEY, target: newTodo});
-            }
-        }],
-        ['CompletingAllItems', function (newTodo, contentWindow, contentDocument) {
-            var checkboxes = contentDocument.querySelectorAll('.toggle');
-            for (var i = 0; i < checkboxes.length; i++)
-                checkboxes[i].click();
-        }],
-        ['DeletingAllItems', function (newTodo, contentWindow, contentDocument) {
-            var deleteButtons = contentDocument.querySelectorAll('.destroy');
-            for (var i = 0; i < deleteButtons.length; i++)
-                deleteButtons[i].click();
-        }],
-    ]
-});
+window.addEventListener('load', function () {
+    var self = BenchmarkRunner;
+    var control = document.createElement('nav');
 
-BenchmarkRunner.suite({
-    name: 'EmberJS/TodoMVC',
-    url: 'todomvc/architecture-examples/emberjs/index.html',
-    prepare: function (contentWindow, contentDocument) {
-        contentWindow.Todos.Store = contentWindow.DS.Store.extend({
-            revision: 12,
-            adapter: 'Todos.LSAdapter',
-            commit: function () { }
-        });
+    var suites = BenchmarkRunner._suites;
+    var ol = document.createElement('ol');
+    var checkboxes = [];
+    for (var suiteIndex = 0; suiteIndex < suites.length; suiteIndex++) {
+        var suite = suites[suiteIndex];
+        var li = document.createElement('li');
+        var checkbox = document.createElement('input');
+        checkbox.id = suite.name;
+        checkbox.type = 'checkbox';
+        checkbox.checked = true;
+        checkbox.onchange = (function (suite, checkbox) { return function () { suite.disabled = !checkbox.checked; } })(suite, checkbox);
+        checkbox.onchange();
+        checkboxes.push(checkbox);
 
-        return BenchmarkRunner.waitForElement('#new-todo').then(function (element) {
-            element.focus();
-            return {
-                views: contentWindow.Ember.View.views,
-                emberRun: contentWindow.Ember.run,
-            }
-        });
-    },
-    tests: [
-        ['Adding' + numberOfItemsToAdd + 'Items', function (params) {
-            for (var i = 0; i < numberOfItemsToAdd; i++) {
-                params.emberRun(function () { params.views["new-todo"].set('value', 'Something to do' + i); });
-                params.emberRun(function () { params.views["new-todo"].insertNewline(document.createEvent('Event')); });
-            }
-        }],
-        ['CompletingAllItems', function (params, contentWindow, contentDocument) {
-            var checkboxes = contentDocument.querySelectorAll('.ember-checkbox');
-            for (var i = 0; i < checkboxes.length; i++) {
-                var view = params.views[checkboxes[i].id];
-                params.emberRun(function () { view.set('checked', true); });
-            }
-        }],
-        ['DeletingItems', function (params, contentWindow, contentDocument) {
-            var deleteButtons = contentDocument.querySelectorAll('.destroy');
-            for (var i = 0; i < deleteButtons.length; i++)
-                params.emberRun(function () { deleteButtons[i].click(); });
-        }],
-    ]
-});
+        li.appendChild(checkbox);
+        var label = document.createElement('label');
+        label.appendChild(document.createTextNode(self._testName(suite)));
+        li.appendChild(label);
+        label.htmlFor = checkbox.id;
 
-BenchmarkRunner.suite({
-    name: 'BackboneJS/TodoMVC',
-    url: 'todomvc/architecture-examples/backbone/index.html',
-    prepare: function (contentWindow, contentDocument) {
-    contentWindow.Backbone.sync = function () {}
-        return BenchmarkRunner.waitForElement('#new-todo').then(function (element) {
-            element.focus();
-            return element;
-        });
-    },
-    tests: [
-        ['Adding' + numberOfItemsToAdd + 'Items', function (newTodo, contentWindow, contentDocument) {
-            var appView = contentWindow.appView;
-            var fakeEvent = {which: contentWindow.ENTER_KEY};
-            for (var i = 0; i < numberOfItemsToAdd; i++) {
-                newTodo.value = 'Something to do ' + i;
-                appView.createOnEnter(fakeEvent);
-            }
-        }],
-        ['CompletingAllItems', function (newTodo, contentWindow, contentDocument) {
-            var checkboxes = contentDocument.querySelectorAll('.toggle');
-            for (var i = 0; i < checkboxes.length; i++)
-                checkboxes[i].click();
-        }],
-        ['DeletingAllItems', function (newTodo, contentWindow, contentDocument) {
-            var deleteButtons = contentDocument.querySelectorAll('.destroy');
-            for (var i = 0; i < deleteButtons.length; i++)
-                deleteButtons[i].click();
-        }],
-    ]
-});
+        var testList = document.createElement('ol');
+        for (var testIndex = 0; testIndex < suite.tests.length; testIndex++) {
+            var testItem = document.createElement('li');
+            var test = suite.tests[testIndex];
+            var anchor = document.createElement('a');
+            anchor.id = suite.name + '-' + test.name;
+            test.anchor = anchor;
+            anchor.appendChild(document.createTextNode(self._testName(suite, test.name)));
+            testItem.appendChild(anchor);
+            testList.appendChild(testItem);
+        }
+        li.appendChild(testList);
 
-BenchmarkRunner.suite({
-    name: 'jQuery/TodoMVC',
-    url: 'todomvc/architecture-examples/jquery/index.html',
-    prepare: function (contentWindow, contentDocument) {
-        return BenchmarkRunner.waitForElement('#new-todo').then(function (element) {
-            element.focus();
-            return element;
-        });
-    },
-    tests: [
-        ['Adding' + numberOfItemsToAdd + 'Items', function (newTodo, contentWindow, contentDocument) {
-            var app = contentWindow.app;
-            var fakeEvent = {which: app.ENTER_KEY};
-            for (var i = 0; i < numberOfItemsToAdd; i++) {
-                newTodo.value = 'Something to do ' + i;
-                app.create.call(newTodo, fakeEvent);
-            }
-        }],
-        ['CompletingAllItems', function (newTodo, contentWindow, contentDocument) {
-            var app = contentWindow.app;
-            var checkboxes = contentDocument.querySelectorAll('.toggle');
-            var $ = contentWindow.$;
+        ol.appendChild(li);
+    }
 
-            itemIndexToId = new Array(checkboxes.length);
-            for (var i = 0; i < checkboxes.length; i++)
-                itemIndexToId[i] = $(checkboxes[i]).closest('li').data('id');
+    control.appendChild(ol);
 
-            app.getTodo = function (element, callback) {
-                var self = this;
-                var id = itemIndexToId[this.currentItemIndex];
-                $.each(this.todos, function (j, val) {
-                    if (val.id === id) {
-                        callback.apply(self, arguments);
-                        return false;
+    BenchmarkRunner.setClient({
+        willRunTest: function (suite, test) {
+            test.anchor.classList.add('running');
+        },
+        didRunTest: function (suite, test) {
+            var classList = test.anchor.classList;
+            classList.remove('running');
+            classList.add('ran');
+        },
+        didRunSuites: function (measuredValues) {
+            var results = '';
+            var total = 0; // FIXME: Compute the total properly.
+            for (var suiteName in measuredValues) {
+                var suiteResults = measuredValues[suiteName];
+                for (var testName in suiteResults.tests) {
+                    var testResults = suiteResults.tests[testName];
+                    for (var subtestName in testResults) {
+                        results += suiteName + ' : ' + testName + ' : ' + subtestName
+                            + ': ' + testResults[subtestName] + ' ms\n';
                     }
-                });
+                }
+                results += suiteName + ' : ' + suiteResults.total + ' ms\n';
+                total += suiteResults.total;
             }
+            results += 'Total : ' + total + ' ms\n';
 
-            for (var i = 0; i < checkboxes.length; i++) {
-                app.currentItemIndex = i;
-                app.toggle.call(checkboxes[i]);
-            }
-        }],
-        ['DeletingAllItems', function (newTodo, contentWindow, contentDocument) {
-            contentDocument.querySelector('#clear-completed').click();
-            var app = contentWindow.app;
-            var deleteButtons = contentDocument.querySelectorAll('.destroy');
+            if (!results)
+                return;
 
-            for (var i = 0; i < deleteButtons.length; i++) {
-                app.currentItemIndex = i;
-                app.destroy.call(deleteButtons[i]);
-            }
-        }],
-    ]
-});
+            var pre = document.createElement('pre');
+            document.body.appendChild(pre);
+            pre.textContent = results;
+        }
+    });
 
-BenchmarkRunner.suite({
-    name: 'AngularJS/TodoMVC',
-    url: 'todomvc/architecture-examples/angularjs/index.html',
-    prepare: function (contentWindow, contentDocument) {
-        return BenchmarkRunner.waitForElement('#new-todo').then(function (element) {
-            element.focus();
-            return element;
-        });
-    },
-    tests: [
-        ['Adding' + numberOfItemsToAdd + 'Items', function (newTodo, contentWindow, contentDocument) {
-            var todomvc = contentWindow.todomvc;
-            var submitEvent = document.createEvent('Event');
-            submitEvent.initEvent('submit', true, true);
-            var inputEvent = document.createEvent('Event');
-            inputEvent.initEvent('input', true, true);
-            for (var i = 0; i < numberOfItemsToAdd; i++) {
-                newTodo.value = 'Something to do ' + i;
-                newTodo.dispatchEvent(inputEvent);
-                newTodo.form.dispatchEvent(submitEvent);
-            }
-        }],
-        ['CompletingAllItems', function (newTodo, contentWindow, contentDocument) {
-            var checkboxes = contentDocument.querySelectorAll('.toggle');
-            for (var i = 0; i < checkboxes.length; i++)
-                checkboxes[i].click();
-        }],
-        ['DeletingAllItems', function (newTodo, contentWindow, contentDocument) {
-            var deleteButtons = contentDocument.querySelectorAll('.destroy');
-            for (var i = 0; i < deleteButtons.length; i++)
-                deleteButtons[i].click();
-        }],
-    ]
-});
+    var currentState = null;
+
+    // Don't call step while step is already executing.
+    var button = document.createElement('button');
+    button.textContent = 'Step';
+    button.onclick = function () {
+        self.step(currentState).then(function (state) { currentState = state; });
+    }
+    control.appendChild(button);
 
-BenchmarkRunner.suite({
-    name: 'React/TodoMVC',
-    url: 'todomvc/labs/architecture-examples/react/index.html',
-    prepare: function (contentWindow, contentDocument) {
-        contentWindow.Utils.store = function () {}
-        return BenchmarkRunner.waitForElement('#new-todo').then(function (element) {
-            element.focus();
-            return element;
+    function callNextStep(state) {
+        self.step(state).then(function (newState) {
+            currentState = newState;
+            if (newState)
+                callNextStep(newState);
         });
-    },
-    tests: [
-        ['Adding' + numberOfItemsToAdd + 'Items', function (newTodo, contentWindow, contentDocument) {
-            var todomvc = contentWindow.todomvc;
-            for (var i = 0; i < numberOfItemsToAdd; i++) {
-                newTodo.value = 'Something to do ' + i;
+    }
 
-                var keydownEvent = document.createEvent('Event');
-                keydownEvent.initEvent('keydown', true, true);
-                keydownEvent.which = 13; // VK_ENTER
-                newTodo.dispatchEvent(keydownEvent);
-            }
-        }],
-        ['CompletingAllItems', function (newTodo, contentWindow, contentDocument) {
-            var checkboxes = contentDocument.querySelectorAll('.toggle');
-            for (var i = 0; i < checkboxes.length; i++)
-                checkboxes[i].click();
-        }],
-        ['DeletingAllItems', function (newTodo, contentWindow, contentDocument) {
-            var deleteButtons = contentDocument.querySelectorAll('.destroy');
-            for (var i = 0; i < deleteButtons.length; i++)
-                deleteButtons[i].click();
-        }],
-    ]
-});
+    var button = document.createElement('button');
+    button.textContent = 'Run';
+    button.onclick = function () { callNextStep(currentState); }
+    control.appendChild(button);
 
-var actionCount = 50;
-BenchmarkRunner.suite({
-    name: 'FlightJS/MailClient',
-    url: 'flightjs-example-app/index.html',
-    prepare: function (contentWindow, contentDocument) {
-        return BenchmarkRunner.waitForElement('.span8').then(function (element) {
-            element.focus();
-            return element;
-        });
-    },
-    tests: [
-        ['OpeningTabs' + actionCount + 'Times', function (newTodo, contentWindow, contentDocument) {
-            contentDocument.getElementById('inbox').click();
-            for (var i = 0; i < actionCount; i++) {
-                contentDocument.getElementById('later').click();
-                contentDocument.getElementById('sent').click();
-                contentDocument.getElementById('trash').click();
-                contentDocument.getElementById('inbox').click();
-            }
-        }],
-        ['MovingEmails' + actionCount + 'Times', function (newTodo, contentWindow, contentDocument) {
-            contentDocument.getElementById('inbox').click();
-            for (var i = 0; i < actionCount; i++) {
-                contentDocument.getElementById('mail_2139').click();
-                contentDocument.getElementById('move_mail').click();
-                contentDocument.querySelector('#move_to_selector #later').click();
-                contentDocument.getElementById('later').click();
-                contentDocument.getElementById('mail_2139').click();
-                contentDocument.getElementById('move_mail').click();
-                contentDocument.querySelector('#move_to_selector #trash').click();
-                contentDocument.getElementById('trash').click();
-                contentDocument.getElementById('mail_2139').click();
-                contentDocument.getElementById('move_mail').click();
-                contentDocument.querySelector('#move_to_selector #inbox').click();
-                contentDocument.getElementById('inbox').click();
-            }
-        }],
-        ['Sending' + actionCount + 'NewEmails', function (newTodo, contentWindow, contentDocument) {
-            for (var i = 0; i < actionCount; i++) {
-                contentDocument.getElementById('new_mail').click();
-                contentDocument.getElementById('recipient_select').selectedIndex = 1;
-                var subject = contentDocument.getElementById('compose_subject');
-                var message = contentDocument.getElementById('compose_message');
-                subject.focus();
-                contentWindow.$(subject).trigger('keydown');
-                contentWindow.$(subject).text('Hello');
-                message.focus();
-                contentWindow.$(message).trigger('keydown');
-                contentWindow.$(message).text('Hello,\n\nThis is a test message.\n\n- WebKitten');
-                contentDocument.getElementById('send_composed').click();
-            }
-        }],
-    ]
+    document.body.appendChild(control);
 });
 
 </script>
-</head>
-<body>
 </body>
 </html>
@@ -1,4 +1,3 @@
-
 // FIXME: Use the real promise if available.
 // FIXME: Make sure this interface is compatible with the real Promise.
 function SimplePromise() {
@@ -8,7 +7,7 @@ function SimplePromise() {
 
 SimplePromise.prototype.then = function (callback) {
     if (this._callback)
-    throw "SimplePromise doesn't support multiple calls to then";
+        throw "SimplePromise doesn't support multiple calls to then";
     this._callback = callback;
     this._chainedPromise = new SimplePromise;
     
@@ -33,11 +32,19 @@ SimplePromise.prototype.resolve = function (value) {
         this._chainedPromise.resolve(result);
 }
 
-var BenchmarkRunner = {_suites: [], _prepareReturnValue: null, _measuredValues: {}};
+function BenchmarkTestStep(testName, testFunction) {
+    this.name = testName;
+    this.run = testFunction;
+}
+
+var BenchmarkRunner = {_suites: [], _prepareReturnValue: null, _measuredValues: {}, _client: null};
 
 BenchmarkRunner.suite = function (suite) {
-    var self = BenchmarkRunner;
-    self._suites.push(suite);
+    BenchmarkRunner._suites.push(suite);
+}
+
+BenchmarkRunner.setClient = function (client) {
+    BenchmarkRunner._client = client;
 }
 
 BenchmarkRunner.waitForElement = function (selector) {
@@ -67,6 +74,8 @@ BenchmarkRunner._removeFrame = function () {
 BenchmarkRunner._appendFrame = function (src) {
     var self = BenchmarkRunner;
     var frame = document.createElement('iframe');
+    frame.style.width = '800px';
+    frame.style.height = '600px'
     document.body.appendChild(frame);
     self._frame = frame;
     return frame;
@@ -120,76 +129,6 @@ BenchmarkRunner._testName = function (suite, testName, metric) {
     return suite.name + '/' + testName + (metric ? '/' + metric : '');
 }
 
-BenchmarkRunner._testItemId = function (suite, testName) {
-    return suite.name + '-' + testName;
-}
-
-BenchmarkRunner.listSuites = function () {
-    var self = BenchmarkRunner;
-
-    var control = document.createElement('nav');
-
-    var suites = self._suites;
-    var ol = document.createElement('ol');
-    var checkboxes = [];
-    for (var suiteIndex = 0; suiteIndex < suites.length; suiteIndex++) {
-        var suite = suites[suiteIndex];
-        var li = document.createElement('li');
-        var checkbox = document.createElement('input');
-        checkbox.id = suite.name;
-        checkbox.type = 'checkbox';
-        checkbox.checked = true;
-        checkboxes.push(checkbox);
-
-        li.appendChild(checkbox);
-        var label = document.createElement('label');
-        label.appendChild(document.createTextNode(self._testName(suite)));
-        li.appendChild(label);
-        label.htmlFor = checkbox.id;
-
-        var testList = document.createElement('ol');
-        for (var testIndex = 0; testIndex < suite.tests.length; testIndex++) {
-            var testItem = document.createElement('li');
-            var test = suite.tests[testIndex];
-            var anchor = document.createElement('a');
-            anchor.id = self._testItemId(suite, test[0]);
-            anchor.appendChild(document.createTextNode(self._testName(suite, test[0])));
-            testItem.appendChild(anchor);
-            testList.appendChild(testItem);
-        }
-        li.appendChild(testList);
-
-        ol.appendChild(li);
-    }
-
-    control.appendChild(ol);
-
-    var currentState = null;
-
-    // Don't call step while step is already executing.
-    var button = document.createElement('button');
-    button.textContent = 'Step';
-    button.onclick = function () {
-        self.step(currentState).then(function (state) { currentState = state; });
-    }
-    control.appendChild(button);
-
-    function callNextStep(state) {
-        self.step(state).then(function (newState) {
-            currentState = newState;
-            if (newState)
-                callNextStep(newState);
-        });
-    }
-
-    var button = document.createElement('button');
-    button.textContent = 'Run';
-    button.onclick = function () { callNextStep(currentState); }
-    control.appendChild(button);
-
-    document.body.appendChild(control);
-}
-
 function BenchmarkState(suites) {
     this._suites = suites;
     this._suiteIndex = -1;
@@ -216,7 +155,7 @@ BenchmarkState.prototype.next = function () {
     this._testIndex = 0;
     do {
         this._suiteIndex++;
-    } while (this._suiteIndex < this._suites.length && !document.getElementById(this._suites[this._suiteIndex].name).checked);
+    } while (this._suiteIndex < this._suites.length && this._suites[this._suiteIndex].disabled);
 
     return this;
 }
@@ -267,26 +206,22 @@ BenchmarkRunner._runTestAndRecordResults = function (state) {
     var suite = state.currentSuite();
     var test = state.currentTest();
 
-    var testName = test[0];
-    var testItem = document.getElementById(self._testItemId(suite, testName));
-    testItem.classList.add('running');
+    if (self._client && self._client.willRunTest)
+        self._client.willRunTest(suite, test);
+
     setTimeout(function () {
-        self._runTest(suite, test[1], self._prepareReturnValue, function (syncTime, asyncTime) {
-            self._masuredValuesForCurrentSuite[self._testName(suite, testName, 'Sync')] = syncTime;
-            self._masuredValuesForCurrentSuite[self._testName(suite, testName, 'Async')] = asyncTime;
-            testItem.classList.remove('running');
-            testItem.classList.add('ran');
+        self._runTest(suite, test.run, self._prepareReturnValue, function (syncTime, asyncTime) {
+            var suiteResults = self._measuredValues[suite.name] || {tests:{}, total: 0};
+            self._measuredValues[suite.name] = suiteResults;
+            suiteResults.tests[test.name] = {'Sync': syncTime, 'Async': asyncTime};
+            suiteResults.total += syncTime + asyncTime;
+
+            if (self._client && self._client.willRunTest)
+                self._client.didRunTest(suite, test);
+
             state.next();
-            if (state.currentSuite() != suite) {
-                var total = 0;
-                for (var title in self._masuredValuesForCurrentSuite) {
-                    var value = self._masuredValuesForCurrentSuite[title];
-                    total += value;
-                    self._measuredValues[title] = value;
-                }
-                self._measuredValues[self._testName(suite)] = total;
+            if (state.currentSuite() != suite)
                 self._removeFrame();
-            }
             promise.resolve(state);
         });
     }, 0);
@@ -296,34 +231,11 @@ BenchmarkRunner._runTestAndRecordResults = function (state) {
 BenchmarkRunner._finalize = function () {
     var self = BenchmarkRunner;
 
-    var results = '';
-    var total = 0; // FIXME: Compute the total properly.
-    for (var title in self._measuredValues) {
-        results += title + ' : ' + self._measuredValues[title] + ' ms\n';
-        total += self._measuredValues[title];
-    }
-    results += 'Total : ' + (total / 2) + ' ms\n';
-    self._measuredValues = {};
-
     self._removeFrame();
 
-    if (!results)
-        return;
+    if (self._client && self._client.didRunSuites)
+        self._client.didRunSuites(self._measuredValues);
 
-    var pre = document.createElement('pre');
-    document.body.appendChild(pre);
-    pre.textContent = results;
+    // FIXME: This should be done when we start running tests.
+    self._measuredValues = {};
 }
-
-window.addEventListener('load', function () { BenchmarkRunner.listSuites(); });
-
-(function () {
-    var style = document.createElement('style');
-    style.appendChild(document.createTextNode('iframe { width: 1000px; height: 500px; border: 2px solid black; }'
-        + 'ol { list-style: none; margin: 0; padding: 0; }'
-        + 'ol ol { margin-left: 2em; list-position: outside; }'
-        + '.running { text-decoration: underline; }'
-        + '.ran {color: grey}'
-        + 'nav { position: absolute; right: 10px; }'));
-    document.head.appendChild(style);
-})();
diff --git a/PerformanceTests/DoYouEvenBench/resources/tests.js b/PerformanceTests/DoYouEvenBench/resources/tests.js
new file mode 100644 (file)
index 0000000..597d892
--- /dev/null
@@ -0,0 +1,284 @@
+var numberOfItemsToAdd = 100;
+
+BenchmarkRunner.suite({
+    name: 'VanillaJS-TodoMVC',
+    url: 'todomvc/vanilla-examples/vanillajs/index.html',
+    prepare: function (contentWindow, contentDocument) {
+        return BenchmarkRunner.waitForElement('#new-todo').then(function (element) {
+            element.focus();
+            return element;
+        });
+    },
+    tests: [
+        new BenchmarkTestStep('Adding' + numberOfItemsToAdd + 'Items', function (newTodo, contentWindow, contentDocument) {
+            var todoController = contentWindow.todo.controller;
+            for (var i = 0; i < numberOfItemsToAdd; i++) {
+                newTodo.value = 'Something to do ' + i;
+                todoController.addItem({keyCode: todoController.ENTER_KEY, target: newTodo});
+            }
+        }),
+        new BenchmarkTestStep('CompletingAllItems', function (newTodo, contentWindow, contentDocument) {
+            var checkboxes = contentDocument.querySelectorAll('.toggle');
+            for (var i = 0; i < checkboxes.length; i++)
+                checkboxes[i].click();
+        }),
+        new BenchmarkTestStep('DeletingAllItems', function (newTodo, contentWindow, contentDocument) {
+            var deleteButtons = contentDocument.querySelectorAll('.destroy');
+            for (var i = 0; i < deleteButtons.length; i++)
+                deleteButtons[i].click();
+        }),
+    ]
+});
+
+BenchmarkRunner.suite({
+    name: 'EmberJS-TodoMVC',
+    url: 'todomvc/architecture-examples/emberjs/index.html',
+    prepare: function (contentWindow, contentDocument) {
+        contentWindow.Todos.Store = contentWindow.DS.Store.extend({
+            revision: 12,
+            adapter: 'Todos.LSAdapter',
+            commit: function () { }
+        });
+
+        return BenchmarkRunner.waitForElement('#new-todo').then(function (element) {
+            element.focus();
+            return {
+                views: contentWindow.Ember.View.views,
+                emberRun: contentWindow.Ember.run,
+            }
+        });
+    },
+    tests: [
+        new BenchmarkTestStep('Adding' + numberOfItemsToAdd + 'Items', function (params) {
+            for (var i = 0; i < numberOfItemsToAdd; i++) {
+                params.emberRun(function () { params.views["new-todo"].set('value', 'Something to do' + i); });
+                params.emberRun(function () { params.views["new-todo"].insertNewline(document.createEvent('Event')); });
+            }
+        }),
+        new BenchmarkTestStep('CompletingAllItems', function (params, contentWindow, contentDocument) {
+            var checkboxes = contentDocument.querySelectorAll('.ember-checkbox');
+            for (var i = 0; i < checkboxes.length; i++) {
+                var view = params.views[checkboxes[i].id];
+                params.emberRun(function () { view.set('checked', true); });
+            }
+        }),
+        new BenchmarkTestStep('DeletingItems', function (params, contentWindow, contentDocument) {
+            var deleteButtons = contentDocument.querySelectorAll('.destroy');
+            for (var i = 0; i < deleteButtons.length; i++)
+                params.emberRun(function () { deleteButtons[i].click(); });
+        }),
+    ]
+});
+
+BenchmarkRunner.suite({
+    name: 'BackboneJS-TodoMVC',
+    url: 'todomvc/architecture-examples/backbone/index.html',
+    prepare: function (contentWindow, contentDocument) {
+    contentWindow.Backbone.sync = function () {}
+        return BenchmarkRunner.waitForElement('#new-todo').then(function (element) {
+            element.focus();
+            return element;
+        });
+    },
+    tests: [
+        new BenchmarkTestStep('Adding' + numberOfItemsToAdd + 'Items', function (newTodo, contentWindow, contentDocument) {
+            var appView = contentWindow.appView;
+            var fakeEvent = {which: contentWindow.ENTER_KEY};
+            for (var i = 0; i < numberOfItemsToAdd; i++) {
+                newTodo.value = 'Something to do ' + i;
+                appView.createOnEnter(fakeEvent);
+            }
+        }),
+        new BenchmarkTestStep('CompletingAllItems', function (newTodo, contentWindow, contentDocument) {
+            var checkboxes = contentDocument.querySelectorAll('.toggle');
+            for (var i = 0; i < checkboxes.length; i++)
+                checkboxes[i].click();
+        }),
+        new BenchmarkTestStep('DeletingAllItems', function (newTodo, contentWindow, contentDocument) {
+            var deleteButtons = contentDocument.querySelectorAll('.destroy');
+            for (var i = 0; i < deleteButtons.length; i++)
+                deleteButtons[i].click();
+        }),
+    ]
+});
+
+BenchmarkRunner.suite({
+    name: 'jQuery-TodoMVC',
+    url: 'todomvc/architecture-examples/jquery/index.html',
+    prepare: function (contentWindow, contentDocument) {
+        return BenchmarkRunner.waitForElement('#new-todo').then(function (element) {
+            element.focus();
+            return element;
+        });
+    },
+    tests: [
+        new BenchmarkTestStep('Adding' + numberOfItemsToAdd + 'Items', function (newTodo, contentWindow, contentDocument) {
+            var app = contentWindow.app;
+            var fakeEvent = {which: app.ENTER_KEY};
+            for (var i = 0; i < numberOfItemsToAdd; i++) {
+                newTodo.value = 'Something to do ' + i;
+                app.create.call(newTodo, fakeEvent);
+            }
+        }),
+        new BenchmarkTestStep('CompletingAllItems', function (newTodo, contentWindow, contentDocument) {
+            var app = contentWindow.app;
+            var checkboxes = contentDocument.querySelectorAll('.toggle');
+            var $ = contentWindow.$;
+
+            itemIndexToId = new Array(checkboxes.length);
+            for (var i = 0; i < checkboxes.length; i++)
+                itemIndexToId[i] = $(checkboxes[i]).closest('li').data('id');
+
+            app.getTodo = function (element, callback) {
+                var self = this;
+                var id = itemIndexToId[this.currentItemIndex];
+                $.each(this.todos, function (j, val) {
+                    if (val.id === id) {
+                        callback.apply(self, arguments);
+                        return false;
+                    }
+                });
+            }
+
+            for (var i = 0; i < checkboxes.length; i++) {
+                app.currentItemIndex = i;
+                app.toggle.call(checkboxes[i]);
+            }
+        }),
+        new BenchmarkTestStep('DeletingAllItems', function (newTodo, contentWindow, contentDocument) {
+            contentDocument.querySelector('#clear-completed').click();
+            var app = contentWindow.app;
+            var deleteButtons = contentDocument.querySelectorAll('.destroy');
+
+            for (var i = 0; i < deleteButtons.length; i++) {
+                app.currentItemIndex = i;
+                app.destroy.call(deleteButtons[i]);
+            }
+        }),
+    ]
+});
+
+BenchmarkRunner.suite({
+    name: 'AngularJS-TodoMVC',
+    url: 'todomvc/architecture-examples/angularjs/index.html',
+    prepare: function (contentWindow, contentDocument) {
+        return BenchmarkRunner.waitForElement('#new-todo').then(function (element) {
+            element.focus();
+            return element;
+        });
+    },
+    tests: [
+        new BenchmarkTestStep('Adding' + numberOfItemsToAdd + 'Items', function (newTodo, contentWindow, contentDocument) {
+            var todomvc = contentWindow.todomvc;
+            var submitEvent = document.createEvent('Event');
+            submitEvent.initEvent('submit', true, true);
+            var inputEvent = document.createEvent('Event');
+            inputEvent.initEvent('input', true, true);
+            for (var i = 0; i < numberOfItemsToAdd; i++) {
+                newTodo.value = 'Something to do ' + i;
+                newTodo.dispatchEvent(inputEvent);
+                newTodo.form.dispatchEvent(submitEvent);
+            }
+        }),
+        new BenchmarkTestStep('CompletingAllItems', function (newTodo, contentWindow, contentDocument) {
+            var checkboxes = contentDocument.querySelectorAll('.toggle');
+            for (var i = 0; i < checkboxes.length; i++)
+                checkboxes[i].click();
+        }),
+        new BenchmarkTestStep('DeletingAllItems', function (newTodo, contentWindow, contentDocument) {
+            var deleteButtons = contentDocument.querySelectorAll('.destroy');
+            for (var i = 0; i < deleteButtons.length; i++)
+                deleteButtons[i].click();
+        }),
+    ]
+});
+
+BenchmarkRunner.suite({
+    name: 'React-TodoMVC',
+    url: 'todomvc/labs/architecture-examples/react/index.html',
+    prepare: function (contentWindow, contentDocument) {
+        contentWindow.Utils.store = function () {}
+        return BenchmarkRunner.waitForElement('#new-todo').then(function (element) {
+            element.focus();
+            return element;
+        });
+    },
+    tests: [
+        new BenchmarkTestStep('Adding' + numberOfItemsToAdd + 'Items', function (newTodo, contentWindow, contentDocument) {
+            var todomvc = contentWindow.todomvc;
+            for (var i = 0; i < numberOfItemsToAdd; i++) {
+                newTodo.value = 'Something to do ' + i;
+
+                var keydownEvent = document.createEvent('Event');
+                keydownEvent.initEvent('keydown', true, true);
+                keydownEvent.which = 13; // VK_ENTER
+                newTodo.dispatchEvent(keydownEvent);
+            }
+        }),
+        new BenchmarkTestStep('CompletingAllItems', function (newTodo, contentWindow, contentDocument) {
+            var checkboxes = contentDocument.querySelectorAll('.toggle');
+            for (var i = 0; i < checkboxes.length; i++)
+                checkboxes[i].click();
+        }),
+        new BenchmarkTestStep('DeletingAllItems', function (newTodo, contentWindow, contentDocument) {
+            var deleteButtons = contentDocument.querySelectorAll('.destroy');
+            for (var i = 0; i < deleteButtons.length; i++)
+                deleteButtons[i].click();
+        }),
+    ]
+});
+
+var actionCount = 50;
+BenchmarkRunner.suite({
+    name: 'FlightJS-MailClient',
+    url: 'flightjs-example-app/index.html',
+    prepare: function (contentWindow, contentDocument) {
+        return BenchmarkRunner.waitForElement('.span8').then(function (element) {
+            element.focus();
+            return element;
+        });
+    },
+    tests: [
+        new BenchmarkTestStep('OpeningTabs' + actionCount + 'Times', function (newTodo, contentWindow, contentDocument) {
+            contentDocument.getElementById('inbox').click();
+            for (var i = 0; i < actionCount; i++) {
+                contentDocument.getElementById('later').click();
+                contentDocument.getElementById('sent').click();
+                contentDocument.getElementById('trash').click();
+                contentDocument.getElementById('inbox').click();
+            }
+        }),
+        new BenchmarkTestStep('MovingEmails' + actionCount + 'Times', function (newTodo, contentWindow, contentDocument) {
+            contentDocument.getElementById('inbox').click();
+            for (var i = 0; i < actionCount; i++) {
+                contentDocument.getElementById('mail_2139').click();
+                contentDocument.getElementById('move_mail').click();
+                contentDocument.querySelector('#move_to_selector #later').click();
+                contentDocument.getElementById('later').click();
+                contentDocument.getElementById('mail_2139').click();
+                contentDocument.getElementById('move_mail').click();
+                contentDocument.querySelector('#move_to_selector #trash').click();
+                contentDocument.getElementById('trash').click();
+                contentDocument.getElementById('mail_2139').click();
+                contentDocument.getElementById('move_mail').click();
+                contentDocument.querySelector('#move_to_selector #inbox').click();
+                contentDocument.getElementById('inbox').click();
+            }
+        }),
+        new BenchmarkTestStep('Sending' + actionCount + 'NewEmails', function (newTodo, contentWindow, contentDocument) {
+            for (var i = 0; i < actionCount; i++) {
+                contentDocument.getElementById('new_mail').click();
+                contentDocument.getElementById('recipient_select').selectedIndex = 1;
+                var subject = contentDocument.getElementById('compose_subject');
+                var message = contentDocument.getElementById('compose_message');
+                subject.focus();
+                contentWindow.$(subject).trigger('keydown');
+                contentWindow.$(subject).text('Hello');
+                message.focus();
+                contentWindow.$(message).trigger('keydown');
+                contentWindow.$(message).text('Hello,\n\nThis is a test message.\n\n- WebKitten');
+                contentDocument.getElementById('send_composed').click();
+            }
+        }),
+    ]
+});