Web Inspector: chromium: I'd like to add a script for running perf tests for WebInspe...
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftestsrunner_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2011 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Unit tests for run_perf_tests."""
31
32 import unittest
33
34 from webkitpy.common import array_stream
35 from webkitpy.common.system.filesystem_mock import MockFileSystem
36 from webkitpy.layout_tests.port.driver import DriverInput, DriverOutput
37 from webkitpy.layout_tests.views import printing
38 from webkitpy.performance_tests.perftestsrunner import PerfTestsRunner
39
40
41 class MainTest(unittest.TestCase):
42     class TestPort:
43         def create_driver(self, worker_number=None):
44             return MainTest.TestDriver()
45
46     class TestDriver:
47         def run_test(self, driver_input):
48             text = ''
49             timeout = False
50             crash = False
51             if driver_input.test_name == 'pass.html':
52                 text = 'RESULT group_name: test_name= 42 ms'
53             elif driver_input.test_name == 'timeout.html':
54                 timeout = True
55             elif driver_input.test_name == 'failed.html':
56                 text = None
57             elif driver_input.test_name == 'tonguey.html':
58                 text = 'we are not expecting an output from perf tests but RESULT blablabla'
59             elif driver_input.test_name == 'crash.html':
60                 crash = True
61             return DriverOutput(text, '', '', '', crash=crash, timeout=timeout)
62
63         def stop(self):
64             """do nothing"""
65
66     def create_runner(self, buildbot_output=None):
67         buildbot_output = buildbot_output or array_stream.ArrayStream()
68         regular_output = array_stream.ArrayStream()
69         return PerfTestsRunner('', regular_output, buildbot_output, args=[])
70
71     def run_test(self, test_name):
72         runner = self.create_runner()
73         driver = MainTest.TestDriver()
74         return runner._run_single_test(test_name, driver)
75
76     def test_run_passing_test(self):
77         test_failed, driver_need_restart = self.run_test('pass.html')
78         self.assertFalse(test_failed)
79         self.assertFalse(driver_need_restart)
80
81     def test_run_silent_test(self):
82         test_failed, driver_need_restart = self.run_test('silent.html')
83         self.assertTrue(test_failed)
84         self.assertFalse(driver_need_restart)
85
86     def test_run_failed_test(self):
87         test_failed, driver_need_restart = self.run_test('failed.html')
88         self.assertTrue(test_failed)
89         self.assertFalse(driver_need_restart)
90
91     def test_run_tonguey_test(self):
92         test_failed, driver_need_restart = self.run_test('tonguey.html')
93         self.assertTrue(test_failed)
94         self.assertFalse(driver_need_restart)
95
96     def test_run_timeout_test(self):
97         test_failed, driver_need_restart = self.run_test('timeout.html')
98         self.assertTrue(test_failed)
99         self.assertTrue(driver_need_restart)
100
101     def test_run_crash_test(self):
102         test_failed, driver_need_restart = self.run_test('crash.html')
103         self.assertTrue(test_failed)
104         self.assertTrue(driver_need_restart)
105
106     def test_run_test_set(self):
107         buildbot_output = array_stream.ArrayStream()
108         runner = self.create_runner(buildbot_output)
109         port = MainTest.TestPort()
110         tests = ['pass.html', 'silent.html', 'failed.html', 'tonguey.html', 'timeout.html', 'crash.html']
111         unexpected_result_count = runner._run_tests_set(tests, port)
112         self.assertEqual(unexpected_result_count, len(tests) - 1)
113         self.assertEqual(len(buildbot_output.get()), 1)
114         self.assertEqual(buildbot_output.get()[0], 'RESULT group_name: test_name= 42 ms\n')
115
116     def test_collect_tests(self):
117         runner = self.create_runner()
118         webkit_base = '/test.checkout'
119         filesystem = MockFileSystem()
120         filename = filesystem.join(webkit_base, 'PerformanceTests', 'a_file.html')
121         filesystem.files[filename] = 'a content'
122         tests = runner._collect_tests(webkit_base, filesystem)
123         self.assertEqual(len(tests), 1)
124
125     def test_parse_args(self):
126         runner = self.create_runner()
127         options, args = runner._parse_args([
128                 '--verbose',
129                 '--build-directory=folder42',
130                 '--platform=platform42',
131                 '--time-out-ms=42',
132                 '--debug', 'an_arg'])
133         self.assertEqual(options.verbose, True)
134         self.assertEqual(options.help_printing, None)
135         self.assertEqual(options.build_directory, 'folder42')
136         self.assertEqual(options.platform, 'platform42')
137         self.assertEqual(options.time_out_ms, '42')
138         self.assertEqual(options.configuration, 'Debug')
139         self.assertEqual(options.print_options, None)
140
141
142 if __name__ == '__main__':
143     unittest.main()