Need a script to run tests in PerformanceTests
[WebKit-https.git] / Tools / Scripts / webkitpy / performance_tests / perftestsrunner_unittest.py
1 #!/usr/bin/python
2 # Copyright (C) 2011 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Unit tests for run_perf_tests."""
31
32 import unittest
33
34 from webkitpy.common import array_stream
35 from webkitpy.common.system.filesystem_mock import MockFileSystem
36 from webkitpy.layout_tests.port.driver import DriverInput, DriverOutput
37 from webkitpy.layout_tests.views import printing
38 from webkitpy.performance_tests.perftestsrunner import PerfTestsRunner
39
40
41 class MainTest(unittest.TestCase):
42     class TestPort:
43         def create_driver(self, worker_number=None):
44             return MainTest.TestDriver()
45
46     class TestDriver:
47         def run_test(self, driver_input):
48             text = ''
49             timeout = False
50             crash = False
51             if driver_input.test_name.endswith('pass.html'):
52                 text = 'RESULT group_name: test_name= 42 ms'
53             elif driver_input.test_name.endswith('timeout.html'):
54                 timeout = True
55             elif driver_input.test_name.endswith('failed.html'):
56                 text = None
57             elif driver_input.test_name.endswith('tonguey.html'):
58                 text = 'we are not expecting an output from perf tests but RESULT blablabla'
59             elif driver_input.test_name.endswith('crash.html'):
60                 crash = True
61             elif driver_input.test_name.endswith('event-target-wrapper.html'):
62                 text = """Running 20 times
63 Ignoring warm-up run (1502)
64 1504
65 1505
66 1510
67 1504
68 1507
69 1509
70 1510
71 1487
72 1488
73 1472
74 1472
75 1488
76 1473
77 1472
78 1475
79 1487
80 1486
81 1486
82 1475
83 1471
84
85 avg 1489.05
86 median 1487
87 stdev 14.46
88 min 1471
89 max 1510
90 """
91             elif driver_input.test_name.endswith('some-parser.html'):
92                 text = """Running 20 times
93 Ignoring warm-up run (1115)
94
95 avg 1100
96 median 1101
97 stdev 11
98 min 1080
99 max 1120
100 """
101             return DriverOutput(text, '', '', '', crash=crash, timeout=timeout)
102
103         def stop(self):
104             """do nothing"""
105
106     def create_runner(self, buildbot_output=None):
107         buildbot_output = buildbot_output or array_stream.ArrayStream()
108         regular_output = array_stream.ArrayStream()
109         return PerfTestsRunner(regular_output, buildbot_output, args=[])
110
111     def run_test(self, test_name):
112         runner = self.create_runner()
113         driver = MainTest.TestDriver()
114         return runner._run_single_test(test_name, driver, is_chromium_style=True)
115
116     def test_run_passing_test(self):
117         test_failed, driver_need_restart = self.run_test('pass.html')
118         self.assertFalse(test_failed)
119         self.assertFalse(driver_need_restart)
120
121     def test_run_silent_test(self):
122         test_failed, driver_need_restart = self.run_test('silent.html')
123         self.assertTrue(test_failed)
124         self.assertFalse(driver_need_restart)
125
126     def test_run_failed_test(self):
127         test_failed, driver_need_restart = self.run_test('failed.html')
128         self.assertTrue(test_failed)
129         self.assertFalse(driver_need_restart)
130
131     def test_run_tonguey_test(self):
132         test_failed, driver_need_restart = self.run_test('tonguey.html')
133         self.assertTrue(test_failed)
134         self.assertFalse(driver_need_restart)
135
136     def test_run_timeout_test(self):
137         test_failed, driver_need_restart = self.run_test('timeout.html')
138         self.assertTrue(test_failed)
139         self.assertTrue(driver_need_restart)
140
141     def test_run_crash_test(self):
142         test_failed, driver_need_restart = self.run_test('crash.html')
143         self.assertTrue(test_failed)
144         self.assertTrue(driver_need_restart)
145
146     def test_run_test_set(self):
147         buildbot_output = array_stream.ArrayStream()
148         runner = self.create_runner(buildbot_output)
149         runner._base_path = '/test.checkout/PerformanceTests'
150         port = MainTest.TestPort()
151         dirname = runner._base_path + '/inspector/'
152         tests = [dirname + 'pass.html', dirname + 'silent.html', dirname + 'failed.html',
153             dirname + 'tonguey.html', dirname + 'timeout.html', dirname + 'crash.html']
154         unexpected_result_count = runner._run_tests_set(tests, port)
155         self.assertEqual(unexpected_result_count, len(tests) - 1)
156         self.assertEqual(len(buildbot_output.get()), 1)
157         self.assertEqual(buildbot_output.get()[0], 'RESULT group_name: test_name= 42 ms\n')
158
159     def test_run_test_set_for_parser_tests(self):
160         buildbot_output = array_stream.ArrayStream()
161         runner = self.create_runner(buildbot_output)
162         runner._base_path = '/test.checkout/PerformanceTests/'
163         port = MainTest.TestPort()
164         tests = [runner._base_path + 'Bindings/event-target-wrapper.html', runner._base_path + 'Parser/some-parser.html']
165         unexpected_result_count = runner._run_tests_set(tests, port)
166         self.assertEqual(unexpected_result_count, 0)
167         self.assertEqual(buildbot_output.get()[0], 'RESULT Bindings: event-target-wrapper= 1489.05 ms\n')
168         self.assertEqual(buildbot_output.get()[1], 'median= 1487 ms, stdev= 14.46 ms, min= 1471 ms, max= 1510 ms\n')
169         self.assertEqual(buildbot_output.get()[2], 'RESULT Parser: some-parser= 1100 ms\n')
170         self.assertEqual(buildbot_output.get()[3], 'median= 1101 ms, stdev= 11 ms, min= 1080 ms, max= 1120 ms\n')
171
172     def test_collect_tests(self):
173         runner = self.create_runner()
174         runner._base_path = '/test.checkout/PerformanceTests'
175         filesystem = MockFileSystem()
176         filename = filesystem.join(runner._base_path, 'inspector', 'a_file.html')
177         filesystem.maybe_make_directory(runner._base_path, 'inspector')
178         filesystem.files[filename] = 'a content'
179         runner._host.filesystem = filesystem
180         tests = runner._collect_tests()
181         self.assertEqual(len(tests), 1)
182
183     def test_parse_args(self):
184         runner = self.create_runner()
185         options, args = runner._parse_args([
186                 '--verbose',
187                 '--build-directory=folder42',
188                 '--platform=platform42',
189                 '--time-out-ms=42',
190                 '--debug', 'an_arg'])
191         self.assertEqual(options.verbose, True)
192         self.assertEqual(options.help_printing, None)
193         self.assertEqual(options.build_directory, 'folder42')
194         self.assertEqual(options.platform, 'platform42')
195         self.assertEqual(options.time_out_ms, '42')
196         self.assertEqual(options.configuration, 'Debug')
197         self.assertEqual(options.print_options, None)
198
199
200 if __name__ == '__main__':
201     unittest.main()