2011-02-08 Dirk Pranke <dpranke@chromium.org>
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / layout_package / single_test_runner.py
1 # Copyright (C) 2011 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29
30 import logging
31 import threading
32 import time
33
34 from webkitpy.layout_tests.port import base
35
36 from webkitpy.layout_tests.test_types import text_diff
37 from webkitpy.layout_tests.test_types import image_diff
38
39 from webkitpy.layout_tests.layout_package import test_failures
40 from webkitpy.layout_tests.layout_package.test_results import TestResult
41
42
43 _log = logging.getLogger(__name__)
44
45
46 class ExpectedDriverOutput:
47     """Groups information about an expected driver output."""
48     def __init__(self, text, image, image_hash):
49         self.text = text
50         self.image = image
51         self.image_hash = image_hash
52
53
54 class SingleTestRunner:
55
56     def __init__(self, options, port, worker_name, worker_number):
57         self._options = options
58         self._port = port
59         self._worker_name = worker_name
60         self._worker_number = worker_number
61         self._driver = None
62         self._test_types = []
63         for cls in self._get_test_type_classes():
64             self._test_types.append(cls(self._port,
65                                         self._options.results_directory))
66
67     def cleanup(self):
68         self.kill_dump_render_tree()
69
70     def _get_test_type_classes(self):
71         classes = [text_diff.TestTextDiff]
72         if self._options.pixel_tests:
73             classes.append(image_diff.ImageDiff)
74         return classes
75
76     def timeout(self, test_input):
77         # We calculate how long we expect the test to take.
78         #
79         # The DumpRenderTree watchdog uses 2.5x the timeout; we want to be
80         # larger than that. We also add a little more padding if we're
81         # running tests in a separate thread.
82         #
83         # Note that we need to convert the test timeout from a
84         # string value in milliseconds to a float for Python.
85         driver_timeout_sec = 3.0 * float(test_input.timeout) / 1000.0
86         if not self._options.run_singly:
87             return driver_timeout_sec
88
89         thread_padding_sec = 1.0
90         thread_timeout_sec = driver_timeout_sec + thread_padding_sec
91         return thread_timeout_sec
92
93     def run_test(self, test_input, timeout):
94         if self._options.run_singly:
95             return self._run_test_in_another_thread(test_input, timeout)
96         else:
97             return self._run_test_in_this_thread(test_input)
98         return result
99
100     def _run_test_in_another_thread(self, test_input, thread_timeout_sec):
101         """Run a test in a separate thread, enforcing a hard time limit.
102
103         Since we can only detect the termination of a thread, not any internal
104         state or progress, we can only run per-test timeouts when running test
105         files singly.
106
107         Args:
108           test_input: Object containing the test filename and timeout
109           thread_timeout_sec: time to wait before killing the driver process.
110         Returns:
111           A TestResult
112         """
113         worker = self
114         result = None
115
116         driver = worker._port.create_driver(worker._worker_number)
117         driver.start()
118
119         class SingleTestThread(threading.Thread):
120             def run(self):
121                 result = worker.run(test_input, driver)
122
123         thread = SingleTestThread()
124         thread.start()
125         thread.join(thread_timeout_sec)
126         if thread.isAlive():
127             # If join() returned with the thread still running, the
128             # DumpRenderTree is completely hung and there's nothing
129             # more we can do with it.  We have to kill all the
130             # DumpRenderTrees to free it up. If we're running more than
131             # one DumpRenderTree thread, we'll end up killing the other
132             # DumpRenderTrees too, introducing spurious crashes. We accept
133             # that tradeoff in order to avoid losing the rest of this
134             # thread's results.
135             _log.error('Test thread hung: killing all DumpRenderTrees')
136
137         driver.stop()
138
139         if not result:
140             result = TestResult(test_input.filename, failures=[],
141                 test_run_time=0, total_time_for_all_diffs=0, time_for_diffs={})
142         return result
143
144     def _run_test_in_this_thread(self, test_input):
145         """Run a single test file using a shared DumpRenderTree process.
146
147         Args:
148           test_input: Object containing the test filename, uri and timeout
149
150         Returns: a TestResult object.
151         """
152         # poll() is not threadsafe and can throw OSError due to:
153         # http://bugs.python.org/issue1731717
154         if not self._driver or self._driver.poll() is not None:
155             self._driver = self._port.create_driver(self._worker_number)
156             self._driver.start()
157         return self._run(self._driver, test_input)
158
159     def _expected_driver_output(self):
160         return ExpectedDriverOutput(self._port.expected_text(self._filename),
161                                     self._port.expected_image(self._filename),
162                                     self._port.expected_checksum(self._filename))
163
164     def _should_fetch_expected_checksum(self):
165         return (self._options.pixel_tests and
166                 not (self._options.new_baseline or self._options.reset_results))
167
168     def _driver_input(self, test_input):
169         self._filename = test_input.filename
170         self._timeout = test_input.timeout
171         self._testname = self._port.relative_test_filename(test_input.filename)
172
173         # The image hash is used to avoid doing an image dump if the
174         # checksums match, so it should be set to a blank value if we
175         # are generating a new baseline.  (Otherwise, an image from a
176         # previous run will be copied into the baseline."""
177         image_hash = None
178         if self._should_fetch_expected_checksum():
179             image_hash = self._port.expected_checksum(self._filename)
180         return base.DriverInput(self._filename, self._timeout, image_hash)
181
182     def _run(self, driver, test_input):
183         driver_output = self._driver.run_test(self._driver_input(test_input))
184         return self._process_output(driver_output)
185
186     def _process_output(self, driver_output):
187         """Receives the output from a DumpRenderTree process, subjects it to a
188         number of tests, and returns a list of failure types the test produced.
189         Args:
190           driver_output: a DriverOutput object containing the output from the driver
191
192         Returns: a TestResult object
193         """
194         failures = []
195         fs = self._port._filesystem
196
197         if driver_output.crash:
198             failures.append(test_failures.FailureCrash())
199         if driver_output.timeout:
200             failures.append(test_failures.FailureTimeout())
201
202         if driver_output.crash:
203             _log.debug("%s Stacktrace for %s:\n%s" % (self._worker_name, self._testname,
204                                                       driver_output.error))
205             stack_filename = fs.join(self._options.results_directory, self._testname)
206             stack_filename = fs.splitext(stack_filename)[0] + "-stack.txt"
207             fs.maybe_make_directory(fs.dirname(stack_filename))
208             fs.write_text_file(stack_filename, driver_output.error)
209         elif driver_output.error:
210             _log.debug("%s %s output stderr lines:\n%s" % (self._worker_name, self._testname,
211                                                            driver_output.error))
212
213         expected_driver_output = self._expected_driver_output()
214
215         # Check the output and save the results.
216         start_time = time.time()
217         time_for_diffs = {}
218         for test_type in self._test_types:
219             start_diff_time = time.time()
220             new_failures = test_type.compare_output(
221                 self._port, self._filename, self._options, driver_output,
222                 expected_driver_output)
223             # Don't add any more failures if we already have a crash, so we don't
224             # double-report those tests. We do double-report for timeouts since
225             # we still want to see the text and image output.
226             if not driver_output.crash:
227                 failures.extend(new_failures)
228             time_for_diffs[test_type.__class__.__name__] = (
229                 time.time() - start_diff_time)
230
231         total_time_for_all_diffs = time.time() - start_diff_time
232         return TestResult(self._filename, failures, driver_output.test_time,
233                           total_time_for_all_diffs, time_for_diffs)
234
235     def kill_dump_render_tree(self):
236         """Kill the DumpRenderTree process if it's running."""
237         if self._driver:
238             self._driver.stop()
239             self._driver = None