2010-04-21 Eric Seidel <eric@webkit.org>
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / layout_tests / layout_package / dump_render_tree_thread.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """A Thread object for running DumpRenderTree and processing URLs from a
31 shared queue.
32
33 Each thread runs a separate instance of the DumpRenderTree binary and validates
34 the output.  When there are no more URLs to process in the shared queue, the
35 thread exits.
36 """
37
38 import copy
39 import logging
40 import os
41 import Queue
42 import signal
43 import sys
44 import thread
45 import threading
46 import time
47
48 import test_failures
49
50 _log = logging.getLogger("webkitpy.layout_tests.layout_package."
51                          "dump_render_tree_thread")
52
53
54 def process_output(port, test_info, test_types, test_args, configuration,
55                    output_dir, crash, timeout, test_run_time, actual_checksum,
56                    output, error):
57     """Receives the output from a DumpRenderTree process, subjects it to a
58     number of tests, and returns a list of failure types the test produced.
59
60     Args:
61       port: port-specific hooks
62       proc: an active DumpRenderTree process
63       test_info: Object containing the test filename, uri and timeout
64       test_types: list of test types to subject the output to
65       test_args: arguments to be passed to each test
66       configuration: Debug or Release
67       output_dir: directory to put crash stack traces into
68
69     Returns: a TestResult object
70     """
71     failures = []
72
73     # Some test args, such as the image hash, may be added or changed on a
74     # test-by-test basis.
75     local_test_args = copy.copy(test_args)
76
77     local_test_args.hash = actual_checksum
78
79     if crash:
80         failures.append(test_failures.FailureCrash())
81     if timeout:
82         failures.append(test_failures.FailureTimeout())
83
84     if crash:
85         _log.debug("Stacktrace for %s:\n%s" % (test_info.filename, error))
86         # Strip off "file://" since RelativeTestFilename expects
87         # filesystem paths.
88         filename = os.path.join(output_dir, port.relative_test_filename(
89                                 test_info.filename))
90         filename = os.path.splitext(filename)[0] + "-stack.txt"
91         port.maybe_make_directory(os.path.split(filename)[0])
92         open(filename, "wb").write(error)  # FIXME: This leaks a file handle.
93     elif error:
94         _log.debug("Previous test output extra lines after dump:\n%s" %
95                    error)
96
97     # Check the output and save the results.
98     start_time = time.time()
99     time_for_diffs = {}
100     for test_type in test_types:
101         start_diff_time = time.time()
102         new_failures = test_type.compare_output(port, test_info.filename,
103                                                 output, local_test_args,
104                                                 configuration)
105         # Don't add any more failures if we already have a crash, so we don't
106         # double-report those tests. We do double-report for timeouts since
107         # we still want to see the text and image output.
108         if not crash:
109             failures.extend(new_failures)
110         time_for_diffs[test_type.__class__.__name__] = (
111             time.time() - start_diff_time)
112
113     total_time_for_all_diffs = time.time() - start_diff_time
114     return TestResult(test_info.filename, failures, test_run_time,
115                       total_time_for_all_diffs, time_for_diffs)
116
117
118 class TestResult(object):
119
120     def __init__(self, filename, failures, test_run_time,
121                  total_time_for_all_diffs, time_for_diffs):
122         self.failures = failures
123         self.filename = filename
124         self.test_run_time = test_run_time
125         self.time_for_diffs = time_for_diffs
126         self.total_time_for_all_diffs = total_time_for_all_diffs
127         self.type = test_failures.determine_result_type(failures)
128
129
130 class SingleTestThread(threading.Thread):
131     """Thread wrapper for running a single test file."""
132
133     def __init__(self, port, image_path, shell_args, test_info,
134         test_types, test_args, configuration, output_dir):
135         """
136         Args:
137           port: object implementing port-specific hooks
138           test_info: Object containing the test filename, uri and timeout
139           output_dir: Directory to put crash stacks into.
140           See TestShellThread for documentation of the remaining arguments.
141         """
142
143         threading.Thread.__init__(self)
144         self._port = port
145         self._image_path = image_path
146         self._shell_args = shell_args
147         self._test_info = test_info
148         self._test_types = test_types
149         self._test_args = test_args
150         self._configuration = configuration
151         self._output_dir = output_dir
152
153     def run(self):
154         test_info = self._test_info
155         driver = self._port.start_driver(self._image_path, self._shell_args)
156         start = time.time()
157         crash, timeout, actual_checksum, output, error = \
158             driver.run_test(test_info.uri.strip(), test_info.timeout,
159                             test_info.image_hash())
160         end = time.time()
161         self._test_result = process_output(self._port,
162             test_info, self._test_types, self._test_args,
163             self._configuration, self._output_dir, crash, timeout, end - start,
164             actual_checksum, output, error)
165         driver.stop()
166
167     def get_test_result(self):
168         return self._test_result
169
170
171 class TestShellThread(threading.Thread):
172
173     def __init__(self, port, filename_list_queue, result_queue,
174                  test_types, test_args, image_path, shell_args, options):
175         """Initialize all the local state for this DumpRenderTree thread.
176
177         Args:
178           port: interface to port-specific hooks
179           filename_list_queue: A thread safe Queue class that contains lists
180               of tuples of (filename, uri) pairs.
181           result_queue: A thread safe Queue class that will contain tuples of
182               (test, failure lists) for the test results.
183           test_types: A list of TestType objects to run the test output
184               against.
185           test_args: A TestArguments object to pass to each TestType.
186           shell_args: Any extra arguments to be passed to DumpRenderTree.
187           options: A property dictionary as produced by optparse. The
188               command-line options should match those expected by
189               run_webkit_tests; they are typically passed via the
190               run_webkit_tests.TestRunner class."""
191         threading.Thread.__init__(self)
192         self._port = port
193         self._filename_list_queue = filename_list_queue
194         self._result_queue = result_queue
195         self._filename_list = []
196         self._test_types = test_types
197         self._test_args = test_args
198         self._driver = None
199         self._image_path = image_path
200         self._shell_args = shell_args
201         self._options = options
202         self._canceled = False
203         self._exception_info = None
204         self._directory_timing_stats = {}
205         self._test_results = []
206         self._num_tests = 0
207         self._start_time = 0
208         self._stop_time = 0
209
210         # Current directory of tests we're running.
211         self._current_dir = None
212         # Number of tests in self._current_dir.
213         self._num_tests_in_current_dir = None
214         # Time at which we started running tests from self._current_dir.
215         self._current_dir_start_time = None
216
217     def get_directory_timing_stats(self):
218         """Returns a dictionary mapping test directory to a tuple of
219         (number of tests in that directory, time to run the tests)"""
220         return self._directory_timing_stats
221
222     def get_test_results(self):
223         """Return the list of all tests run on this thread.
224
225         This is used to calculate per-thread statistics.
226
227         """
228         return self._test_results
229
230     def cancel(self):
231         """Set a flag telling this thread to quit."""
232         self._canceled = True
233
234     def get_exception_info(self):
235         """If run() terminated on an uncaught exception, return it here
236         ((type, value, traceback) tuple).
237         Returns None if run() terminated normally. Meant to be called after
238         joining this thread."""
239         return self._exception_info
240
241     def get_total_time(self):
242         return max(self._stop_time - self._start_time, 0.0)
243
244     def get_num_tests(self):
245         return self._num_tests
246
247     def run(self):
248         """Delegate main work to a helper method and watch for uncaught
249         exceptions."""
250         self._start_time = time.time()
251         self._num_tests = 0
252         try:
253             _log.debug('%s starting' % (self.getName()))
254             self._run(test_runner=None, result_summary=None)
255             _log.debug('%s done (%d tests)' % (self.getName(),
256                        self.get_num_tests()))
257         except:
258             # Save the exception for our caller to see.
259             self._exception_info = sys.exc_info()
260             self._stop_time = time.time()
261             # Re-raise it and die.
262             _log.error('%s dying: %s' % (self.getName(),
263                        self._exception_info))
264             raise
265         self._stop_time = time.time()
266
267     def run_in_main_thread(self, test_runner, result_summary):
268         """This hook allows us to run the tests from the main thread if
269         --num-test-shells==1, instead of having to always run two or more
270         threads. This allows us to debug the test harness without having to
271         do multi-threaded debugging."""
272         self._run(test_runner, result_summary)
273
274     def _run(self, test_runner, result_summary):
275         """Main work entry point of the thread. Basically we pull urls from the
276         filename queue and run the tests until we run out of urls.
277
278         If test_runner is not None, then we call test_runner.UpdateSummary()
279         with the results of each test."""
280         batch_size = 0
281         batch_count = 0
282         if self._options.batch_size:
283             try:
284                 batch_size = int(self._options.batch_size)
285             except:
286                 _log.info("Ignoring invalid batch size '%s'" %
287                           self._options.batch_size)
288
289         # Append tests we're running to the existing tests_run.txt file.
290         # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput.
291         tests_run_filename = os.path.join(self._options.results_directory,
292                                           "tests_run.txt")
293         tests_run_file = open(tests_run_filename, "a")
294
295         while True:
296             if self._canceled:
297                 _log.info('Testing canceled')
298                 tests_run_file.close()
299                 return
300
301             if len(self._filename_list) is 0:
302                 if self._current_dir is not None:
303                     self._directory_timing_stats[self._current_dir] = \
304                         (self._num_tests_in_current_dir,
305                          time.time() - self._current_dir_start_time)
306
307                 try:
308                     self._current_dir, self._filename_list = \
309                         self._filename_list_queue.get_nowait()
310                 except Queue.Empty:
311                     self._kill_dump_render_tree()
312                     tests_run_file.close()
313                     return
314
315                 self._num_tests_in_current_dir = len(self._filename_list)
316                 self._current_dir_start_time = time.time()
317
318             test_info = self._filename_list.pop()
319
320             # We have a url, run tests.
321             batch_count += 1
322             self._num_tests += 1
323             if self._options.run_singly:
324                 result = self._run_test_singly(test_info)
325             else:
326                 result = self._run_test(test_info)
327
328             filename = test_info.filename
329             tests_run_file.write(filename + "\n")
330             if result.failures:
331                 # Check and kill DumpRenderTree if we need to.
332                 if len([1 for f in result.failures
333                         if f.should_kill_dump_render_tree()]):
334                     self._kill_dump_render_tree()
335                     # Reset the batch count since the shell just bounced.
336                     batch_count = 0
337                 # Print the error message(s).
338                 error_str = '\n'.join(['  ' + f.message() for
339                                        f in result.failures])
340                 _log.debug("%s %s failed:\n%s" % (self.getName(),
341                            self._port.relative_test_filename(filename),
342                            error_str))
343             else:
344                 _log.debug("%s %s passed" % (self.getName(),
345                            self._port.relative_test_filename(filename)))
346             self._result_queue.put(result)
347
348             if batch_size > 0 and batch_count > batch_size:
349                 # Bounce the shell and reset count.
350                 self._kill_dump_render_tree()
351                 batch_count = 0
352
353             if test_runner:
354                 test_runner.update_summary(result_summary)
355
356     def _run_test_singly(self, test_info):
357         """Run a test in a separate thread, enforcing a hard time limit.
358
359         Since we can only detect the termination of a thread, not any internal
360         state or progress, we can only run per-test timeouts when running test
361         files singly.
362
363         Args:
364           test_info: Object containing the test filename, uri and timeout
365
366         Returns:
367           A TestResult
368
369         """
370         worker = SingleTestThread(self._port, self._image_path,
371                                   self._shell_args,
372                                   test_info,
373                                   self._test_types,
374                                   self._test_args,
375                                   self._options.configuration,
376                                   self._options.results_directory)
377
378         worker.start()
379
380         # When we're running one test per DumpRenderTree process, we can
381         # enforce a hard timeout.  The DumpRenderTree watchdog uses 2.5x
382         # the timeout; we want to be larger than that.
383         worker.join(int(test_info.timeout) * 3.0 / 1000.0)
384         if worker.isAlive():
385             # If join() returned with the thread still running, the
386             # DumpRenderTree is completely hung and there's nothing
387             # more we can do with it.  We have to kill all the
388             # DumpRenderTrees to free it up. If we're running more than
389             # one DumpRenderTree thread, we'll end up killing the other
390             # DumpRenderTrees too, introducing spurious crashes. We accept
391             # that tradeoff in order to avoid losing the rest of this
392             # thread's results.
393             _log.error('Test thread hung: killing all DumpRenderTrees')
394             worker._driver.stop()
395
396         try:
397             result = worker.get_test_result()
398         except AttributeError, e:
399             failures = []
400             _log.error('Cannot get results of test: %s' %
401                        test_info.filename)
402             result = TestResult(test_info.filename, failures=[],
403                                 test_run_time=0, total_time_for_all_diffs=0,
404                                 time_for_diffs=0)
405
406         return result
407
408     def _run_test(self, test_info):
409         """Run a single test file using a shared DumpRenderTree process.
410
411         Args:
412           test_info: Object containing the test filename, uri and timeout
413
414         Returns:
415           A list of TestFailure objects describing the error.
416
417         """
418         self._ensure_dump_render_tree_is_running()
419         # The pixel_hash is used to avoid doing an image dump if the
420         # checksums match, so it should be set to a blank value if we
421         # are generating a new baseline.  (Otherwise, an image from a
422         # previous run will be copied into the baseline.)
423         image_hash = test_info.image_hash()
424         if image_hash and self._test_args.new_baseline:
425             image_hash = ""
426         start = time.time()
427         crash, timeout, actual_checksum, output, error = \
428            self._driver.run_test(test_info.uri, test_info.timeout, image_hash)
429         end = time.time()
430
431         result = process_output(self._port, test_info, self._test_types,
432                                 self._test_args, self._options.configuration,
433                                 self._options.results_directory, crash,
434                                 timeout, end - start, actual_checksum,
435                                 output, error)
436         self._test_results.append(result)
437         return result
438
439     def _ensure_dump_render_tree_is_running(self):
440         """Start the shared DumpRenderTree, if it's not running.
441
442         This is not for use when running tests singly, since those each start
443         a separate DumpRenderTree in their own thread.
444
445         """
446         if (not self._driver or self._driver.poll() is not None):
447             self._driver = self._port.start_driver(
448                 self._image_path, self._shell_args)
449
450     def _kill_dump_render_tree(self):
451         """Kill the DumpRenderTree process if it's running."""
452         if self._driver:
453             self._driver.stop()
454             self._driver = None