2010-04-21 Eric Seidel <eric@webkit.org>
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / layout_tests / layout_package / dump_render_tree_thread.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """A Thread object for running DumpRenderTree and processing URLs from a
31 shared queue.
32
33 Each thread runs a separate instance of the DumpRenderTree binary and validates
34 the output.  When there are no more URLs to process in the shared queue, the
35 thread exits.
36 """
37
38 from __future__ import with_statement
39
40 import codecs
41 import copy
42 import logging
43 import os
44 import Queue
45 import signal
46 import sys
47 import thread
48 import threading
49 import time
50
51 import test_failures
52
53 _log = logging.getLogger("webkitpy.layout_tests.layout_package."
54                          "dump_render_tree_thread")
55
56
57 def process_output(port, test_info, test_types, test_args, configuration,
58                    output_dir, crash, timeout, test_run_time, actual_checksum,
59                    output, error):
60     """Receives the output from a DumpRenderTree process, subjects it to a
61     number of tests, and returns a list of failure types the test produced.
62
63     Args:
64       port: port-specific hooks
65       proc: an active DumpRenderTree process
66       test_info: Object containing the test filename, uri and timeout
67       test_types: list of test types to subject the output to
68       test_args: arguments to be passed to each test
69       configuration: Debug or Release
70       output_dir: directory to put crash stack traces into
71
72     Returns: a TestResult object
73     """
74     failures = []
75
76     # Some test args, such as the image hash, may be added or changed on a
77     # test-by-test basis.
78     local_test_args = copy.copy(test_args)
79
80     local_test_args.hash = actual_checksum
81
82     if crash:
83         failures.append(test_failures.FailureCrash())
84     if timeout:
85         failures.append(test_failures.FailureTimeout())
86
87     if crash:
88         _log.debug("Stacktrace for %s:\n%s" % (test_info.filename, error))
89         # Strip off "file://" since RelativeTestFilename expects
90         # filesystem paths.
91         filename = os.path.join(output_dir, port.relative_test_filename(
92                                 test_info.filename))
93         filename = os.path.splitext(filename)[0] + "-stack.txt"
94         port.maybe_make_directory(os.path.split(filename)[0])
95         with codecs.open(filename, "wb", "utf-8") as file:
96             file.write(error)
97     elif error:
98         _log.debug("Previous test output extra lines after dump:\n%s" %
99                    error)
100
101     # Check the output and save the results.
102     start_time = time.time()
103     time_for_diffs = {}
104     for test_type in test_types:
105         start_diff_time = time.time()
106         new_failures = test_type.compare_output(port, test_info.filename,
107                                                 output, local_test_args,
108                                                 configuration)
109         # Don't add any more failures if we already have a crash, so we don't
110         # double-report those tests. We do double-report for timeouts since
111         # we still want to see the text and image output.
112         if not crash:
113             failures.extend(new_failures)
114         time_for_diffs[test_type.__class__.__name__] = (
115             time.time() - start_diff_time)
116
117     total_time_for_all_diffs = time.time() - start_diff_time
118     return TestResult(test_info.filename, failures, test_run_time,
119                       total_time_for_all_diffs, time_for_diffs)
120
121
122 class TestResult(object):
123
124     def __init__(self, filename, failures, test_run_time,
125                  total_time_for_all_diffs, time_for_diffs):
126         self.failures = failures
127         self.filename = filename
128         self.test_run_time = test_run_time
129         self.time_for_diffs = time_for_diffs
130         self.total_time_for_all_diffs = total_time_for_all_diffs
131         self.type = test_failures.determine_result_type(failures)
132
133
134 class SingleTestThread(threading.Thread):
135     """Thread wrapper for running a single test file."""
136
137     def __init__(self, port, image_path, shell_args, test_info,
138         test_types, test_args, configuration, output_dir):
139         """
140         Args:
141           port: object implementing port-specific hooks
142           test_info: Object containing the test filename, uri and timeout
143           output_dir: Directory to put crash stacks into.
144           See TestShellThread for documentation of the remaining arguments.
145         """
146
147         threading.Thread.__init__(self)
148         self._port = port
149         self._image_path = image_path
150         self._shell_args = shell_args
151         self._test_info = test_info
152         self._test_types = test_types
153         self._test_args = test_args
154         self._configuration = configuration
155         self._output_dir = output_dir
156
157     def run(self):
158         test_info = self._test_info
159         driver = self._port.start_driver(self._image_path, self._shell_args)
160         start = time.time()
161         crash, timeout, actual_checksum, output, error = \
162             driver.run_test(test_info.uri.strip(), test_info.timeout,
163                             test_info.image_hash())
164         end = time.time()
165         self._test_result = process_output(self._port,
166             test_info, self._test_types, self._test_args,
167             self._configuration, self._output_dir, crash, timeout, end - start,
168             actual_checksum, output, error)
169         driver.stop()
170
171     def get_test_result(self):
172         return self._test_result
173
174
175 class TestShellThread(threading.Thread):
176
177     def __init__(self, port, filename_list_queue, result_queue,
178                  test_types, test_args, image_path, shell_args, options):
179         """Initialize all the local state for this DumpRenderTree thread.
180
181         Args:
182           port: interface to port-specific hooks
183           filename_list_queue: A thread safe Queue class that contains lists
184               of tuples of (filename, uri) pairs.
185           result_queue: A thread safe Queue class that will contain tuples of
186               (test, failure lists) for the test results.
187           test_types: A list of TestType objects to run the test output
188               against.
189           test_args: A TestArguments object to pass to each TestType.
190           shell_args: Any extra arguments to be passed to DumpRenderTree.
191           options: A property dictionary as produced by optparse. The
192               command-line options should match those expected by
193               run_webkit_tests; they are typically passed via the
194               run_webkit_tests.TestRunner class."""
195         threading.Thread.__init__(self)
196         self._port = port
197         self._filename_list_queue = filename_list_queue
198         self._result_queue = result_queue
199         self._filename_list = []
200         self._test_types = test_types
201         self._test_args = test_args
202         self._driver = None
203         self._image_path = image_path
204         self._shell_args = shell_args
205         self._options = options
206         self._canceled = False
207         self._exception_info = None
208         self._directory_timing_stats = {}
209         self._test_results = []
210         self._num_tests = 0
211         self._start_time = 0
212         self._stop_time = 0
213
214         # Current directory of tests we're running.
215         self._current_dir = None
216         # Number of tests in self._current_dir.
217         self._num_tests_in_current_dir = None
218         # Time at which we started running tests from self._current_dir.
219         self._current_dir_start_time = None
220
221     def get_directory_timing_stats(self):
222         """Returns a dictionary mapping test directory to a tuple of
223         (number of tests in that directory, time to run the tests)"""
224         return self._directory_timing_stats
225
226     def get_test_results(self):
227         """Return the list of all tests run on this thread.
228
229         This is used to calculate per-thread statistics.
230
231         """
232         return self._test_results
233
234     def cancel(self):
235         """Set a flag telling this thread to quit."""
236         self._canceled = True
237
238     def get_exception_info(self):
239         """If run() terminated on an uncaught exception, return it here
240         ((type, value, traceback) tuple).
241         Returns None if run() terminated normally. Meant to be called after
242         joining this thread."""
243         return self._exception_info
244
245     def get_total_time(self):
246         return max(self._stop_time - self._start_time, 0.0)
247
248     def get_num_tests(self):
249         return self._num_tests
250
251     def run(self):
252         """Delegate main work to a helper method and watch for uncaught
253         exceptions."""
254         self._start_time = time.time()
255         self._num_tests = 0
256         try:
257             _log.debug('%s starting' % (self.getName()))
258             self._run(test_runner=None, result_summary=None)
259             _log.debug('%s done (%d tests)' % (self.getName(),
260                        self.get_num_tests()))
261         except:
262             # Save the exception for our caller to see.
263             self._exception_info = sys.exc_info()
264             self._stop_time = time.time()
265             # Re-raise it and die.
266             _log.error('%s dying: %s' % (self.getName(),
267                        self._exception_info))
268             raise
269         self._stop_time = time.time()
270
271     def run_in_main_thread(self, test_runner, result_summary):
272         """This hook allows us to run the tests from the main thread if
273         --num-test-shells==1, instead of having to always run two or more
274         threads. This allows us to debug the test harness without having to
275         do multi-threaded debugging."""
276         self._run(test_runner, result_summary)
277
278     def _run(self, test_runner, result_summary):
279         """Main work entry point of the thread. Basically we pull urls from the
280         filename queue and run the tests until we run out of urls.
281
282         If test_runner is not None, then we call test_runner.UpdateSummary()
283         with the results of each test."""
284         batch_size = 0
285         batch_count = 0
286         if self._options.batch_size:
287             try:
288                 batch_size = int(self._options.batch_size)
289             except:
290                 _log.info("Ignoring invalid batch size '%s'" %
291                           self._options.batch_size)
292
293         # Append tests we're running to the existing tests_run.txt file.
294         # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput.
295         tests_run_filename = os.path.join(self._options.results_directory,
296                                           "tests_run.txt")
297         tests_run_file = codecs.open(tests_run_filename, "a", "utf-8")
298
299         while True:
300             if self._canceled:
301                 _log.info('Testing canceled')
302                 tests_run_file.close()
303                 return
304
305             if len(self._filename_list) is 0:
306                 if self._current_dir is not None:
307                     self._directory_timing_stats[self._current_dir] = \
308                         (self._num_tests_in_current_dir,
309                          time.time() - self._current_dir_start_time)
310
311                 try:
312                     self._current_dir, self._filename_list = \
313                         self._filename_list_queue.get_nowait()
314                 except Queue.Empty:
315                     self._kill_dump_render_tree()
316                     tests_run_file.close()
317                     return
318
319                 self._num_tests_in_current_dir = len(self._filename_list)
320                 self._current_dir_start_time = time.time()
321
322             test_info = self._filename_list.pop()
323
324             # We have a url, run tests.
325             batch_count += 1
326             self._num_tests += 1
327             if self._options.run_singly:
328                 result = self._run_test_singly(test_info)
329             else:
330                 result = self._run_test(test_info)
331
332             filename = test_info.filename
333             tests_run_file.write(filename + "\n")
334             if result.failures:
335                 # Check and kill DumpRenderTree if we need to.
336                 if len([1 for f in result.failures
337                         if f.should_kill_dump_render_tree()]):
338                     self._kill_dump_render_tree()
339                     # Reset the batch count since the shell just bounced.
340                     batch_count = 0
341                 # Print the error message(s).
342                 error_str = '\n'.join(['  ' + f.message() for
343                                        f in result.failures])
344                 _log.debug("%s %s failed:\n%s" % (self.getName(),
345                            self._port.relative_test_filename(filename),
346                            error_str))
347             else:
348                 _log.debug("%s %s passed" % (self.getName(),
349                            self._port.relative_test_filename(filename)))
350             self._result_queue.put(result)
351
352             if batch_size > 0 and batch_count > batch_size:
353                 # Bounce the shell and reset count.
354                 self._kill_dump_render_tree()
355                 batch_count = 0
356
357             if test_runner:
358                 test_runner.update_summary(result_summary)
359
360     def _run_test_singly(self, test_info):
361         """Run a test in a separate thread, enforcing a hard time limit.
362
363         Since we can only detect the termination of a thread, not any internal
364         state or progress, we can only run per-test timeouts when running test
365         files singly.
366
367         Args:
368           test_info: Object containing the test filename, uri and timeout
369
370         Returns:
371           A TestResult
372
373         """
374         worker = SingleTestThread(self._port, self._image_path,
375                                   self._shell_args,
376                                   test_info,
377                                   self._test_types,
378                                   self._test_args,
379                                   self._options.configuration,
380                                   self._options.results_directory)
381
382         worker.start()
383
384         # When we're running one test per DumpRenderTree process, we can
385         # enforce a hard timeout.  The DumpRenderTree watchdog uses 2.5x
386         # the timeout; we want to be larger than that.
387         worker.join(int(test_info.timeout) * 3.0 / 1000.0)
388         if worker.isAlive():
389             # If join() returned with the thread still running, the
390             # DumpRenderTree is completely hung and there's nothing
391             # more we can do with it.  We have to kill all the
392             # DumpRenderTrees to free it up. If we're running more than
393             # one DumpRenderTree thread, we'll end up killing the other
394             # DumpRenderTrees too, introducing spurious crashes. We accept
395             # that tradeoff in order to avoid losing the rest of this
396             # thread's results.
397             _log.error('Test thread hung: killing all DumpRenderTrees')
398             worker._driver.stop()
399
400         try:
401             result = worker.get_test_result()
402         except AttributeError, e:
403             failures = []
404             _log.error('Cannot get results of test: %s' %
405                        test_info.filename)
406             result = TestResult(test_info.filename, failures=[],
407                                 test_run_time=0, total_time_for_all_diffs=0,
408                                 time_for_diffs=0)
409
410         return result
411
412     def _run_test(self, test_info):
413         """Run a single test file using a shared DumpRenderTree process.
414
415         Args:
416           test_info: Object containing the test filename, uri and timeout
417
418         Returns:
419           A list of TestFailure objects describing the error.
420
421         """
422         self._ensure_dump_render_tree_is_running()
423         # The pixel_hash is used to avoid doing an image dump if the
424         # checksums match, so it should be set to a blank value if we
425         # are generating a new baseline.  (Otherwise, an image from a
426         # previous run will be copied into the baseline.)
427         image_hash = test_info.image_hash()
428         if image_hash and self._test_args.new_baseline:
429             image_hash = ""
430         start = time.time()
431         crash, timeout, actual_checksum, output, error = \
432            self._driver.run_test(test_info.uri, test_info.timeout, image_hash)
433         end = time.time()
434
435         result = process_output(self._port, test_info, self._test_types,
436                                 self._test_args, self._options.configuration,
437                                 self._options.results_directory, crash,
438                                 timeout, end - start, actual_checksum,
439                                 output, error)
440         self._test_results.append(result)
441         return result
442
443     def _ensure_dump_render_tree_is_running(self):
444         """Start the shared DumpRenderTree, if it's not running.
445
446         This is not for use when running tests singly, since those each start
447         a separate DumpRenderTree in their own thread.
448
449         """
450         if (not self._driver or self._driver.poll() is not None):
451             self._driver = self._port.start_driver(
452                 self._image_path, self._shell_args)
453
454     def _kill_dump_render_tree(self):
455         """Kill the DumpRenderTree process if it's running."""
456         if self._driver:
457             self._driver.stop()
458             self._driver = None