2010-04-22 Eric Seidel <eric@webkit.org>
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / layout_tests / layout_package / dump_render_tree_thread.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the name of Google Inc. nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """A Thread object for running DumpRenderTree and processing URLs from a
31 shared queue.
32
33 Each thread runs a separate instance of the DumpRenderTree binary and validates
34 the output.  When there are no more URLs to process in the shared queue, the
35 thread exits.
36 """
37
38 from __future__ import with_statement
39
40 import codecs
41 import copy
42 import logging
43 import os
44 import Queue
45 import signal
46 import sys
47 import thread
48 import threading
49 import time
50
51 import test_failures
52
53 _log = logging.getLogger("webkitpy.layout_tests.layout_package."
54                          "dump_render_tree_thread")
55
56
57 def process_output(port, test_info, test_types, test_args, configuration,
58                    output_dir, crash, timeout, test_run_time, actual_checksum,
59                    output, error):
60     """Receives the output from a DumpRenderTree process, subjects it to a
61     number of tests, and returns a list of failure types the test produced.
62
63     Args:
64       port: port-specific hooks
65       proc: an active DumpRenderTree process
66       test_info: Object containing the test filename, uri and timeout
67       test_types: list of test types to subject the output to
68       test_args: arguments to be passed to each test
69       configuration: Debug or Release
70       output_dir: directory to put crash stack traces into
71
72     Returns: a TestResult object
73     """
74     failures = []
75
76     # Some test args, such as the image hash, may be added or changed on a
77     # test-by-test basis.
78     local_test_args = copy.copy(test_args)
79
80     local_test_args.hash = actual_checksum
81
82     if crash:
83         failures.append(test_failures.FailureCrash())
84     if timeout:
85         failures.append(test_failures.FailureTimeout())
86
87     if crash:
88         _log.debug("Stacktrace for %s:\n%s" % (test_info.filename, error))
89         # Strip off "file://" since RelativeTestFilename expects
90         # filesystem paths.
91         filename = os.path.join(output_dir, port.relative_test_filename(
92                                 test_info.filename))
93         filename = os.path.splitext(filename)[0] + "-stack.txt"
94         port.maybe_make_directory(os.path.split(filename)[0])
95         with codecs.open(filename, "wb", "utf-8") as file:
96             file.write(error)
97     elif error:
98         _log.debug("Previous test output stderr lines:\n%s" % error)
99
100     # Check the output and save the results.
101     start_time = time.time()
102     time_for_diffs = {}
103     for test_type in test_types:
104         start_diff_time = time.time()
105         new_failures = test_type.compare_output(port, test_info.filename,
106                                                 output, local_test_args,
107                                                 configuration)
108         # Don't add any more failures if we already have a crash, so we don't
109         # double-report those tests. We do double-report for timeouts since
110         # we still want to see the text and image output.
111         if not crash:
112             failures.extend(new_failures)
113         time_for_diffs[test_type.__class__.__name__] = (
114             time.time() - start_diff_time)
115
116     total_time_for_all_diffs = time.time() - start_diff_time
117     return TestResult(test_info.filename, failures, test_run_time,
118                       total_time_for_all_diffs, time_for_diffs)
119
120
121 class TestResult(object):
122
123     def __init__(self, filename, failures, test_run_time,
124                  total_time_for_all_diffs, time_for_diffs):
125         self.failures = failures
126         self.filename = filename
127         self.test_run_time = test_run_time
128         self.time_for_diffs = time_for_diffs
129         self.total_time_for_all_diffs = total_time_for_all_diffs
130         self.type = test_failures.determine_result_type(failures)
131
132
133 class SingleTestThread(threading.Thread):
134     """Thread wrapper for running a single test file."""
135
136     def __init__(self, port, image_path, shell_args, test_info,
137         test_types, test_args, configuration, output_dir):
138         """
139         Args:
140           port: object implementing port-specific hooks
141           test_info: Object containing the test filename, uri and timeout
142           output_dir: Directory to put crash stacks into.
143           See TestShellThread for documentation of the remaining arguments.
144         """
145
146         threading.Thread.__init__(self)
147         self._port = port
148         self._image_path = image_path
149         self._shell_args = shell_args
150         self._test_info = test_info
151         self._test_types = test_types
152         self._test_args = test_args
153         self._configuration = configuration
154         self._output_dir = output_dir
155
156     def run(self):
157         test_info = self._test_info
158         driver = self._port.start_driver(self._image_path, self._shell_args)
159         start = time.time()
160         crash, timeout, actual_checksum, output, error = \
161             driver.run_test(test_info.uri.strip(), test_info.timeout,
162                             test_info.image_hash())
163         end = time.time()
164         self._test_result = process_output(self._port,
165             test_info, self._test_types, self._test_args,
166             self._configuration, self._output_dir, crash, timeout, end - start,
167             actual_checksum, output, error)
168         driver.stop()
169
170     def get_test_result(self):
171         return self._test_result
172
173
174 class TestShellThread(threading.Thread):
175
176     def __init__(self, port, filename_list_queue, result_queue,
177                  test_types, test_args, image_path, shell_args, options):
178         """Initialize all the local state for this DumpRenderTree thread.
179
180         Args:
181           port: interface to port-specific hooks
182           filename_list_queue: A thread safe Queue class that contains lists
183               of tuples of (filename, uri) pairs.
184           result_queue: A thread safe Queue class that will contain tuples of
185               (test, failure lists) for the test results.
186           test_types: A list of TestType objects to run the test output
187               against.
188           test_args: A TestArguments object to pass to each TestType.
189           shell_args: Any extra arguments to be passed to DumpRenderTree.
190           options: A property dictionary as produced by optparse. The
191               command-line options should match those expected by
192               run_webkit_tests; they are typically passed via the
193               run_webkit_tests.TestRunner class."""
194         threading.Thread.__init__(self)
195         self._port = port
196         self._filename_list_queue = filename_list_queue
197         self._result_queue = result_queue
198         self._filename_list = []
199         self._test_types = test_types
200         self._test_args = test_args
201         self._driver = None
202         self._image_path = image_path
203         self._shell_args = shell_args
204         self._options = options
205         self._canceled = False
206         self._exception_info = None
207         self._directory_timing_stats = {}
208         self._test_results = []
209         self._num_tests = 0
210         self._start_time = 0
211         self._stop_time = 0
212
213         # Current directory of tests we're running.
214         self._current_dir = None
215         # Number of tests in self._current_dir.
216         self._num_tests_in_current_dir = None
217         # Time at which we started running tests from self._current_dir.
218         self._current_dir_start_time = None
219
220     def get_directory_timing_stats(self):
221         """Returns a dictionary mapping test directory to a tuple of
222         (number of tests in that directory, time to run the tests)"""
223         return self._directory_timing_stats
224
225     def get_test_results(self):
226         """Return the list of all tests run on this thread.
227
228         This is used to calculate per-thread statistics.
229
230         """
231         return self._test_results
232
233     def cancel(self):
234         """Set a flag telling this thread to quit."""
235         self._canceled = True
236
237     def get_exception_info(self):
238         """If run() terminated on an uncaught exception, return it here
239         ((type, value, traceback) tuple).
240         Returns None if run() terminated normally. Meant to be called after
241         joining this thread."""
242         return self._exception_info
243
244     def get_total_time(self):
245         return max(self._stop_time - self._start_time, 0.0)
246
247     def get_num_tests(self):
248         return self._num_tests
249
250     def run(self):
251         """Delegate main work to a helper method and watch for uncaught
252         exceptions."""
253         self._start_time = time.time()
254         self._num_tests = 0
255         try:
256             _log.debug('%s starting' % (self.getName()))
257             self._run(test_runner=None, result_summary=None)
258             _log.debug('%s done (%d tests)' % (self.getName(),
259                        self.get_num_tests()))
260         except:
261             # Save the exception for our caller to see.
262             self._exception_info = sys.exc_info()
263             self._stop_time = time.time()
264             # Re-raise it and die.
265             _log.error('%s dying: %s' % (self.getName(),
266                        self._exception_info))
267             raise
268         self._stop_time = time.time()
269
270     def run_in_main_thread(self, test_runner, result_summary):
271         """This hook allows us to run the tests from the main thread if
272         --num-test-shells==1, instead of having to always run two or more
273         threads. This allows us to debug the test harness without having to
274         do multi-threaded debugging."""
275         self._run(test_runner, result_summary)
276
277     def _run(self, test_runner, result_summary):
278         """Main work entry point of the thread. Basically we pull urls from the
279         filename queue and run the tests until we run out of urls.
280
281         If test_runner is not None, then we call test_runner.UpdateSummary()
282         with the results of each test."""
283         batch_size = 0
284         batch_count = 0
285         if self._options.batch_size:
286             try:
287                 batch_size = int(self._options.batch_size)
288             except:
289                 _log.info("Ignoring invalid batch size '%s'" %
290                           self._options.batch_size)
291
292         # Append tests we're running to the existing tests_run.txt file.
293         # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput.
294         tests_run_filename = os.path.join(self._options.results_directory,
295                                           "tests_run.txt")
296         tests_run_file = codecs.open(tests_run_filename, "a", "utf-8")
297
298         while True:
299             if self._canceled:
300                 _log.info('Testing canceled')
301                 tests_run_file.close()
302                 return
303
304             if len(self._filename_list) is 0:
305                 if self._current_dir is not None:
306                     self._directory_timing_stats[self._current_dir] = \
307                         (self._num_tests_in_current_dir,
308                          time.time() - self._current_dir_start_time)
309
310                 try:
311                     self._current_dir, self._filename_list = \
312                         self._filename_list_queue.get_nowait()
313                 except Queue.Empty:
314                     self._kill_dump_render_tree()
315                     tests_run_file.close()
316                     return
317
318                 self._num_tests_in_current_dir = len(self._filename_list)
319                 self._current_dir_start_time = time.time()
320
321             test_info = self._filename_list.pop()
322
323             # We have a url, run tests.
324             batch_count += 1
325             self._num_tests += 1
326             if self._options.run_singly:
327                 result = self._run_test_singly(test_info)
328             else:
329                 result = self._run_test(test_info)
330
331             filename = test_info.filename
332             tests_run_file.write(filename + "\n")
333             if result.failures:
334                 # Check and kill DumpRenderTree if we need to.
335                 if len([1 for f in result.failures
336                         if f.should_kill_dump_render_tree()]):
337                     self._kill_dump_render_tree()
338                     # Reset the batch count since the shell just bounced.
339                     batch_count = 0
340                 # Print the error message(s).
341                 error_str = '\n'.join(['  ' + f.message() for
342                                        f in result.failures])
343                 _log.debug("%s %s failed:\n%s" % (self.getName(),
344                            self._port.relative_test_filename(filename),
345                            error_str))
346             else:
347                 _log.debug("%s %s passed" % (self.getName(),
348                            self._port.relative_test_filename(filename)))
349             self._result_queue.put(result)
350
351             if batch_size > 0 and batch_count > batch_size:
352                 # Bounce the shell and reset count.
353                 self._kill_dump_render_tree()
354                 batch_count = 0
355
356             if test_runner:
357                 test_runner.update_summary(result_summary)
358
359     def _run_test_singly(self, test_info):
360         """Run a test in a separate thread, enforcing a hard time limit.
361
362         Since we can only detect the termination of a thread, not any internal
363         state or progress, we can only run per-test timeouts when running test
364         files singly.
365
366         Args:
367           test_info: Object containing the test filename, uri and timeout
368
369         Returns:
370           A TestResult
371
372         """
373         worker = SingleTestThread(self._port, self._image_path,
374                                   self._shell_args,
375                                   test_info,
376                                   self._test_types,
377                                   self._test_args,
378                                   self._options.configuration,
379                                   self._options.results_directory)
380
381         worker.start()
382
383         # When we're running one test per DumpRenderTree process, we can
384         # enforce a hard timeout.  The DumpRenderTree watchdog uses 2.5x
385         # the timeout; we want to be larger than that.
386         worker.join(int(test_info.timeout) * 3.0 / 1000.0)
387         if worker.isAlive():
388             # If join() returned with the thread still running, the
389             # DumpRenderTree is completely hung and there's nothing
390             # more we can do with it.  We have to kill all the
391             # DumpRenderTrees to free it up. If we're running more than
392             # one DumpRenderTree thread, we'll end up killing the other
393             # DumpRenderTrees too, introducing spurious crashes. We accept
394             # that tradeoff in order to avoid losing the rest of this
395             # thread's results.
396             _log.error('Test thread hung: killing all DumpRenderTrees')
397             worker._driver.stop()
398
399         try:
400             result = worker.get_test_result()
401         except AttributeError, e:
402             failures = []
403             _log.error('Cannot get results of test: %s' %
404                        test_info.filename)
405             result = TestResult(test_info.filename, failures=[],
406                                 test_run_time=0, total_time_for_all_diffs=0,
407                                 time_for_diffs=0)
408
409         return result
410
411     def _run_test(self, test_info):
412         """Run a single test file using a shared DumpRenderTree process.
413
414         Args:
415           test_info: Object containing the test filename, uri and timeout
416
417         Returns:
418           A list of TestFailure objects describing the error.
419
420         """
421         self._ensure_dump_render_tree_is_running()
422         # The pixel_hash is used to avoid doing an image dump if the
423         # checksums match, so it should be set to a blank value if we
424         # are generating a new baseline.  (Otherwise, an image from a
425         # previous run will be copied into the baseline.)
426         image_hash = test_info.image_hash()
427         if image_hash and self._test_args.new_baseline:
428             image_hash = ""
429         start = time.time()
430         crash, timeout, actual_checksum, output, error = \
431            self._driver.run_test(test_info.uri, test_info.timeout, image_hash)
432         end = time.time()
433
434         result = process_output(self._port, test_info, self._test_types,
435                                 self._test_args, self._options.configuration,
436                                 self._options.results_directory, crash,
437                                 timeout, end - start, actual_checksum,
438                                 output, error)
439         self._test_results.append(result)
440         return result
441
442     def _ensure_dump_render_tree_is_running(self):
443         """Start the shared DumpRenderTree, if it's not running.
444
445         This is not for use when running tests singly, since those each start
446         a separate DumpRenderTree in their own thread.
447
448         """
449         if (not self._driver or self._driver.poll() is not None):
450             self._driver = self._port.start_driver(
451                 self._image_path, self._shell_args)
452
453     def _kill_dump_render_tree(self):
454         """Kill the DumpRenderTree process if it's running."""
455         if self._driver:
456             self._driver.stop()
457             self._driver = None