65d0b0912e95d88ca627eb2709d9a57560ddbfbf
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / layout_package / dump_render_tree_thread.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """A Thread object for running DumpRenderTree and processing URLs from a
32 shared queue.
33
34 Each thread runs a separate instance of the DumpRenderTree binary and validates
35 the output.  When there are no more URLs to process in the shared queue, the
36 thread exits.
37 """
38
39 from __future__ import with_statement
40
41 import codecs
42 import copy
43 import logging
44 import os
45 import Queue
46 import signal
47 import sys
48 import thread
49 import threading
50 import time
51
52
53 from webkitpy.layout_tests.test_types import image_diff
54 from webkitpy.layout_tests.test_types import test_type_base
55 from webkitpy.layout_tests.test_types import text_diff
56
57 import test_failures
58 import test_output
59 import test_results
60
61 _log = logging.getLogger("webkitpy.layout_tests.layout_package."
62                          "dump_render_tree_thread")
63
64
65 class WatchableThread(threading.Thread):
66     """This class abstracts an interface used by
67     run_webkit_tests.TestRunner._wait_for_threads_to_finish for thread
68     management."""
69     def __init__(self):
70         threading.Thread.__init__(self)
71         self._canceled = False
72         self._exception_info = None
73         self._next_timeout = None
74         self._thread_id = None
75
76     def cancel(self):
77         """Set a flag telling this thread to quit."""
78         self._canceled = True
79
80     def clear_next_timeout(self):
81         """Mark a flag telling this thread to stop setting timeouts."""
82         self._timeout = 0
83
84     def exception_info(self):
85         """If run() terminated on an uncaught exception, return it here
86         ((type, value, traceback) tuple).
87         Returns None if run() terminated normally. Meant to be called after
88         joining this thread."""
89         return self._exception_info
90
91     def id(self):
92         """Return a thread identifier."""
93         return self._thread_id
94
95     def next_timeout(self):
96         """Return the time the test is supposed to finish by."""
97         return self._next_timeout
98
99
100 class TestShellThread(WatchableThread):
101     def __init__(self, port, options, worker_number, worker_name,
102                  filename_list_queue, result_queue):
103         """Initialize all the local state for this DumpRenderTree thread.
104
105         Args:
106           port: interface to port-specific hooks
107           options: command line options argument from optparse
108           worker_number: identifier for a particular worker thread.
109           worker_name: for logging.
110           filename_list_queue: A thread safe Queue class that contains lists
111               of tuples of (filename, uri) pairs.
112           result_queue: A thread safe Queue class that will contain
113               serialized TestResult objects.
114         """
115         WatchableThread.__init__(self)
116         self._port = port
117         self._options = options
118         self._worker_number = worker_number
119         self._name = worker_name
120         self._filename_list_queue = filename_list_queue
121         self._result_queue = result_queue
122
123         self._batch_count = 0
124         self._batch_size = self._options.batch_size
125         self._driver = None
126         self._have_http_lock = False
127
128         self._test_runner = None
129         self._result_summary = None
130         self._test_list_timing_stats = {}
131         self._test_results = []
132         self._num_tests = 0
133         self._start_time = 0
134         self._stop_time = 0
135         self._http_lock_wait_begin = 0
136         self._http_lock_wait_end = 0
137
138         self._test_types = []
139         for cls in self._get_test_type_classes():
140             self._test_types.append(cls(self._port,
141                                         self._options.results_directory))
142         self._test_args = self._get_test_args(worker_number)
143
144         # Append tests we're running to the existing tests_run.txt file.
145         # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput.
146         tests_run_filename = os.path.join(self._options.results_directory,
147                                           "tests_run.txt")
148         self._tests_run_file = codecs.open(tests_run_filename, "a", "utf-8")
149
150     def __del__(self):
151         self._cleanup()
152
153     def _get_test_args(self, worker_number):
154         """Returns the tuple of arguments for tests and for DumpRenderTree."""
155         test_args = test_type_base.TestArguments()
156         test_args.new_baseline = self._options.new_baseline
157         test_args.reset_results = self._options.reset_results
158
159         return test_args
160
161     def _get_test_type_classes(self):
162         classes = [text_diff.TestTextDiff]
163         if self._options.pixel_tests:
164             classes.append(image_diff.ImageDiff)
165         return classes
166
167     def get_test_group_timing_stats(self):
168         """Returns a dictionary mapping test group to a tuple of
169         (number of tests in that group, time to run the tests)"""
170         return self._test_list_timing_stats
171
172     def get_test_results(self):
173         """Return the list of all tests run on this thread.
174
175         This is used to calculate per-thread statistics.
176
177         """
178         return self._test_results
179
180     def get_total_time(self):
181         return max(self._stop_time - self._start_time -
182                    self._http_lock_wait_time(), 0.0)
183
184     def get_num_tests(self):
185         return self._num_tests
186
187     def next_timeout(self):
188         """Return the time the test is supposed to finish by."""
189         if self._next_timeout:
190             return self._next_timeout + self._http_lock_wait_time()
191         return self._next_timeout
192
193     def run(self):
194         """Delegate main work to a helper method and watch for uncaught
195         exceptions."""
196         self._covered_run()
197
198     def _covered_run(self):
199         # FIXME: this is a separate routine to work around a bug
200         # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
201         self._thread_id = thread.get_ident()
202         self._start_time = time.time()
203         try:
204             _log.debug('%s starting' % (self._name))
205             self._run(test_runner=None, result_summary=None)
206             _log.debug('%s done (%d tests)' % (self._name, self._num_tests))
207         except KeyboardInterrupt:
208             self._exception_info = sys.exc_info()
209             _log.debug("%s interrupted" % self._name)
210         except:
211             # Save the exception for our caller to see.
212             self._exception_info = sys.exc_info()
213             self._stop_time = time.time()
214             _log.error('%s dying, exception raised' % self._name)
215
216         self._stop_time = time.time()
217
218     def run_in_main_thread(self, test_runner, result_summary):
219         """This hook allows us to run the tests from the main thread if
220         --num-test-shells==1, instead of having to always run two or more
221         threads. This allows us to debug the test harness without having to
222         do multi-threaded debugging."""
223         self._run(test_runner, result_summary)
224
225     def _run(self, test_runner, result_summary):
226         """Main work entry point of the thread. Basically we pull urls from the
227         filename queue and run the tests until we run out of urls.
228
229         If test_runner is not None, then we call test_runner.UpdateSummary()
230         with the results of each test during _tear_down_test(), below."""
231         self._test_runner = test_runner
232         self._result_summary = result_summary
233
234         while not self._canceled:
235             try:
236                 current_group, filename_list = \
237                     self._filename_list_queue.get_nowait()
238                 self.handle_test_list(current_group, filename_list)
239             except Queue.Empty:
240                 break
241
242         if self._canceled:
243             _log.debug('Testing canceled')
244
245         self._cleanup()
246
247     def _cleanup(self):
248         self._kill_dump_render_tree()
249         if self._have_http_lock:
250             self._stop_servers_with_lock()
251         if self._tests_run_file:
252             self._tests_run_file.close()
253             self._tests_run_file = None
254
255     def handle_test_list(self, list_name, test_list):
256         if list_name == "tests_to_http_lock":
257             self._start_servers_with_lock()
258
259         start_time = time.time()
260         num_tests = 0
261         for test_input in test_list:
262             self._run_test(test_input)
263             if self._canceled:
264                 break
265             num_tests += 1
266
267         elapsed_time = time.time() - start_time
268
269         if self._have_http_lock:
270             self._stop_servers_with_lock()
271
272         self._test_list_timing_stats[list_name] = \
273            (num_tests, elapsed_time)
274
275     def _run_test(self, test_input):
276         self._set_up_test(test_input)
277
278         # We calculate how long we expect the test to take.
279         #
280         # The DumpRenderTree watchdog uses 2.5x the timeout; we want to be
281         # larger than that. We also add a little more padding if we're
282         # running tests in a separate thread.
283         #
284         # Note that we need to convert the test timeout from a
285         # string value in milliseconds to a float for Python.
286         driver_timeout_sec = 3.0 * float(test_input.timeout) / 1000.0
287         thread_padding_sec = 1.0
288         thread_timeout_sec = driver_timeout_sec + thread_padding_sec
289         if self._options.run_singly:
290             test_timeout_sec = thread_timeout_sec
291         else:
292             test_timeout_sec = driver_timeout_sec
293
294         start = time.time()
295         self._next_timeout = start + test_timeout_sec
296
297         if self._options.run_singly:
298             result = self._run_test_in_another_thread(test_input,
299                                                       thread_timeout_sec)
300         else:
301             result = self._run_test_in_this_thread(test_input)
302
303         self._tear_down_test(test_input, result)
304
305     def _set_up_test(self, test_input):
306         test_input.uri = self._port.filename_to_uri(test_input.filename)
307         if self._should_fetch_expected_checksum():
308             test_input.image_checksum = self._port.expected_checksum(
309                 test_input.filename)
310
311     def _should_fetch_expected_checksum(self):
312         return (self._options.pixel_tests and not
313                 (self._options.new_baseline or self._options.reset_results))
314
315     def _run_test_in_another_thread(self, test_input, thread_timeout_sec):
316         """Run a test in a separate thread, enforcing a hard time limit.
317
318         Since we can only detect the termination of a thread, not any internal
319         state or progress, we can only run per-test timeouts when running test
320         files singly.
321
322         Args:
323           test_input: Object containing the test filename and timeout
324           thread_timeout_sec: time to wait before killing the driver process.
325         Returns:
326           A TestResult
327         """
328         worker = self
329         result = None
330
331         driver = worker._port.create_driver(worker._worker_number)
332         driver.start()
333
334         class SingleTestThread(threading.Thread):
335             def run(self):
336                 result = worker._run_single_test(driver, test_input)
337
338         thread = SingleTestThread()
339         thread.start()
340         thread.join(thread_timeout_sec)
341         if thread.isAlive():
342             # If join() returned with the thread still running, the
343             # DumpRenderTree is completely hung and there's nothing
344             # more we can do with it.  We have to kill all the
345             # DumpRenderTrees to free it up. If we're running more than
346             # one DumpRenderTree thread, we'll end up killing the other
347             # DumpRenderTrees too, introducing spurious crashes. We accept
348             # that tradeoff in order to avoid losing the rest of this
349             # thread's results.
350             _log.error('Test thread hung: killing all DumpRenderTrees')
351
352         driver.stop()
353
354         if not result:
355             result = test_results.TestResult(test_input.filename, failures=[],
356                 test_run_time=0, total_time_for_all_diffs=0, time_for_diffs={})
357
358         return result
359
360     def _run_test_in_this_thread(self, test_input):
361         """Run a single test file using a shared DumpRenderTree process.
362
363         Args:
364           test_input: Object containing the test filename, uri and timeout
365
366         Returns: a TestResult object.
367         """
368         # poll() is not threadsafe and can throw OSError due to:
369         # http://bugs.python.org/issue1731717
370         if not self._driver or self._driver.poll() is not None:
371             self._driver = self._port.create_driver(self._worker_number)
372             self._driver.start()
373
374         test_result = self._run_single_test(test_input, self._driver)
375         self._test_results.append(test_result)
376         return test_result
377
378     def _run_single_test(self, test_input, driver):
379         # The image hash is used to avoid doing an image dump if the
380         # checksums match, so it should be set to a blank value if we
381         # are generating a new baseline.  (Otherwise, an image from a
382         # previous run will be copied into the baseline."""
383         if self._should_fetch_expected_checksum():
384             test_input.image_hash = self._port.expected_checksum(
385                 test_input.filename)
386         test_output = driver.run_test(test_input)
387         return self._process_output(test_input.filename, test_output)
388
389     def _process_output(self, test_filename, test_output):
390         """Receives the output from a DumpRenderTree process, subjects it to a
391         number of tests, and returns a list of failure types the test produced.
392
393         Args:
394         test_filename: full path to the test in question.
395         test_output: a TestOutput object containing the output of the test
396
397         Returns: a TestResult object
398         """
399         failures = []
400
401         if test_output.crash:
402             failures.append(test_failures.FailureCrash())
403         if test_output.timeout:
404             failures.append(test_failures.FailureTimeout())
405
406         test_name = self._port.relative_test_filename(test_filename)
407         if test_output.crash:
408             _log.debug("%s Stacktrace for %s:\n%s" %
409                        (self._name, test_name, test_output.error))
410             filename = os.path.join(self._options.results_directory, test_name)
411             filename = os.path.splitext(filename)[0] + "-stack.txt"
412             self._port.maybe_make_directory(os.path.split(filename)[0])
413             with codecs.open(filename, "wb", "utf-8") as file:
414                 file.write(test_output.error)
415         elif test_output.error:
416             _log.debug("%s %s output stderr lines:\n%s" %
417                        (self._name, test_name, test_output.error))
418
419         expected_test_output = self._expected_test_output(test_filename)
420
421         # Check the output and save the results.
422         start_time = time.time()
423         time_for_diffs = {}
424         for test_type in self._test_types:
425             start_diff_time = time.time()
426             new_failures = test_type.compare_output(self._port,
427                                                     test_filename,
428                                                     self._test_args,
429                                                     test_output,
430                                                     expected_test_output)
431             # Don't add any more failures if we already have a crash, so we
432             # don't double-report those tests. We do double-report for timeouts
433             # since we still want to see the text and image output.
434             if not test_output.crash:
435                 failures.extend(new_failures)
436             time_for_diffs[test_type.__class__.__name__] = (
437                 time.time() - start_diff_time)
438
439         total_time_for_all_diffs = time.time() - start_diff_time
440         return test_results.TestResult(test_filename,
441                                        failures,
442                                        test_output.test_time,
443                                        total_time_for_all_diffs,
444                                        time_for_diffs)
445
446     def _expected_test_output(self, filename):
447         """Returns an expected TestOutput object."""
448         return test_output.TestOutput(self._port.expected_text(filename),
449                                     self._port.expected_image(filename),
450                                     self._port.expected_checksum(filename))
451
452     def _tear_down_test(self, test_input, result):
453         self._num_tests += 1
454         self._batch_count += 1
455         self._tests_run_file.write(test_input.filename + "\n")
456         test_name = self._port.relative_test_filename(test_input.filename)
457
458         if result.failures:
459             # Check and kill DumpRenderTree if we need to.
460             if any([f.should_kill_dump_render_tree() for f in result.failures]):
461                 self._kill_dump_render_tree()
462                 # Reset the batch count since the shell just bounced.
463                 self._batch_count = 0
464
465             # Print the error message(s).
466             _log.debug("%s %s failed:" % (self._name, test_name))
467             for f in result.failures:
468                 _log.debug("%s  %s" % (self._name, f.message()))
469         else:
470             _log.debug("%s %s passed" % (self._name, test_name))
471
472         self._result_queue.put(result.dumps())
473
474         if self._batch_size > 0 and self._batch_count >= self._batch_size:
475             # Bounce the shell and reset count.
476             self._kill_dump_render_tree()
477             self._batch_count = 0
478
479         if self._test_runner:
480             self._test_runner.update_summary(self._result_summary)
481
482     def _start_servers_with_lock(self):
483         self._http_lock_wait_begin = time.time()
484         _log.debug('Acquiring http lock ...')
485         self._port.acquire_http_lock()
486         _log.debug('Starting HTTP server ...')
487         self._port.start_http_server()
488         _log.debug('Starting WebSocket server ...')
489         self._port.start_websocket_server()
490         self._http_lock_wait_end = time.time()
491         self._have_http_lock = True
492
493     def _http_lock_wait_time(self):
494         """Return the time what http locking takes."""
495         if self._http_lock_wait_begin == 0:
496             return 0
497         if self._http_lock_wait_end == 0:
498             return time.time() - self._http_lock_wait_begin
499         return self._http_lock_wait_end - self._http_lock_wait_begin
500
501     def _stop_servers_with_lock(self):
502         """Stop the servers and release http lock."""
503         if self._have_http_lock:
504             _log.debug('Stopping HTTP server ...')
505             self._port.stop_http_server()
506             _log.debug('Stopping WebSocket server ...')
507             self._port.stop_websocket_server()
508             _log.debug('Release http lock ...')
509             self._port.release_http_lock()
510             self._have_http_lock = False
511
512     def _kill_dump_render_tree(self):
513         """Kill the DumpRenderTree process if it's running."""
514         if self._driver:
515             self._driver.stop()
516             self._driver = None