880cc60a882ccbf6325c212ef4900c5d9ec14fd3
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / layout_tests / layout_package / dump_render_tree_thread.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """A Thread object for running DumpRenderTree and processing URLs from a
32 shared queue.
33
34 Each thread runs a separate instance of the DumpRenderTree binary and validates
35 the output.  When there are no more URLs to process in the shared queue, the
36 thread exits.
37 """
38
39 from __future__ import with_statement
40
41 import codecs
42 import copy
43 import logging
44 import os
45 import Queue
46 import signal
47 import sys
48 import thread
49 import threading
50 import time
51
52
53 from webkitpy.layout_tests.test_types import image_diff
54 from webkitpy.layout_tests.test_types import test_type_base
55 from webkitpy.layout_tests.test_types import text_diff
56
57 import test_failures
58 import test_output
59 import test_results
60
61 _log = logging.getLogger("webkitpy.layout_tests.layout_package."
62                          "dump_render_tree_thread")
63
64
65 class WatchableThread(threading.Thread):
66     """This class abstracts an interface used by
67     run_webkit_tests.TestRunner._wait_for_threads_to_finish for thread
68     management."""
69     def __init__(self):
70         threading.Thread.__init__(self)
71         self._canceled = False
72         self._exception_info = None
73         self._next_timeout = None
74         self._thread_id = None
75
76     def cancel(self):
77         """Set a flag telling this thread to quit."""
78         self._canceled = True
79
80     def clear_next_timeout(self):
81         """Mark a flag telling this thread to stop setting timeouts."""
82         self._timeout = 0
83
84     def exception_info(self):
85         """If run() terminated on an uncaught exception, return it here
86         ((type, value, traceback) tuple).
87         Returns None if run() terminated normally. Meant to be called after
88         joining this thread."""
89         return self._exception_info
90
91     def id(self):
92         """Return a thread identifier."""
93         return self._thread_id
94
95     def next_timeout(self):
96         """Return the time the test is supposed to finish by."""
97         return self._next_timeout
98
99
100 class TestShellThread(WatchableThread):
101     def __init__(self, port, options, worker_number,
102                  filename_list_queue, result_queue):
103         """Initialize all the local state for this DumpRenderTree thread.
104
105         Args:
106           port: interface to port-specific hooks
107           options: command line options argument from optparse
108           worker_number: identifier for a particular worker thread.
109           filename_list_queue: A thread safe Queue class that contains lists
110               of tuples of (filename, uri) pairs.
111           result_queue: A thread safe Queue class that will contain
112               serialized TestResult objects.
113         """
114         WatchableThread.__init__(self)
115         self._port = port
116         self._options = options
117         self._worker_number = worker_number
118         self._name = 'worker/%d' % worker_number
119         self._filename_list_queue = filename_list_queue
120         self._result_queue = result_queue
121
122         self._batch_count = 0
123         self._batch_size = self._options.batch_size
124         self._driver = None
125         self._have_http_lock = False
126
127         self._test_runner = None
128         self._result_summary = None
129         self._test_list_timing_stats = {}
130         self._test_results = []
131         self._num_tests = 0
132         self._start_time = 0
133         self._stop_time = 0
134         self._http_lock_wait_begin = 0
135         self._http_lock_wait_end = 0
136
137         self._test_types = []
138         for cls in self._get_test_type_classes():
139             self._test_types.append(cls(self._port,
140                                         self._options.results_directory))
141         self._test_args = self._get_test_args(worker_number)
142
143         # Append tests we're running to the existing tests_run.txt file.
144         # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput.
145         tests_run_filename = os.path.join(self._options.results_directory,
146                                           "tests_run.txt")
147         self._tests_run_file = codecs.open(tests_run_filename, "a", "utf-8")
148
149     def __del__(self):
150         self._cleanup()
151
152     def _get_test_args(self, worker_number):
153         """Returns the tuple of arguments for tests and for DumpRenderTree."""
154         test_args = test_type_base.TestArguments()
155         test_args.new_baseline = self._options.new_baseline
156         test_args.reset_results = self._options.reset_results
157
158         return test_args
159
160     def _get_test_type_classes(self):
161         classes = [text_diff.TestTextDiff]
162         if self._options.pixel_tests:
163             classes.append(image_diff.ImageDiff)
164         return classes
165
166     def get_test_group_timing_stats(self):
167         """Returns a dictionary mapping test group to a tuple of
168         (number of tests in that group, time to run the tests)"""
169         return self._test_list_timing_stats
170
171     def get_test_results(self):
172         """Return the list of all tests run on this thread.
173
174         This is used to calculate per-thread statistics.
175
176         """
177         return self._test_results
178
179     def get_total_time(self):
180         return max(self._stop_time - self._start_time -
181                    self._http_lock_wait_time(), 0.0)
182
183     def get_num_tests(self):
184         return self._num_tests
185
186     def name(self):
187         return self._name
188
189     def next_timeout(self):
190         """Return the time the test is supposed to finish by."""
191         if self._next_timeout:
192             return self._next_timeout + self._http_lock_wait_time()
193         return self._next_timeout
194
195     def run(self):
196         """Delegate main work to a helper method and watch for uncaught
197         exceptions."""
198         self._covered_run()
199
200     def _covered_run(self):
201         # FIXME: this is a separate routine to work around a bug
202         # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
203         self._thread_id = thread.get_ident()
204         self._start_time = time.time()
205         try:
206             _log.debug('%s starting' % (self._name))
207             self._run(test_runner=None, result_summary=None)
208             _log.debug('%s done (%d tests)' % (self._name, self._num_tests))
209         except KeyboardInterrupt:
210             self._exception_info = sys.exc_info()
211             _log.debug("%s interrupted" % self._name)
212         except:
213             # Save the exception for our caller to see.
214             self._exception_info = sys.exc_info()
215             self._stop_time = time.time()
216             _log.error('%s dying, exception raised' % self._name)
217
218         self._stop_time = time.time()
219
220     def run_in_main_thread(self, test_runner, result_summary):
221         """This hook allows us to run the tests from the main thread if
222         --num-test-shells==1, instead of having to always run two or more
223         threads. This allows us to debug the test harness without having to
224         do multi-threaded debugging."""
225         self._run(test_runner, result_summary)
226
227     def _run(self, test_runner, result_summary):
228         """Main work entry point of the thread. Basically we pull urls from the
229         filename queue and run the tests until we run out of urls.
230
231         If test_runner is not None, then we call test_runner.UpdateSummary()
232         with the results of each test during _tear_down_test(), below."""
233         self._test_runner = test_runner
234         self._result_summary = result_summary
235
236         while not self._canceled:
237             try:
238                 current_group, filename_list = \
239                     self._filename_list_queue.get_nowait()
240                 self.handle_test_list(current_group, filename_list)
241             except Queue.Empty:
242                 break
243
244         if self._canceled:
245             _log.debug('Testing canceled')
246
247         self._cleanup()
248
249     def _cleanup(self):
250         self._kill_dump_render_tree()
251         if self._have_http_lock:
252             self._stop_servers_with_lock()
253         if self._tests_run_file:
254             self._tests_run_file.close()
255             self._tests_run_file = None
256
257     def handle_test_list(self, list_name, test_list):
258         if list_name == "tests_to_http_lock":
259             self._start_servers_with_lock()
260
261         start_time = time.time()
262         num_tests = 0
263         for test_input in test_list:
264             self._run_test(test_input)
265             if self._canceled:
266                 break
267             num_tests += 1
268
269         elapsed_time = time.time() - start_time
270
271         if self._have_http_lock:
272             self._stop_servers_with_lock()
273
274         self._test_list_timing_stats[list_name] = \
275            (num_tests, elapsed_time)
276
277     def _run_test(self, test_input):
278         self._set_up_test(test_input)
279
280         # We calculate how long we expect the test to take.
281         #
282         # The DumpRenderTree watchdog uses 2.5x the timeout; we want to be
283         # larger than that. We also add a little more padding if we're
284         # running tests in a separate thread.
285         #
286         # Note that we need to convert the test timeout from a
287         # string value in milliseconds to a float for Python.
288         driver_timeout_sec = 3.0 * float(test_input.timeout) / 1000.0
289         thread_padding_sec = 1.0
290         thread_timeout_sec = driver_timeout_sec + thread_padding_sec
291         if self._options.run_singly:
292             test_timeout_sec = thread_timeout_sec
293         else:
294             test_timeout_sec = driver_timeout_sec
295
296         start = time.time()
297         self._next_timeout = start + test_timeout_sec
298
299         if self._options.run_singly:
300             result = self._run_test_in_another_thread(test_input,
301                                                       thread_timeout_sec)
302         else:
303             result = self._run_test_in_this_thread(test_input)
304
305         self._tear_down_test(test_input, result)
306
307     def _set_up_test(self, test_input):
308         test_input.uri = self._port.filename_to_uri(test_input.filename)
309         if self._should_fetch_expected_checksum():
310             test_input.image_checksum = self._port.expected_checksum(
311                 test_input.filename)
312
313     def _should_fetch_expected_checksum(self):
314         return (self._options.pixel_tests and not
315                 (self._options.new_baseline or self._options.reset_results))
316
317     def _run_test_in_another_thread(self, test_input, thread_timeout_sec):
318         """Run a test in a separate thread, enforcing a hard time limit.
319
320         Since we can only detect the termination of a thread, not any internal
321         state or progress, we can only run per-test timeouts when running test
322         files singly.
323
324         Args:
325           test_input: Object containing the test filename and timeout
326           thread_timeout_sec: time to wait before killing the driver process.
327         Returns:
328           A TestResult
329         """
330         worker = self
331         result = None
332
333         driver = worker._port.create_driver(worker._worker_number)
334         driver.start()
335
336         class SingleTestThread(threading.Thread):
337             def run(self):
338                 result = worker._run_single_test(test_input, driver)
339
340         thread = SingleTestThread()
341         thread.start()
342         thread.join(thread_timeout_sec)
343         if thread.isAlive():
344             # If join() returned with the thread still running, the
345             # DumpRenderTree is completely hung and there's nothing
346             # more we can do with it.  We have to kill all the
347             # DumpRenderTrees to free it up. If we're running more than
348             # one DumpRenderTree thread, we'll end up killing the other
349             # DumpRenderTrees too, introducing spurious crashes. We accept
350             # that tradeoff in order to avoid losing the rest of this
351             # thread's results.
352             _log.error('Test thread hung: killing all DumpRenderTrees')
353
354         driver.stop()
355
356         if not result:
357             result = test_results.TestResult(test_input.filename, failures=[],
358                 test_run_time=0, total_time_for_all_diffs=0, time_for_diffs={})
359
360         return result
361
362     def _run_test_in_this_thread(self, test_input):
363         """Run a single test file using a shared DumpRenderTree process.
364
365         Args:
366           test_input: Object containing the test filename, uri and timeout
367
368         Returns: a TestResult object.
369         """
370         # poll() is not threadsafe and can throw OSError due to:
371         # http://bugs.python.org/issue1731717
372         if not self._driver or self._driver.poll() is not None:
373             self._driver = self._port.create_driver(self._worker_number)
374             self._driver.start()
375
376         test_result = self._run_single_test(test_input, self._driver)
377         self._test_results.append(test_result)
378         return test_result
379
380     def _run_single_test(self, test_input, driver):
381         # The image hash is used to avoid doing an image dump if the
382         # checksums match, so it should be set to a blank value if we
383         # are generating a new baseline.  (Otherwise, an image from a
384         # previous run will be copied into the baseline."""
385         if self._should_fetch_expected_checksum():
386             test_input.image_hash = self._port.expected_checksum(
387                 test_input.filename)
388         test_output = driver.run_test(test_input)
389         return self._process_output(test_input.filename, test_output)
390
391     def _process_output(self, test_filename, test_output):
392         """Receives the output from a DumpRenderTree process, subjects it to a
393         number of tests, and returns a list of failure types the test produced.
394
395         Args:
396         test_filename: full path to the test in question.
397         test_output: a TestOutput object containing the output of the test
398
399         Returns: a TestResult object
400         """
401         failures = []
402
403         if test_output.crash:
404             failures.append(test_failures.FailureCrash())
405         if test_output.timeout:
406             failures.append(test_failures.FailureTimeout())
407
408         test_name = self._port.relative_test_filename(test_filename)
409         if test_output.crash:
410             _log.debug("%s Stacktrace for %s:\n%s" %
411                        (self._name, test_name, test_output.error))
412             filename = os.path.join(self._options.results_directory, test_name)
413             filename = os.path.splitext(filename)[0] + "-stack.txt"
414             self._port.maybe_make_directory(os.path.split(filename)[0])
415             with codecs.open(filename, "wb", "utf-8") as file:
416                 file.write(test_output.error)
417         elif test_output.error:
418             _log.debug("%s %s output stderr lines:\n%s" %
419                        (self._name, test_name, test_output.error))
420
421         expected_test_output = self._expected_test_output(test_filename)
422
423         # Check the output and save the results.
424         start_time = time.time()
425         time_for_diffs = {}
426         for test_type in self._test_types:
427             start_diff_time = time.time()
428             new_failures = test_type.compare_output(self._port,
429                                                     test_filename,
430                                                     self._test_args,
431                                                     test_output,
432                                                     expected_test_output)
433             # Don't add any more failures if we already have a crash, so we
434             # don't double-report those tests. We do double-report for timeouts
435             # since we still want to see the text and image output.
436             if not test_output.crash:
437                 failures.extend(new_failures)
438             time_for_diffs[test_type.__class__.__name__] = (
439                 time.time() - start_diff_time)
440
441         total_time_for_all_diffs = time.time() - start_diff_time
442         return test_results.TestResult(test_filename,
443                                        failures,
444                                        test_output.test_time,
445                                        total_time_for_all_diffs,
446                                        time_for_diffs)
447
448     def _expected_test_output(self, filename):
449         """Returns an expected TestOutput object."""
450         return test_output.TestOutput(self._port.expected_text(filename),
451                                     self._port.expected_image(filename),
452                                     self._port.expected_checksum(filename))
453
454     def _tear_down_test(self, test_input, result):
455         self._num_tests += 1
456         self._batch_count += 1
457         self._tests_run_file.write(test_input.filename + "\n")
458         test_name = self._port.relative_test_filename(test_input.filename)
459
460         if result.failures:
461             # Check and kill DumpRenderTree if we need to.
462             if any([f.should_kill_dump_render_tree() for f in result.failures]):
463                 self._kill_dump_render_tree()
464                 # Reset the batch count since the shell just bounced.
465                 self._batch_count = 0
466
467             # Print the error message(s).
468             _log.debug("%s %s failed:" % (self._name, test_name))
469             for f in result.failures:
470                 _log.debug("%s  %s" % (self._name, f.message()))
471         else:
472             _log.debug("%s %s passed" % (self._name, test_name))
473
474         self._result_queue.put(result.dumps())
475
476         if self._batch_size > 0 and self._batch_count >= self._batch_size:
477             # Bounce the shell and reset count.
478             self._kill_dump_render_tree()
479             self._batch_count = 0
480
481         if self._test_runner:
482             self._test_runner.update_summary(self._result_summary)
483
484     def _start_servers_with_lock(self):
485         self._http_lock_wait_begin = time.time()
486         _log.debug('Acquiring http lock ...')
487         self._port.acquire_http_lock()
488         _log.debug('Starting HTTP server ...')
489         self._port.start_http_server()
490         _log.debug('Starting WebSocket server ...')
491         self._port.start_websocket_server()
492         self._http_lock_wait_end = time.time()
493         self._have_http_lock = True
494
495     def _http_lock_wait_time(self):
496         """Return the time what http locking takes."""
497         if self._http_lock_wait_begin == 0:
498             return 0
499         if self._http_lock_wait_end == 0:
500             return time.time() - self._http_lock_wait_begin
501         return self._http_lock_wait_end - self._http_lock_wait_begin
502
503     def _stop_servers_with_lock(self):
504         """Stop the servers and release http lock."""
505         if self._have_http_lock:
506             _log.debug('Stopping HTTP server ...')
507             self._port.stop_http_server()
508             _log.debug('Stopping WebSocket server ...')
509             self._port.stop_websocket_server()
510             _log.debug('Release http lock ...')
511             self._port.release_http_lock()
512             self._have_http_lock = False
513
514     def _kill_dump_render_tree(self):
515         """Kill the DumpRenderTree process if it's running."""
516         if self._driver:
517             self._driver.stop()
518             self._driver = None