2010-11-24 Dirk Pranke <dpranke@chromium.org>
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / layout_package / dump_render_tree_thread.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """A Thread object for running DumpRenderTree and processing URLs from a
32 shared queue.
33
34 Each thread runs a separate instance of the DumpRenderTree binary and validates
35 the output.  When there are no more URLs to process in the shared queue, the
36 thread exits.
37 """
38
39 from __future__ import with_statement
40
41 import codecs
42 import copy
43 import logging
44 import os
45 import Queue
46 import signal
47 import sys
48 import thread
49 import threading
50 import time
51
52
53 from webkitpy.layout_tests.test_types import image_diff
54 from webkitpy.layout_tests.test_types import test_type_base
55 from webkitpy.layout_tests.test_types import text_diff
56
57 import test_failures
58 import test_output
59 import test_results
60
61 _log = logging.getLogger("webkitpy.layout_tests.layout_package."
62                          "dump_render_tree_thread")
63
64
65 def _expected_test_output(port, filename):
66     """Returns an expected TestOutput object."""
67     return test_output.TestOutput(port.expected_text(filename),
68                                   port.expected_image(filename),
69                                   port.expected_checksum(filename))
70
71 def _process_output(port, options, test_input, test_types, test_args,
72                     test_output, worker_name):
73     """Receives the output from a DumpRenderTree process, subjects it to a
74     number of tests, and returns a list of failure types the test produced.
75
76     Args:
77       port: port-specific hooks
78       options: command line options argument from optparse
79       proc: an active DumpRenderTree process
80       test_input: Object containing the test filename and timeout
81       test_types: list of test types to subject the output to
82       test_args: arguments to be passed to each test
83       test_output: a TestOutput object containing the output of the test
84       worker_name: worker name for logging
85
86     Returns: a TestResult object
87     """
88     failures = []
89
90     if test_output.crash:
91         failures.append(test_failures.FailureCrash())
92     if test_output.timeout:
93         failures.append(test_failures.FailureTimeout())
94
95     test_name = port.relative_test_filename(test_input.filename)
96     if test_output.crash:
97         _log.debug("%s Stacktrace for %s:\n%s" % (worker_name, test_name,
98                                                   test_output.error))
99         filename = os.path.join(options.results_directory, test_name)
100         filename = os.path.splitext(filename)[0] + "-stack.txt"
101         port.maybe_make_directory(os.path.split(filename)[0])
102         with codecs.open(filename, "wb", "utf-8") as file:
103             file.write(test_output.error)
104     elif test_output.error:
105         _log.debug("%s %s output stderr lines:\n%s" % (worker_name, test_name,
106                                                        test_output.error))
107
108     expected_test_output = _expected_test_output(port, test_input.filename)
109
110     # Check the output and save the results.
111     start_time = time.time()
112     time_for_diffs = {}
113     for test_type in test_types:
114         start_diff_time = time.time()
115         new_failures = test_type.compare_output(port, test_input.filename,
116                                                 test_args, test_output,
117                                                 expected_test_output)
118         # Don't add any more failures if we already have a crash, so we don't
119         # double-report those tests. We do double-report for timeouts since
120         # we still want to see the text and image output.
121         if not test_output.crash:
122             failures.extend(new_failures)
123         time_for_diffs[test_type.__class__.__name__] = (
124             time.time() - start_diff_time)
125
126     total_time_for_all_diffs = time.time() - start_diff_time
127     return test_results.TestResult(test_input.filename, failures, test_output.test_time,
128                                    total_time_for_all_diffs, time_for_diffs)
129
130
131 def _pad_timeout(timeout):
132     """Returns a safe multiple of the per-test timeout value to use
133     to detect hung test threads.
134
135     """
136     # When we're running one test per DumpRenderTree process, we can
137     # enforce a hard timeout.  The DumpRenderTree watchdog uses 2.5x
138     # the timeout; we want to be larger than that.
139     return timeout * 3
140
141
142 def _milliseconds_to_seconds(msecs):
143     return float(msecs) / 1000.0
144
145
146 def _should_fetch_expected_checksum(options):
147     return options.pixel_tests and not (options.new_baseline or options.reset_results)
148
149
150 def _run_single_test(port, options, test_input, test_types, test_args, driver, worker_name):
151     # FIXME: Pull this into TestShellThread._run().
152
153     # The image hash is used to avoid doing an image dump if the
154     # checksums match, so it should be set to a blank value if we
155     # are generating a new baseline.  (Otherwise, an image from a
156     # previous run will be copied into the baseline."""
157     if _should_fetch_expected_checksum(options):
158         test_input.image_hash = port.expected_checksum(test_input.filename)
159     test_output = driver.run_test(test_input)
160     return _process_output(port, options, test_input, test_types, test_args,
161                            test_output, worker_name)
162
163
164 class SingleTestThread(threading.Thread):
165     """Thread wrapper for running a single test file."""
166
167     def __init__(self, port, options, worker_number, worker_name,
168                  test_input, test_types, test_args):
169         """
170         Args:
171           port: object implementing port-specific hooks
172           options: command line argument object from optparse
173           worker_number: worker number for tests
174               (FIXME: this should be passed to port.create_driver()).
175           worker_name: for logging
176           test_input: Object containing the test filename and timeout
177           test_types: A list of TestType objects to run the test output
178               against.
179           test_args: A TestArguments object to pass to each TestType.
180         """
181
182         threading.Thread.__init__(self)
183         self._port = port
184         self._options = options
185         self._test_input = test_input
186         self._test_types = test_types
187         self._test_args = test_args
188         self._driver = None
189         self._worker_number = worker_number
190         self._name = worker_name
191
192     def run(self):
193         self._covered_run()
194
195     def _covered_run(self):
196         # FIXME: this is a separate routine to work around a bug
197         # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
198         self._driver = self._port.create_driver(self._test_args.png_path,
199                                                 self._options)
200         self._driver.start()
201         self._test_result = _run_single_test(self._port, self._options,
202                                              self._test_input, self._test_types,
203                                              self._test_args, self._driver,
204                                              self._name)
205         self._driver.stop()
206
207     def get_test_result(self):
208         return self._test_result
209
210
211 class WatchableThread(threading.Thread):
212     """This class abstracts an interface used by
213     run_webkit_tests.TestRunner._wait_for_threads_to_finish for thread
214     management."""
215     def __init__(self):
216         threading.Thread.__init__(self)
217         self._canceled = False
218         self._exception_info = None
219         self._next_timeout = None
220         self._thread_id = None
221
222     def cancel(self):
223         """Set a flag telling this thread to quit."""
224         self._canceled = True
225
226     def clear_next_timeout(self):
227         """Mark a flag telling this thread to stop setting timeouts."""
228         self._timeout = 0
229
230     def exception_info(self):
231         """If run() terminated on an uncaught exception, return it here
232         ((type, value, traceback) tuple).
233         Returns None if run() terminated normally. Meant to be called after
234         joining this thread."""
235         return self._exception_info
236
237     def id(self):
238         """Return a thread identifier."""
239         return self._thread_id
240
241     def next_timeout(self):
242         """Return the time the test is supposed to finish by."""
243         return self._next_timeout
244
245
246 class TestShellThread(WatchableThread):
247     def __init__(self, port, options, worker_number, worker_name,
248                  filename_list_queue, result_queue):
249         """Initialize all the local state for this DumpRenderTree thread.
250
251         Args:
252           port: interface to port-specific hooks
253           options: command line options argument from optparse
254           worker_number: identifier for a particular worker thread.
255           worker_name: for logging.
256           filename_list_queue: A thread safe Queue class that contains lists
257               of tuples of (filename, uri) pairs.
258           result_queue: A thread safe Queue class that will contain
259               serialized TestResult objects.
260         """
261         WatchableThread.__init__(self)
262         self._port = port
263         self._options = options
264         self._worker_number = worker_number
265         self._name = worker_name
266         self._filename_list_queue = filename_list_queue
267         self._result_queue = result_queue
268         self._filename_list = []
269         self._driver = None
270         self._test_group_timing_stats = {}
271         self._test_results = []
272         self._num_tests = 0
273         self._start_time = 0
274         self._stop_time = 0
275         self._have_http_lock = False
276         self._http_lock_wait_begin = 0
277         self._http_lock_wait_end = 0
278
279         self._test_types = []
280         for cls in self._get_test_type_classes():
281             self._test_types.append(cls(self._port,
282                                         self._options.results_directory))
283         self._test_args = self._get_test_args(worker_number)
284
285         # Current group of tests we're running.
286         self._current_group = None
287         # Number of tests in self._current_group.
288         self._num_tests_in_current_group = None
289         # Time at which we started running tests from self._current_group.
290         self._current_group_start_time = None
291
292     def _get_test_args(self, worker_number):
293         """Returns the tuple of arguments for tests and for DumpRenderTree."""
294         test_args = test_type_base.TestArguments()
295         test_args.png_path = None
296         if self._options.pixel_tests:
297             png_path = os.path.join(self._options.results_directory,
298                                     "png_result%s.png" %
299                                     self._worker_number)
300             test_args.png_path = png_path
301         test_args.new_baseline = self._options.new_baseline
302         test_args.reset_results = self._options.reset_results
303
304         return test_args
305
306     def _get_test_type_classes(self):
307         classes = [text_diff.TestTextDiff]
308         if self._options.pixel_tests:
309             classes.append(image_diff.ImageDiff)
310         return classes
311
312     def get_test_group_timing_stats(self):
313         """Returns a dictionary mapping test group to a tuple of
314         (number of tests in that group, time to run the tests)"""
315         return self._test_group_timing_stats
316
317     def get_test_results(self):
318         """Return the list of all tests run on this thread.
319
320         This is used to calculate per-thread statistics.
321
322         """
323         return self._test_results
324
325     def get_total_time(self):
326         return max(self._stop_time - self._start_time -
327                    self._http_lock_wait_time(), 0.0)
328
329     def get_num_tests(self):
330         return self._num_tests
331
332     def run(self):
333         """Delegate main work to a helper method and watch for uncaught
334         exceptions."""
335         self._covered_run()
336
337     def _covered_run(self):
338         # FIXME: this is a separate routine to work around a bug
339         # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
340         self._thread_id = thread.get_ident()
341         self._start_time = time.time()
342         self._num_tests = 0
343         try:
344             _log.debug('%s starting' % (self.getName()))
345             self._run(test_runner=None, result_summary=None)
346             _log.debug('%s done (%d tests)' % (self.getName(),
347                        self.get_num_tests()))
348         except KeyboardInterrupt:
349             self._exception_info = sys.exc_info()
350             _log.debug("%s interrupted" % self.getName())
351         except:
352             # Save the exception for our caller to see.
353             self._exception_info = sys.exc_info()
354             self._stop_time = time.time()
355             _log.error('%s dying, exception raised' % self.getName())
356
357         self._stop_time = time.time()
358
359     def run_in_main_thread(self, test_runner, result_summary):
360         """This hook allows us to run the tests from the main thread if
361         --num-test-shells==1, instead of having to always run two or more
362         threads. This allows us to debug the test harness without having to
363         do multi-threaded debugging."""
364         self._run(test_runner, result_summary)
365
366     def cancel(self):
367         """Clean up http lock and set a flag telling this thread to quit."""
368         self._stop_servers_with_lock()
369         WatchableThread.cancel(self)
370
371     def next_timeout(self):
372         """Return the time the test is supposed to finish by."""
373         if self._next_timeout:
374             return self._next_timeout + self._http_lock_wait_time()
375         return self._next_timeout
376
377     def _http_lock_wait_time(self):
378         """Return the time what http locking takes."""
379         if self._http_lock_wait_begin == 0:
380             return 0
381         if self._http_lock_wait_end == 0:
382             return time.time() - self._http_lock_wait_begin
383         return self._http_lock_wait_end - self._http_lock_wait_begin
384
385     def _run(self, test_runner, result_summary):
386         """Main work entry point of the thread. Basically we pull urls from the
387         filename queue and run the tests until we run out of urls.
388
389         If test_runner is not None, then we call test_runner.UpdateSummary()
390         with the results of each test."""
391         batch_size = self._options.batch_size
392         batch_count = 0
393
394         # Append tests we're running to the existing tests_run.txt file.
395         # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput.
396         tests_run_filename = os.path.join(self._options.results_directory,
397                                           "tests_run.txt")
398         tests_run_file = codecs.open(tests_run_filename, "a", "utf-8")
399
400         while True:
401             if self._canceled:
402                 _log.debug('Testing cancelled')
403                 tests_run_file.close()
404                 return
405
406             if len(self._filename_list) is 0:
407                 if self._current_group is not None:
408                     self._test_group_timing_stats[self._current_group] = \
409                         (self._num_tests_in_current_group,
410                          time.time() - self._current_group_start_time)
411
412                 try:
413                     self._current_group, self._filename_list = \
414                         self._filename_list_queue.get_nowait()
415                 except Queue.Empty:
416                     self._stop_servers_with_lock()
417                     self._kill_dump_render_tree()
418                     tests_run_file.close()
419                     return
420
421                 if self._current_group == "tests_to_http_lock":
422                     self._start_servers_with_lock()
423                 elif self._have_http_lock:
424                     self._stop_servers_with_lock()
425
426                 self._num_tests_in_current_group = len(self._filename_list)
427                 self._current_group_start_time = time.time()
428
429             test_input = self._filename_list.pop()
430
431             # We have a url, run tests.
432             batch_count += 1
433             self._num_tests += 1
434             if self._options.run_singly:
435                 result = self._run_test_in_another_thread(test_input)
436             else:
437                 result = self._run_test_in_this_thread(test_input)
438
439             filename = test_input.filename
440             tests_run_file.write(filename + "\n")
441             if result.failures:
442                 # Check and kill DumpRenderTree if we need to.
443                 if len([1 for f in result.failures
444                         if f.should_kill_dump_render_tree()]):
445                     self._kill_dump_render_tree()
446                     # Reset the batch count since the shell just bounced.
447                     batch_count = 0
448                 # Print the error message(s).
449                 error_str = '\n'.join(['  ' + f.message() for
450                                        f in result.failures])
451                 _log.debug("%s %s failed:\n%s" % (self.getName(),
452                            self._port.relative_test_filename(filename),
453                            error_str))
454             else:
455                 _log.debug("%s %s passed" % (self.getName(),
456                            self._port.relative_test_filename(filename)))
457             self._result_queue.put(result.dumps())
458
459             if batch_size > 0 and batch_count >= batch_size:
460                 # Bounce the shell and reset count.
461                 self._kill_dump_render_tree()
462                 batch_count = 0
463
464             if test_runner:
465                 test_runner.update_summary(result_summary)
466
467     def _run_test_in_another_thread(self, test_input):
468         """Run a test in a separate thread, enforcing a hard time limit.
469
470         Since we can only detect the termination of a thread, not any internal
471         state or progress, we can only run per-test timeouts when running test
472         files singly.
473
474         Args:
475           test_input: Object containing the test filename and timeout
476
477         Returns:
478           A TestResult
479         """
480         worker = SingleTestThread(self._port,
481                                   self._options,
482                                   self._worker_number,
483                                   self._name,
484                                   test_input,
485                                   self._test_types,
486                                   self._test_args)
487
488         worker.start()
489
490         thread_timeout = _milliseconds_to_seconds(
491             _pad_timeout(int(test_input.timeout)))
492         thread._next_timeout = time.time() + thread_timeout
493         worker.join(thread_timeout)
494         if worker.isAlive():
495             # If join() returned with the thread still running, the
496             # DumpRenderTree is completely hung and there's nothing
497             # more we can do with it.  We have to kill all the
498             # DumpRenderTrees to free it up. If we're running more than
499             # one DumpRenderTree thread, we'll end up killing the other
500             # DumpRenderTrees too, introducing spurious crashes. We accept
501             # that tradeoff in order to avoid losing the rest of this
502             # thread's results.
503             _log.error('Test thread hung: killing all DumpRenderTrees')
504             if worker._driver:
505                 worker._driver.stop()
506
507         try:
508             result = worker.get_test_result()
509         except AttributeError, e:
510             # This gets raised if the worker thread has already exited.
511             failures = []
512             _log.error('Cannot get results of test: %s' %
513                        test_input.filename)
514             result = test_results.TestResult(test_input.filename, failures=[],
515                 test_run_time=0, total_time_for_all_diffs=0, time_for_diffs={})
516
517         return result
518
519     def _run_test_in_this_thread(self, test_input):
520         """Run a single test file using a shared DumpRenderTree process.
521
522         Args:
523           test_input: Object containing the test filename, uri and timeout
524
525         Returns: a TestResult object.
526         """
527         self._ensure_dump_render_tree_is_running()
528         thread_timeout = _milliseconds_to_seconds(
529              _pad_timeout(int(test_input.timeout)))
530         self._next_timeout = time.time() + thread_timeout
531         test_result = _run_single_test(self._port, self._options, test_input,
532                                        self._test_types, self._test_args,
533                                        self._driver, self._name)
534         self._test_results.append(test_result)
535         return test_result
536
537     def _ensure_dump_render_tree_is_running(self):
538         """Start the shared DumpRenderTree, if it's not running.
539
540         This is not for use when running tests singly, since those each start
541         a separate DumpRenderTree in their own thread.
542
543         """
544         # poll() is not threadsafe and can throw OSError due to:
545         # http://bugs.python.org/issue1731717
546         if (not self._driver or self._driver.poll() is not None):
547             self._driver = self._port.create_driver(self._test_args.png_path,
548                                                     self._options)
549             self._driver.start()
550
551     def _start_servers_with_lock(self):
552         """Acquire http lock and start the servers."""
553         self._http_lock_wait_begin = time.time()
554         _log.debug('Acquire http lock ...')
555         self._port.acquire_http_lock()
556         _log.debug('Starting HTTP server ...')
557         self._port.start_http_server()
558         _log.debug('Starting WebSocket server ...')
559         self._port.start_websocket_server()
560         self._http_lock_wait_end = time.time()
561         self._have_http_lock = True
562
563     def _stop_servers_with_lock(self):
564         """Stop the servers and release http lock."""
565         if self._have_http_lock:
566             _log.debug('Stopping HTTP server ...')
567             self._port.stop_http_server()
568             _log.debug('Stopping WebSocket server ...')
569             self._port.stop_websocket_server()
570             _log.debug('Release http lock ...')
571             self._port.release_http_lock()
572             self._have_http_lock = False
573
574     def _kill_dump_render_tree(self):
575         """Kill the DumpRenderTree process if it's running."""
576         if self._driver:
577             self._driver.stop()
578             self._driver = None