bd52e652da69dd13d82de1ae2fdaee34b1c88d6b
[WebKit.git] / Tools / Scripts / webkitpy / layout_tests / layout_package / dump_render_tree_thread.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """A Thread object for running DumpRenderTree and processing URLs from a
32 shared queue.
33
34 Each thread runs a separate instance of the DumpRenderTree binary and validates
35 the output.  When there are no more URLs to process in the shared queue, the
36 thread exits.
37 """
38
39 import copy
40 import logging
41 import os
42 import Queue
43 import signal
44 import sys
45 import thread
46 import threading
47 import time
48
49
50 from webkitpy.layout_tests.test_types import image_diff
51 from webkitpy.layout_tests.test_types import test_type_base
52 from webkitpy.layout_tests.test_types import text_diff
53
54 import test_failures
55 import test_output
56 import test_results
57
58 _log = logging.getLogger("webkitpy.layout_tests.layout_package."
59                          "dump_render_tree_thread")
60
61
62 def _expected_test_output(port, filename):
63     """Returns an expected TestOutput object."""
64     return test_output.TestOutput(port.expected_text(filename),
65                                   port.expected_image(filename),
66                                   port.expected_checksum(filename))
67
68 def _process_output(port, options, test_input, test_types, test_args,
69                     test_output, worker_name):
70     """Receives the output from a DumpRenderTree process, subjects it to a
71     number of tests, and returns a list of failure types the test produced.
72
73     Args:
74       port: port-specific hooks
75       options: command line options argument from optparse
76       proc: an active DumpRenderTree process
77       test_input: Object containing the test filename and timeout
78       test_types: list of test types to subject the output to
79       test_args: arguments to be passed to each test
80       test_output: a TestOutput object containing the output of the test
81       worker_name: worker name for logging
82
83     Returns: a TestResult object
84     """
85     failures = []
86     fs = port._filesystem
87
88     if test_output.crash:
89         failures.append(test_failures.FailureCrash())
90     if test_output.timeout:
91         failures.append(test_failures.FailureTimeout())
92
93     test_name = port.relative_test_filename(test_input.filename)
94     if test_output.crash:
95         _log.debug("%s Stacktrace for %s:\n%s" % (worker_name, test_name,
96                                                   test_output.error))
97         filename = fs.join(options.results_directory, test_name)
98         filename = fs.splitext(filename)[0] + "-stack.txt"
99         fs.maybe_make_directory(fs.dirname(filename))
100         fs.write_text_file(filename, test_output.error)
101     elif test_output.error:
102         _log.debug("%s %s output stderr lines:\n%s" % (worker_name, test_name,
103                                                        test_output.error))
104
105     expected_test_output = _expected_test_output(port, test_input.filename)
106
107     # Check the output and save the results.
108     start_time = time.time()
109     time_for_diffs = {}
110     for test_type in test_types:
111         start_diff_time = time.time()
112         new_failures = test_type.compare_output(port, test_input.filename,
113                                                 test_args, test_output,
114                                                 expected_test_output)
115         # Don't add any more failures if we already have a crash, so we don't
116         # double-report those tests. We do double-report for timeouts since
117         # we still want to see the text and image output.
118         if not test_output.crash:
119             failures.extend(new_failures)
120         time_for_diffs[test_type.__class__.__name__] = (
121             time.time() - start_diff_time)
122
123     total_time_for_all_diffs = time.time() - start_diff_time
124     return test_results.TestResult(test_input.filename, failures, test_output.test_time,
125                                    total_time_for_all_diffs, time_for_diffs)
126
127
128 def _pad_timeout(timeout):
129     """Returns a safe multiple of the per-test timeout value to use
130     to detect hung test threads.
131
132     """
133     # When we're running one test per DumpRenderTree process, we can
134     # enforce a hard timeout.  The DumpRenderTree watchdog uses 2.5x
135     # the timeout; we want to be larger than that.
136     return timeout * 3
137
138
139 def _milliseconds_to_seconds(msecs):
140     return float(msecs) / 1000.0
141
142
143 def _should_fetch_expected_checksum(options):
144     return options.pixel_tests and not (options.new_baseline or options.reset_results)
145
146
147 def _run_single_test(port, options, test_input, test_types, test_args, driver, worker_name):
148     # FIXME: Pull this into TestShellThread._run().
149
150     # The image hash is used to avoid doing an image dump if the
151     # checksums match, so it should be set to a blank value if we
152     # are generating a new baseline.  (Otherwise, an image from a
153     # previous run will be copied into the baseline."""
154     if _should_fetch_expected_checksum(options):
155         test_input.image_hash = port.expected_checksum(test_input.filename)
156     test_output = driver.run_test(test_input)
157     return _process_output(port, options, test_input, test_types, test_args,
158                            test_output, worker_name)
159
160
161 class SingleTestThread(threading.Thread):
162     """Thread wrapper for running a single test file."""
163
164     def __init__(self, port, options, worker_number, worker_name,
165                  test_input, test_types, test_args):
166         """
167         Args:
168           port: object implementing port-specific hooks
169           options: command line argument object from optparse
170           worker_number: worker number for tests
171           worker_name: for logging
172           test_input: Object containing the test filename and timeout
173           test_types: A list of TestType objects to run the test output
174               against.
175           test_args: A TestArguments object to pass to each TestType.
176         """
177
178         threading.Thread.__init__(self)
179         self._port = port
180         self._options = options
181         self._test_input = test_input
182         self._test_types = test_types
183         self._test_args = test_args
184         self._driver = None
185         self._worker_number = worker_number
186         self._name = worker_name
187
188     def run(self):
189         self._covered_run()
190
191     def _covered_run(self):
192         # FIXME: this is a separate routine to work around a bug
193         # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
194         self._driver = self._port.create_driver(self._worker_number)
195         self._driver.start()
196         self._test_result = _run_single_test(self._port, self._options,
197                                              self._test_input, self._test_types,
198                                              self._test_args, self._driver,
199                                              self._name)
200         self._driver.stop()
201
202     def get_test_result(self):
203         return self._test_result
204
205
206 class WatchableThread(threading.Thread):
207     """This class abstracts an interface used by
208     run_webkit_tests.TestRunner._wait_for_threads_to_finish for thread
209     management."""
210     def __init__(self):
211         threading.Thread.__init__(self)
212         self._canceled = False
213         self._exception_info = None
214         self._next_timeout = None
215         self._thread_id = None
216
217     def cancel(self):
218         """Set a flag telling this thread to quit."""
219         self._canceled = True
220
221     def clear_next_timeout(self):
222         """Mark a flag telling this thread to stop setting timeouts."""
223         self._timeout = 0
224
225     def exception_info(self):
226         """If run() terminated on an uncaught exception, return it here
227         ((type, value, traceback) tuple).
228         Returns None if run() terminated normally. Meant to be called after
229         joining this thread."""
230         return self._exception_info
231
232     def id(self):
233         """Return a thread identifier."""
234         return self._thread_id
235
236     def next_timeout(self):
237         """Return the time the test is supposed to finish by."""
238         return self._next_timeout
239
240
241 class TestShellThread(WatchableThread):
242     def __init__(self, port, options, worker_number, worker_name,
243                  filename_list_queue, result_queue):
244         """Initialize all the local state for this DumpRenderTree thread.
245
246         Args:
247           port: interface to port-specific hooks
248           options: command line options argument from optparse
249           worker_number: identifier for a particular worker thread.
250           worker_name: for logging.
251           filename_list_queue: A thread safe Queue class that contains lists
252               of tuples of (filename, uri) pairs.
253           result_queue: A thread safe Queue class that will contain
254               serialized TestResult objects.
255         """
256         WatchableThread.__init__(self)
257         self._port = port
258         self._options = options
259         self._worker_number = worker_number
260         self._name = worker_name
261         self._filename_list_queue = filename_list_queue
262         self._result_queue = result_queue
263         self._filename_list = []
264         self._driver = None
265         self._test_group_timing_stats = {}
266         self._test_results = []
267         self._num_tests = 0
268         self._start_time = 0
269         self._stop_time = 0
270         self._have_http_lock = False
271         self._http_lock_wait_begin = 0
272         self._http_lock_wait_end = 0
273
274         self._test_types = []
275         for cls in self._get_test_type_classes():
276             self._test_types.append(cls(self._port,
277                                         self._options.results_directory))
278         self._test_args = self._get_test_args(worker_number)
279
280         # Current group of tests we're running.
281         self._current_group = None
282         # Number of tests in self._current_group.
283         self._num_tests_in_current_group = None
284         # Time at which we started running tests from self._current_group.
285         self._current_group_start_time = None
286
287     def _get_test_args(self, worker_number):
288         """Returns the tuple of arguments for tests and for DumpRenderTree."""
289         test_args = test_type_base.TestArguments()
290         test_args.new_baseline = self._options.new_baseline
291         test_args.reset_results = self._options.reset_results
292
293         return test_args
294
295     def _get_test_type_classes(self):
296         classes = [text_diff.TestTextDiff]
297         if self._options.pixel_tests:
298             classes.append(image_diff.ImageDiff)
299         return classes
300
301     def get_test_group_timing_stats(self):
302         """Returns a dictionary mapping test group to a tuple of
303         (number of tests in that group, time to run the tests)"""
304         return self._test_group_timing_stats
305
306     def get_test_results(self):
307         """Return the list of all tests run on this thread.
308
309         This is used to calculate per-thread statistics.
310
311         """
312         return self._test_results
313
314     def get_total_time(self):
315         return max(self._stop_time - self._start_time -
316                    self._http_lock_wait_time(), 0.0)
317
318     def get_num_tests(self):
319         return self._num_tests
320
321     def run(self):
322         """Delegate main work to a helper method and watch for uncaught
323         exceptions."""
324         self._covered_run()
325
326     def _covered_run(self):
327         # FIXME: this is a separate routine to work around a bug
328         # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
329         self._thread_id = thread.get_ident()
330         self._start_time = time.time()
331         self._num_tests = 0
332         try:
333             _log.debug('%s starting' % (self.getName()))
334             self._run(test_runner=None, result_summary=None)
335             _log.debug('%s done (%d tests)' % (self.getName(),
336                        self.get_num_tests()))
337         except KeyboardInterrupt:
338             self._exception_info = sys.exc_info()
339             _log.debug("%s interrupted" % self.getName())
340         except:
341             # Save the exception for our caller to see.
342             self._exception_info = sys.exc_info()
343             self._stop_time = time.time()
344             _log.error('%s dying, exception raised' % self.getName())
345
346         self._stop_time = time.time()
347
348     def run_in_main_thread(self, test_runner, result_summary):
349         """This hook allows us to run the tests from the main thread if
350         --num-test-shells==1, instead of having to always run two or more
351         threads. This allows us to debug the test harness without having to
352         do multi-threaded debugging."""
353         self._run(test_runner, result_summary)
354
355     def cancel(self):
356         """Clean up http lock and set a flag telling this thread to quit."""
357         self._stop_servers_with_lock()
358         WatchableThread.cancel(self)
359
360     def next_timeout(self):
361         """Return the time the test is supposed to finish by."""
362         if self._next_timeout:
363             return self._next_timeout + self._http_lock_wait_time()
364         return self._next_timeout
365
366     def _http_lock_wait_time(self):
367         """Return the time what http locking takes."""
368         if self._http_lock_wait_begin == 0:
369             return 0
370         if self._http_lock_wait_end == 0:
371             return time.time() - self._http_lock_wait_begin
372         return self._http_lock_wait_end - self._http_lock_wait_begin
373
374     def _run(self, test_runner, result_summary):
375         """Main work entry point of the thread. Basically we pull urls from the
376         filename queue and run the tests until we run out of urls.
377
378         If test_runner is not None, then we call test_runner.UpdateSummary()
379         with the results of each test."""
380         batch_size = self._options.batch_size
381         batch_count = 0
382
383         # Append tests we're running to the existing tests_run.txt file.
384         # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput.
385         tests_run_filename = self._port._filesystem.join(self._options.results_directory,
386                                           "tests_run.txt")
387         tests_run_file = self._port._filesystem.open_text_file_for_writing(tests_run_filename, append=True)
388
389         while True:
390             if self._canceled:
391                 _log.debug('Testing cancelled')
392                 tests_run_file.close()
393                 return
394
395             if len(self._filename_list) is 0:
396                 if self._current_group is not None:
397                     self._test_group_timing_stats[self._current_group] = \
398                         (self._num_tests_in_current_group,
399                          time.time() - self._current_group_start_time)
400
401                 try:
402                     self._current_group, self._filename_list = \
403                         self._filename_list_queue.get_nowait()
404                 except Queue.Empty:
405                     self._stop_servers_with_lock()
406                     self._kill_dump_render_tree()
407                     tests_run_file.close()
408                     return
409
410                 if self._current_group == "tests_to_http_lock":
411                     self._start_servers_with_lock()
412                 elif self._have_http_lock:
413                     self._stop_servers_with_lock()
414
415                 self._num_tests_in_current_group = len(self._filename_list)
416                 self._current_group_start_time = time.time()
417
418             test_input = self._filename_list.pop()
419
420             # We have a url, run tests.
421             batch_count += 1
422             self._num_tests += 1
423             if self._options.run_singly:
424                 result = self._run_test_in_another_thread(test_input)
425             else:
426                 result = self._run_test_in_this_thread(test_input)
427
428             filename = test_input.filename
429             tests_run_file.write(filename + "\n")
430             if result.failures:
431                 # Check and kill DumpRenderTree if we need to.
432                 if len([1 for f in result.failures
433                         if f.should_kill_dump_render_tree()]):
434                     self._kill_dump_render_tree()
435                     # Reset the batch count since the shell just bounced.
436                     batch_count = 0
437                 # Print the error message(s).
438                 error_str = '\n'.join(['  ' + f.message() for
439                                        f in result.failures])
440                 _log.debug("%s %s failed:\n%s" % (self.getName(),
441                            self._port.relative_test_filename(filename),
442                            error_str))
443             else:
444                 _log.debug("%s %s passed" % (self.getName(),
445                            self._port.relative_test_filename(filename)))
446             self._result_queue.put(result.dumps())
447
448             if batch_size > 0 and batch_count >= batch_size:
449                 # Bounce the shell and reset count.
450                 self._kill_dump_render_tree()
451                 batch_count = 0
452
453             if test_runner:
454                 test_runner.update_summary(result_summary)
455
456     def _run_test_in_another_thread(self, test_input):
457         """Run a test in a separate thread, enforcing a hard time limit.
458
459         Since we can only detect the termination of a thread, not any internal
460         state or progress, we can only run per-test timeouts when running test
461         files singly.
462
463         Args:
464           test_input: Object containing the test filename and timeout
465
466         Returns:
467           A TestResult
468         """
469         worker = SingleTestThread(self._port,
470                                   self._options,
471                                   self._worker_number,
472                                   self._name,
473                                   test_input,
474                                   self._test_types,
475                                   self._test_args)
476
477         worker.start()
478
479         thread_timeout = _milliseconds_to_seconds(
480             _pad_timeout(int(test_input.timeout)))
481         thread._next_timeout = time.time() + thread_timeout
482         worker.join(thread_timeout)
483         if worker.isAlive():
484             # If join() returned with the thread still running, the
485             # DumpRenderTree is completely hung and there's nothing
486             # more we can do with it.  We have to kill all the
487             # DumpRenderTrees to free it up. If we're running more than
488             # one DumpRenderTree thread, we'll end up killing the other
489             # DumpRenderTrees too, introducing spurious crashes. We accept
490             # that tradeoff in order to avoid losing the rest of this
491             # thread's results.
492             _log.error('Test thread hung: killing all DumpRenderTrees')
493             if worker._driver:
494                 worker._driver.stop()
495
496         try:
497             result = worker.get_test_result()
498         except AttributeError, e:
499             # This gets raised if the worker thread has already exited.
500             _log.error('Cannot get results of test: %s' % test_input.filename)
501             # FIXME: Seems we want a unique failure type here.
502             result = test_results.TestResult(test_input.filename)
503
504         return result
505
506     def _run_test_in_this_thread(self, test_input):
507         """Run a single test file using a shared DumpRenderTree process.
508
509         Args:
510           test_input: Object containing the test filename, uri and timeout
511
512         Returns: a TestResult object.
513         """
514         self._ensure_dump_render_tree_is_running()
515         thread_timeout = _milliseconds_to_seconds(
516              _pad_timeout(int(test_input.timeout)))
517         self._next_timeout = time.time() + thread_timeout
518         test_result = _run_single_test(self._port, self._options, test_input,
519                                        self._test_types, self._test_args,
520                                        self._driver, self._name)
521         self._test_results.append(test_result)
522         return test_result
523
524     def _ensure_dump_render_tree_is_running(self):
525         """Start the shared DumpRenderTree, if it's not running.
526
527         This is not for use when running tests singly, since those each start
528         a separate DumpRenderTree in their own thread.
529
530         """
531         # poll() is not threadsafe and can throw OSError due to:
532         # http://bugs.python.org/issue1731717
533         if not self._driver or self._driver.poll() is not None:
534             self._driver = self._port.create_driver(self._worker_number)
535             self._driver.start()
536
537     def _start_servers_with_lock(self):
538         """Acquire http lock and start the servers."""
539         self._http_lock_wait_begin = time.time()
540         _log.debug('Acquire http lock ...')
541         self._port.acquire_http_lock()
542         _log.debug('Starting HTTP server ...')
543         self._port.start_http_server()
544         _log.debug('Starting WebSocket server ...')
545         self._port.start_websocket_server()
546         self._http_lock_wait_end = time.time()
547         self._have_http_lock = True
548
549     def _stop_servers_with_lock(self):
550         """Stop the servers and release http lock."""
551         if self._have_http_lock:
552             _log.debug('Stopping HTTP server ...')
553             self._port.stop_http_server()
554             _log.debug('Stopping WebSocket server ...')
555             self._port.stop_websocket_server()
556             _log.debug('Release http lock ...')
557             self._port.release_http_lock()
558             self._have_http_lock = False
559
560     def _kill_dump_render_tree(self):
561         """Kill the DumpRenderTree process if it's running."""
562         if self._driver:
563             self._driver.stop()
564             self._driver = None