1bac7ef18e3e2ea7a6dcf5fa37402678e7ae2a2b
[WebKit.git] / WebKitTools / Scripts / webkitpy / layout_tests / layout_package / dump_render_tree_thread.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """A Thread object for running DumpRenderTree and processing URLs from a
32 shared queue.
33
34 Each thread runs a separate instance of the DumpRenderTree binary and validates
35 the output.  When there are no more URLs to process in the shared queue, the
36 thread exits.
37 """
38
39 from __future__ import with_statement
40
41 import codecs
42 import copy
43 import logging
44 import os
45 import Queue
46 import signal
47 import sys
48 import thread
49 import threading
50 import time
51
52
53 from webkitpy.layout_tests.test_types import image_diff
54 from webkitpy.layout_tests.test_types import test_type_base
55 from webkitpy.layout_tests.test_types import text_diff
56
57 import test_failures
58 import test_output
59 import test_results
60
61 _log = logging.getLogger("webkitpy.layout_tests.layout_package."
62                          "dump_render_tree_thread")
63
64
65 def _expected_test_output(port, filename):
66     """Returns an expected TestOutput object."""
67     return test_output.TestOutput(port.expected_text(filename),
68                                   port.expected_image(filename),
69                                   port.expected_checksum(filename))
70
71 def _process_output(port, options, test_input, test_types, test_args,
72                     test_output, worker_name):
73     """Receives the output from a DumpRenderTree process, subjects it to a
74     number of tests, and returns a list of failure types the test produced.
75
76     Args:
77       port: port-specific hooks
78       options: command line options argument from optparse
79       proc: an active DumpRenderTree process
80       test_input: Object containing the test filename and timeout
81       test_types: list of test types to subject the output to
82       test_args: arguments to be passed to each test
83       test_output: a TestOutput object containing the output of the test
84       worker_name: worker name for logging
85
86     Returns: a TestResult object
87     """
88     failures = []
89
90     if test_output.crash:
91         failures.append(test_failures.FailureCrash())
92     if test_output.timeout:
93         failures.append(test_failures.FailureTimeout())
94
95     test_name = port.relative_test_filename(test_input.filename)
96     if test_output.crash:
97         _log.debug("%s Stacktrace for %s:\n%s" % (worker_name, test_name,
98                                                   test_output.error))
99         filename = os.path.join(options.results_directory, test_name)
100         filename = os.path.splitext(filename)[0] + "-stack.txt"
101         port.maybe_make_directory(os.path.split(filename)[0])
102         with codecs.open(filename, "wb", "utf-8") as file:
103             file.write(test_output.error)
104     elif test_output.error:
105         _log.debug("%s %s output stderr lines:\n%s" % (worker_name, test_name,
106                                                        test_output.error))
107
108     expected_test_output = _expected_test_output(port, test_input.filename)
109
110     # Check the output and save the results.
111     start_time = time.time()
112     time_for_diffs = {}
113     for test_type in test_types:
114         start_diff_time = time.time()
115         new_failures = test_type.compare_output(port, test_input.filename,
116                                                 test_args, test_output,
117                                                 expected_test_output)
118         # Don't add any more failures if we already have a crash, so we don't
119         # double-report those tests. We do double-report for timeouts since
120         # we still want to see the text and image output.
121         if not test_output.crash:
122             failures.extend(new_failures)
123         time_for_diffs[test_type.__class__.__name__] = (
124             time.time() - start_diff_time)
125
126     total_time_for_all_diffs = time.time() - start_diff_time
127     return test_results.TestResult(test_input.filename, failures, test_output.test_time,
128                                    total_time_for_all_diffs, time_for_diffs)
129
130
131 def _pad_timeout(timeout):
132     """Returns a safe multiple of the per-test timeout value to use
133     to detect hung test threads.
134
135     """
136     # When we're running one test per DumpRenderTree process, we can
137     # enforce a hard timeout.  The DumpRenderTree watchdog uses 2.5x
138     # the timeout; we want to be larger than that.
139     return timeout * 3
140
141
142 def _milliseconds_to_seconds(msecs):
143     return float(msecs) / 1000.0
144
145
146 def _should_fetch_expected_checksum(options):
147     return options.pixel_tests and not (options.new_baseline or options.reset_results)
148
149
150 def _run_single_test(port, options, test_input, test_types, test_args, driver, worker_name):
151     # FIXME: Pull this into TestShellThread._run().
152
153     # The image hash is used to avoid doing an image dump if the
154     # checksums match, so it should be set to a blank value if we
155     # are generating a new baseline.  (Otherwise, an image from a
156     # previous run will be copied into the baseline."""
157     if _should_fetch_expected_checksum(options):
158         test_input.image_hash = port.expected_checksum(test_input.filename)
159     test_output = driver.run_test(test_input)
160     return _process_output(port, options, test_input, test_types, test_args,
161                            test_output, worker_name)
162
163
164 class SingleTestThread(threading.Thread):
165     """Thread wrapper for running a single test file."""
166
167     def __init__(self, port, options, worker_number, worker_name,
168                  test_input, test_types, test_args):
169         """
170         Args:
171           port: object implementing port-specific hooks
172           options: command line argument object from optparse
173           worker_number: worker number for tests
174               (FIXME: this should be passed to port.create_driver()).
175           worker_name: for logging
176           test_input: Object containing the test filename and timeout
177           test_types: A list of TestType objects to run the test output
178               against.
179           test_args: A TestArguments object to pass to each TestType.
180         """
181
182         threading.Thread.__init__(self)
183         self._port = port
184         self._options = options
185         self._test_input = test_input
186         self._test_types = test_types
187         self._test_args = test_args
188         self._driver = None
189         self._worker_number = worker_number
190         self._name = worker_name
191
192     def run(self):
193         self._covered_run()
194
195     def _covered_run(self):
196         # FIXME: this is a separate routine to work around a bug
197         # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
198         self._driver = self._port.create_driver(self._test_args.png_path,
199                                                 self._options)
200         self._driver.start()
201         self._test_result = _run_single_test(self._port, self._options,
202                                              self._test_input, self._test_types,
203                                              self._test_args, self._driver,
204                                              self._name)
205         self._driver.stop()
206
207     def get_test_result(self):
208         return self._test_result
209
210
211 class WatchableThread(threading.Thread):
212     """This class abstracts an interface used by
213     run_webkit_tests.TestRunner._wait_for_threads_to_finish for thread
214     management."""
215     def __init__(self):
216         threading.Thread.__init__(self)
217         self._canceled = False
218         self._exception_info = None
219         self._next_timeout = None
220         self._thread_id = None
221
222     def cancel(self):
223         """Set a flag telling this thread to quit."""
224         self._canceled = True
225
226     def clear_next_timeout(self):
227         """Mark a flag telling this thread to stop setting timeouts."""
228         self._timeout = 0
229
230     def exception_info(self):
231         """If run() terminated on an uncaught exception, return it here
232         ((type, value, traceback) tuple).
233         Returns None if run() terminated normally. Meant to be called after
234         joining this thread."""
235         return self._exception_info
236
237     def id(self):
238         """Return a thread identifier."""
239         return self._thread_id
240
241     def next_timeout(self):
242         """Return the time the test is supposed to finish by."""
243         return self._next_timeout
244
245
246 class TestShellThread(WatchableThread):
247     def __init__(self, port, options, worker_number,
248                  filename_list_queue, result_queue):
249         """Initialize all the local state for this DumpRenderTree thread.
250
251         Args:
252           port: interface to port-specific hooks
253           options: command line options argument from optparse
254           worker_number: identifier for a particular worker thread.
255           filename_list_queue: A thread safe Queue class that contains lists
256               of tuples of (filename, uri) pairs.
257           result_queue: A thread safe Queue class that will contain
258               serialized TestResult objects.
259         """
260         WatchableThread.__init__(self)
261         self._port = port
262         self._options = options
263         self._worker_number = worker_number
264         self._name = 'worker-%d' % worker_number
265         self._filename_list_queue = filename_list_queue
266         self._result_queue = result_queue
267         self._filename_list = []
268         self._driver = None
269         self._test_group_timing_stats = {}
270         self._test_results = []
271         self._num_tests = 0
272         self._start_time = 0
273         self._stop_time = 0
274         self._have_http_lock = False
275         self._http_lock_wait_begin = 0
276         self._http_lock_wait_end = 0
277
278         self._test_types = []
279         for cls in self._get_test_type_classes():
280             self._test_types.append(cls(self._port,
281                                         self._options.results_directory))
282         self._test_args = self._get_test_args(worker_number)
283
284         # Current group of tests we're running.
285         self._current_group = None
286         # Number of tests in self._current_group.
287         self._num_tests_in_current_group = None
288         # Time at which we started running tests from self._current_group.
289         self._current_group_start_time = None
290
291     def _get_test_args(self, worker_number):
292         """Returns the tuple of arguments for tests and for DumpRenderTree."""
293         test_args = test_type_base.TestArguments()
294         test_args.png_path = None
295         if self._options.pixel_tests:
296             png_path = os.path.join(self._options.results_directory,
297                                     "png_result%s.png" %
298                                     self._worker_number)
299             test_args.png_path = png_path
300         test_args.new_baseline = self._options.new_baseline
301         test_args.reset_results = self._options.reset_results
302
303         return test_args
304
305     def _get_test_type_classes(self):
306         classes = [text_diff.TestTextDiff]
307         if self._options.pixel_tests:
308             classes.append(image_diff.ImageDiff)
309         return classes
310
311     def get_test_group_timing_stats(self):
312         """Returns a dictionary mapping test group to a tuple of
313         (number of tests in that group, time to run the tests)"""
314         return self._test_group_timing_stats
315
316     def get_test_results(self):
317         """Return the list of all tests run on this thread.
318
319         This is used to calculate per-thread statistics.
320
321         """
322         return self._test_results
323
324     def get_total_time(self):
325         return max(self._stop_time - self._start_time -
326                    self._http_lock_wait_time(), 0.0)
327
328     def get_num_tests(self):
329         return self._num_tests
330
331     def run(self):
332         """Delegate main work to a helper method and watch for uncaught
333         exceptions."""
334         self._covered_run()
335
336     def _covered_run(self):
337         # FIXME: this is a separate routine to work around a bug
338         # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
339         self._thread_id = thread.get_ident()
340         self._start_time = time.time()
341         self._num_tests = 0
342         try:
343             _log.debug('%s starting' % (self.getName()))
344             self._run(test_runner=None, result_summary=None)
345             _log.debug('%s done (%d tests)' % (self.getName(),
346                        self.get_num_tests()))
347         except KeyboardInterrupt:
348             self._exception_info = sys.exc_info()
349             _log.debug("%s interrupted" % self.getName())
350         except:
351             # Save the exception for our caller to see.
352             self._exception_info = sys.exc_info()
353             self._stop_time = time.time()
354             _log.error('%s dying, exception raised' % self.getName())
355
356         self._stop_time = time.time()
357
358     def run_in_main_thread(self, test_runner, result_summary):
359         """This hook allows us to run the tests from the main thread if
360         --num-test-shells==1, instead of having to always run two or more
361         threads. This allows us to debug the test harness without having to
362         do multi-threaded debugging."""
363         self._run(test_runner, result_summary)
364
365     def cancel(self):
366         """Clean up http lock and set a flag telling this thread to quit."""
367         self._stop_servers_with_lock()
368         WatchableThread.cancel(self)
369
370     def next_timeout(self):
371         """Return the time the test is supposed to finish by."""
372         if self._next_timeout:
373             return self._next_timeout + self._http_lock_wait_time()
374         return self._next_timeout
375
376     def _http_lock_wait_time(self):
377         """Return the time what http locking takes."""
378         if self._http_lock_wait_begin == 0:
379             return 0
380         if self._http_lock_wait_end == 0:
381             return time.time() - self._http_lock_wait_begin
382         return self._http_lock_wait_end - self._http_lock_wait_begin
383
384     def _run(self, test_runner, result_summary):
385         """Main work entry point of the thread. Basically we pull urls from the
386         filename queue and run the tests until we run out of urls.
387
388         If test_runner is not None, then we call test_runner.UpdateSummary()
389         with the results of each test."""
390         batch_size = self._options.batch_size
391         batch_count = 0
392
393         # Append tests we're running to the existing tests_run.txt file.
394         # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput.
395         tests_run_filename = os.path.join(self._options.results_directory,
396                                           "tests_run.txt")
397         tests_run_file = codecs.open(tests_run_filename, "a", "utf-8")
398
399         while True:
400             if self._canceled:
401                 _log.debug('Testing cancelled')
402                 tests_run_file.close()
403                 return
404
405             if len(self._filename_list) is 0:
406                 if self._current_group is not None:
407                     self._test_group_timing_stats[self._current_group] = \
408                         (self._num_tests_in_current_group,
409                          time.time() - self._current_group_start_time)
410
411                 try:
412                     self._current_group, self._filename_list = \
413                         self._filename_list_queue.get_nowait()
414                 except Queue.Empty:
415                     self._stop_servers_with_lock()
416                     self._kill_dump_render_tree()
417                     tests_run_file.close()
418                     return
419
420                 if self._current_group == "tests_to_http_lock":
421                     self._start_servers_with_lock()
422                 elif self._have_http_lock:
423                     self._stop_servers_with_lock()
424
425                 self._num_tests_in_current_group = len(self._filename_list)
426                 self._current_group_start_time = time.time()
427
428             test_input = self._filename_list.pop()
429
430             # We have a url, run tests.
431             batch_count += 1
432             self._num_tests += 1
433             if self._options.run_singly:
434                 result = self._run_test_in_another_thread(test_input)
435             else:
436                 result = self._run_test_in_this_thread(test_input)
437
438             filename = test_input.filename
439             tests_run_file.write(filename + "\n")
440             if result.failures:
441                 # Check and kill DumpRenderTree if we need to.
442                 if len([1 for f in result.failures
443                         if f.should_kill_dump_render_tree()]):
444                     self._kill_dump_render_tree()
445                     # Reset the batch count since the shell just bounced.
446                     batch_count = 0
447                 # Print the error message(s).
448                 error_str = '\n'.join(['  ' + f.message() for
449                                        f in result.failures])
450                 _log.debug("%s %s failed:\n%s" % (self.getName(),
451                            self._port.relative_test_filename(filename),
452                            error_str))
453             else:
454                 _log.debug("%s %s passed" % (self.getName(),
455                            self._port.relative_test_filename(filename)))
456             self._result_queue.put(result.dumps())
457
458             if batch_size > 0 and batch_count >= batch_size:
459                 # Bounce the shell and reset count.
460                 self._kill_dump_render_tree()
461                 batch_count = 0
462
463             if test_runner:
464                 test_runner.update_summary(result_summary)
465
466     def _run_test_in_another_thread(self, test_input):
467         """Run a test in a separate thread, enforcing a hard time limit.
468
469         Since we can only detect the termination of a thread, not any internal
470         state or progress, we can only run per-test timeouts when running test
471         files singly.
472
473         Args:
474           test_input: Object containing the test filename and timeout
475
476         Returns:
477           A TestResult
478         """
479         worker = SingleTestThread(self._port,
480                                   self._options,
481                                   self._worker_number,
482                                   self._name,
483                                   test_input,
484                                   self._test_types,
485                                   self._test_args)
486
487         worker.start()
488
489         thread_timeout = _milliseconds_to_seconds(
490             _pad_timeout(int(test_input.timeout)))
491         thread._next_timeout = time.time() + thread_timeout
492         worker.join(thread_timeout)
493         if worker.isAlive():
494             # If join() returned with the thread still running, the
495             # DumpRenderTree is completely hung and there's nothing
496             # more we can do with it.  We have to kill all the
497             # DumpRenderTrees to free it up. If we're running more than
498             # one DumpRenderTree thread, we'll end up killing the other
499             # DumpRenderTrees too, introducing spurious crashes. We accept
500             # that tradeoff in order to avoid losing the rest of this
501             # thread's results.
502             _log.error('Test thread hung: killing all DumpRenderTrees')
503             if worker._driver:
504                 worker._driver.stop()
505
506         try:
507             result = worker.get_test_result()
508         except AttributeError, e:
509             # This gets raised if the worker thread has already exited.
510             failures = []
511             _log.error('Cannot get results of test: %s' %
512                        test_input.filename)
513             result = test_results.TestResult(test_input.filename, failures=[],
514                 test_run_time=0, total_time_for_all_diffs=0, time_for_diffs={})
515
516         return result
517
518     def _run_test_in_this_thread(self, test_input):
519         """Run a single test file using a shared DumpRenderTree process.
520
521         Args:
522           test_input: Object containing the test filename, uri and timeout
523
524         Returns: a TestResult object.
525         """
526         self._ensure_dump_render_tree_is_running()
527         thread_timeout = _milliseconds_to_seconds(
528              _pad_timeout(int(test_input.timeout)))
529         self._next_timeout = time.time() + thread_timeout
530         test_result = _run_single_test(self._port, self._options, test_input,
531                                        self._test_types, self._test_args,
532                                        self._driver, self._name)
533         self._test_results.append(test_result)
534         return test_result
535
536     def _ensure_dump_render_tree_is_running(self):
537         """Start the shared DumpRenderTree, if it's not running.
538
539         This is not for use when running tests singly, since those each start
540         a separate DumpRenderTree in their own thread.
541
542         """
543         # poll() is not threadsafe and can throw OSError due to:
544         # http://bugs.python.org/issue1731717
545         if (not self._driver or self._driver.poll() is not None):
546             self._driver = self._port.create_driver(self._test_args.png_path,
547                                                     self._options)
548             self._driver.start()
549
550     def _start_servers_with_lock(self):
551         """Acquire http lock and start the servers."""
552         self._http_lock_wait_begin = time.time()
553         _log.debug('Acquire http lock ...')
554         self._port.acquire_http_lock()
555         _log.debug('Starting HTTP server ...')
556         self._port.start_http_server()
557         _log.debug('Starting WebSocket server ...')
558         self._port.start_websocket_server()
559         self._http_lock_wait_end = time.time()
560         self._have_http_lock = True
561
562     def _stop_servers_with_lock(self):
563         """Stop the servers and release http lock."""
564         if self._have_http_lock:
565             _log.debug('Stopping HTTP server ...')
566             self._port.stop_http_server()
567             _log.debug('Stopping WebSocket server ...')
568             self._port.stop_websocket_server()
569             _log.debug('Release http lock ...')
570             self._port.release_http_lock()
571             self._have_http_lock = False
572
573     def _kill_dump_render_tree(self):
574         """Kill the DumpRenderTree process if it's running."""
575         if self._driver:
576             self._driver.stop()
577             self._driver = None