Clean up ChunkedUpdateDrawingAreaProxy
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / layout_tests / layout_package / dump_render_tree_thread.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """A Thread object for running DumpRenderTree and processing URLs from a
32 shared queue.
33
34 Each thread runs a separate instance of the DumpRenderTree binary and validates
35 the output.  When there are no more URLs to process in the shared queue, the
36 thread exits.
37 """
38
39 from __future__ import with_statement
40
41 import codecs
42 import copy
43 import logging
44 import os
45 import Queue
46 import signal
47 import sys
48 import thread
49 import threading
50 import time
51
52
53 from webkitpy.layout_tests.test_types import image_diff
54 from webkitpy.layout_tests.test_types import test_type_base
55 from webkitpy.layout_tests.test_types import text_diff
56
57 import test_failures
58 import test_output
59 import test_results
60
61 _log = logging.getLogger("webkitpy.layout_tests.layout_package."
62                          "dump_render_tree_thread")
63
64
65 def _expected_test_output(port, filename):
66     """Returns an expected TestOutput object."""
67     return test_output.TestOutput(port.expected_text(filename),
68                                   port.expected_image(filename),
69                                   port.expected_checksum(filename))
70
71 def _process_output(port, options, test_input, test_types, test_args,
72                     test_output, worker_name):
73     """Receives the output from a DumpRenderTree process, subjects it to a
74     number of tests, and returns a list of failure types the test produced.
75
76     Args:
77       port: port-specific hooks
78       options: command line options argument from optparse
79       proc: an active DumpRenderTree process
80       test_input: Object containing the test filename and timeout
81       test_types: list of test types to subject the output to
82       test_args: arguments to be passed to each test
83       test_output: a TestOutput object containing the output of the test
84       worker_name: worker name for logging
85
86     Returns: a TestResult object
87     """
88     failures = []
89
90     if test_output.crash:
91         failures.append(test_failures.FailureCrash())
92     if test_output.timeout:
93         failures.append(test_failures.FailureTimeout())
94
95     test_name = port.relative_test_filename(test_input.filename)
96     if test_output.crash:
97         _log.debug("%s Stacktrace for %s:\n%s" % (worker_name, test_name,
98                                                   test_output.error))
99         filename = os.path.join(options.results_directory, test_name)
100         filename = os.path.splitext(filename)[0] + "-stack.txt"
101         port.maybe_make_directory(os.path.split(filename)[0])
102         with codecs.open(filename, "wb", "utf-8") as file:
103             file.write(test_output.error)
104     elif test_output.error:
105         _log.debug("%s %s output stderr lines:\n%s" % (worker_name, test_name,
106                                                        test_output.error))
107
108     expected_test_output = _expected_test_output(port, test_input.filename)
109
110     # Check the output and save the results.
111     start_time = time.time()
112     time_for_diffs = {}
113     for test_type in test_types:
114         start_diff_time = time.time()
115         new_failures = test_type.compare_output(port, test_input.filename,
116                                                 test_args, test_output,
117                                                 expected_test_output)
118         # Don't add any more failures if we already have a crash, so we don't
119         # double-report those tests. We do double-report for timeouts since
120         # we still want to see the text and image output.
121         if not test_output.crash:
122             failures.extend(new_failures)
123         time_for_diffs[test_type.__class__.__name__] = (
124             time.time() - start_diff_time)
125
126     total_time_for_all_diffs = time.time() - start_diff_time
127     return test_results.TestResult(test_input.filename, failures, test_output.test_time,
128                                    total_time_for_all_diffs, time_for_diffs)
129
130
131 def _pad_timeout(timeout):
132     """Returns a safe multiple of the per-test timeout value to use
133     to detect hung test threads.
134
135     """
136     # When we're running one test per DumpRenderTree process, we can
137     # enforce a hard timeout.  The DumpRenderTree watchdog uses 2.5x
138     # the timeout; we want to be larger than that.
139     return timeout * 3
140
141
142 def _milliseconds_to_seconds(msecs):
143     return float(msecs) / 1000.0
144
145
146 def _should_fetch_expected_checksum(options):
147     return options.pixel_tests and not (options.new_baseline or options.reset_results)
148
149
150 def _run_single_test(port, options, test_input, test_types, test_args, driver, worker_name):
151     # FIXME: Pull this into TestShellThread._run().
152
153     # The image hash is used to avoid doing an image dump if the
154     # checksums match, so it should be set to a blank value if we
155     # are generating a new baseline.  (Otherwise, an image from a
156     # previous run will be copied into the baseline."""
157     if _should_fetch_expected_checksum(options):
158         test_input.image_hash = port.expected_checksum(test_input.filename)
159     test_output = driver.run_test(test_input)
160     return _process_output(port, options, test_input, test_types, test_args,
161                            test_output, worker_name)
162
163
164 class SingleTestThread(threading.Thread):
165     """Thread wrapper for running a single test file."""
166
167     def __init__(self, port, options, worker_number, worker_name,
168                  test_input, test_types, test_args):
169         """
170         Args:
171           port: object implementing port-specific hooks
172           options: command line argument object from optparse
173           worker_number: worker number for tests
174           worker_name: for logging
175           test_input: Object containing the test filename and timeout
176           test_types: A list of TestType objects to run the test output
177               against.
178           test_args: A TestArguments object to pass to each TestType.
179         """
180
181         threading.Thread.__init__(self)
182         self._port = port
183         self._options = options
184         self._test_input = test_input
185         self._test_types = test_types
186         self._test_args = test_args
187         self._driver = None
188         self._worker_number = worker_number
189         self._name = worker_name
190
191     def run(self):
192         self._covered_run()
193
194     def _covered_run(self):
195         # FIXME: this is a separate routine to work around a bug
196         # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
197         self._driver = self._port.create_driver(self._worker_number)
198         self._driver.start()
199         self._test_result = _run_single_test(self._port, self._options,
200                                              self._test_input, self._test_types,
201                                              self._test_args, self._driver,
202                                              self._name)
203         self._driver.stop()
204
205     def get_test_result(self):
206         return self._test_result
207
208
209 class WatchableThread(threading.Thread):
210     """This class abstracts an interface used by
211     run_webkit_tests.TestRunner._wait_for_threads_to_finish for thread
212     management."""
213     def __init__(self):
214         threading.Thread.__init__(self)
215         self._canceled = False
216         self._exception_info = None
217         self._next_timeout = None
218         self._thread_id = None
219
220     def cancel(self):
221         """Set a flag telling this thread to quit."""
222         self._canceled = True
223
224     def clear_next_timeout(self):
225         """Mark a flag telling this thread to stop setting timeouts."""
226         self._timeout = 0
227
228     def exception_info(self):
229         """If run() terminated on an uncaught exception, return it here
230         ((type, value, traceback) tuple).
231         Returns None if run() terminated normally. Meant to be called after
232         joining this thread."""
233         return self._exception_info
234
235     def id(self):
236         """Return a thread identifier."""
237         return self._thread_id
238
239     def next_timeout(self):
240         """Return the time the test is supposed to finish by."""
241         return self._next_timeout
242
243
244 class TestShellThread(WatchableThread):
245     def __init__(self, port, options, worker_number, worker_name,
246                  filename_list_queue, result_queue):
247         """Initialize all the local state for this DumpRenderTree thread.
248
249         Args:
250           port: interface to port-specific hooks
251           options: command line options argument from optparse
252           worker_number: identifier for a particular worker thread.
253           worker_name: for logging.
254           filename_list_queue: A thread safe Queue class that contains lists
255               of tuples of (filename, uri) pairs.
256           result_queue: A thread safe Queue class that will contain
257               serialized TestResult objects.
258         """
259         WatchableThread.__init__(self)
260         self._port = port
261         self._options = options
262         self._worker_number = worker_number
263         self._name = worker_name
264         self._filename_list_queue = filename_list_queue
265         self._result_queue = result_queue
266         self._filename_list = []
267         self._driver = None
268         self._test_group_timing_stats = {}
269         self._test_results = []
270         self._num_tests = 0
271         self._start_time = 0
272         self._stop_time = 0
273         self._have_http_lock = False
274         self._http_lock_wait_begin = 0
275         self._http_lock_wait_end = 0
276
277         self._test_types = []
278         for cls in self._get_test_type_classes():
279             self._test_types.append(cls(self._port,
280                                         self._options.results_directory))
281         self._test_args = self._get_test_args(worker_number)
282
283         # Current group of tests we're running.
284         self._current_group = None
285         # Number of tests in self._current_group.
286         self._num_tests_in_current_group = None
287         # Time at which we started running tests from self._current_group.
288         self._current_group_start_time = None
289
290     def _get_test_args(self, worker_number):
291         """Returns the tuple of arguments for tests and for DumpRenderTree."""
292         test_args = test_type_base.TestArguments()
293         test_args.new_baseline = self._options.new_baseline
294         test_args.reset_results = self._options.reset_results
295
296         return test_args
297
298     def _get_test_type_classes(self):
299         classes = [text_diff.TestTextDiff]
300         if self._options.pixel_tests:
301             classes.append(image_diff.ImageDiff)
302         return classes
303
304     def get_test_group_timing_stats(self):
305         """Returns a dictionary mapping test group to a tuple of
306         (number of tests in that group, time to run the tests)"""
307         return self._test_group_timing_stats
308
309     def get_test_results(self):
310         """Return the list of all tests run on this thread.
311
312         This is used to calculate per-thread statistics.
313
314         """
315         return self._test_results
316
317     def get_total_time(self):
318         return max(self._stop_time - self._start_time -
319                    self._http_lock_wait_time(), 0.0)
320
321     def get_num_tests(self):
322         return self._num_tests
323
324     def run(self):
325         """Delegate main work to a helper method and watch for uncaught
326         exceptions."""
327         self._covered_run()
328
329     def _covered_run(self):
330         # FIXME: this is a separate routine to work around a bug
331         # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
332         self._thread_id = thread.get_ident()
333         self._start_time = time.time()
334         self._num_tests = 0
335         try:
336             _log.debug('%s starting' % (self.getName()))
337             self._run(test_runner=None, result_summary=None)
338             _log.debug('%s done (%d tests)' % (self.getName(),
339                        self.get_num_tests()))
340         except KeyboardInterrupt:
341             self._exception_info = sys.exc_info()
342             _log.debug("%s interrupted" % self.getName())
343         except:
344             # Save the exception for our caller to see.
345             self._exception_info = sys.exc_info()
346             self._stop_time = time.time()
347             _log.error('%s dying, exception raised' % self.getName())
348
349         self._stop_time = time.time()
350
351     def run_in_main_thread(self, test_runner, result_summary):
352         """This hook allows us to run the tests from the main thread if
353         --num-test-shells==1, instead of having to always run two or more
354         threads. This allows us to debug the test harness without having to
355         do multi-threaded debugging."""
356         self._run(test_runner, result_summary)
357
358     def cancel(self):
359         """Clean up http lock and set a flag telling this thread to quit."""
360         self._stop_servers_with_lock()
361         WatchableThread.cancel(self)
362
363     def next_timeout(self):
364         """Return the time the test is supposed to finish by."""
365         if self._next_timeout:
366             return self._next_timeout + self._http_lock_wait_time()
367         return self._next_timeout
368
369     def _http_lock_wait_time(self):
370         """Return the time what http locking takes."""
371         if self._http_lock_wait_begin == 0:
372             return 0
373         if self._http_lock_wait_end == 0:
374             return time.time() - self._http_lock_wait_begin
375         return self._http_lock_wait_end - self._http_lock_wait_begin
376
377     def _run(self, test_runner, result_summary):
378         """Main work entry point of the thread. Basically we pull urls from the
379         filename queue and run the tests until we run out of urls.
380
381         If test_runner is not None, then we call test_runner.UpdateSummary()
382         with the results of each test."""
383         batch_size = self._options.batch_size
384         batch_count = 0
385
386         # Append tests we're running to the existing tests_run.txt file.
387         # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput.
388         tests_run_filename = os.path.join(self._options.results_directory,
389                                           "tests_run.txt")
390         tests_run_file = codecs.open(tests_run_filename, "a", "utf-8")
391
392         while True:
393             if self._canceled:
394                 _log.debug('Testing cancelled')
395                 tests_run_file.close()
396                 return
397
398             if len(self._filename_list) is 0:
399                 if self._current_group is not None:
400                     self._test_group_timing_stats[self._current_group] = \
401                         (self._num_tests_in_current_group,
402                          time.time() - self._current_group_start_time)
403
404                 try:
405                     self._current_group, self._filename_list = \
406                         self._filename_list_queue.get_nowait()
407                 except Queue.Empty:
408                     self._stop_servers_with_lock()
409                     self._kill_dump_render_tree()
410                     tests_run_file.close()
411                     return
412
413                 if self._current_group == "tests_to_http_lock":
414                     self._start_servers_with_lock()
415                 elif self._have_http_lock:
416                     self._stop_servers_with_lock()
417
418                 self._num_tests_in_current_group = len(self._filename_list)
419                 self._current_group_start_time = time.time()
420
421             test_input = self._filename_list.pop()
422
423             # We have a url, run tests.
424             batch_count += 1
425             self._num_tests += 1
426             if self._options.run_singly:
427                 result = self._run_test_in_another_thread(test_input)
428             else:
429                 result = self._run_test_in_this_thread(test_input)
430
431             filename = test_input.filename
432             tests_run_file.write(filename + "\n")
433             if result.failures:
434                 # Check and kill DumpRenderTree if we need to.
435                 if len([1 for f in result.failures
436                         if f.should_kill_dump_render_tree()]):
437                     self._kill_dump_render_tree()
438                     # Reset the batch count since the shell just bounced.
439                     batch_count = 0
440                 # Print the error message(s).
441                 error_str = '\n'.join(['  ' + f.message() for
442                                        f in result.failures])
443                 _log.debug("%s %s failed:\n%s" % (self.getName(),
444                            self._port.relative_test_filename(filename),
445                            error_str))
446             else:
447                 _log.debug("%s %s passed" % (self.getName(),
448                            self._port.relative_test_filename(filename)))
449             self._result_queue.put(result.dumps())
450
451             if batch_size > 0 and batch_count >= batch_size:
452                 # Bounce the shell and reset count.
453                 self._kill_dump_render_tree()
454                 batch_count = 0
455
456             if test_runner:
457                 test_runner.update_summary(result_summary)
458
459     def _run_test_in_another_thread(self, test_input):
460         """Run a test in a separate thread, enforcing a hard time limit.
461
462         Since we can only detect the termination of a thread, not any internal
463         state or progress, we can only run per-test timeouts when running test
464         files singly.
465
466         Args:
467           test_input: Object containing the test filename and timeout
468
469         Returns:
470           A TestResult
471         """
472         worker = SingleTestThread(self._port,
473                                   self._options,
474                                   self._worker_number,
475                                   self._name,
476                                   test_input,
477                                   self._test_types,
478                                   self._test_args)
479
480         worker.start()
481
482         thread_timeout = _milliseconds_to_seconds(
483             _pad_timeout(int(test_input.timeout)))
484         thread._next_timeout = time.time() + thread_timeout
485         worker.join(thread_timeout)
486         if worker.isAlive():
487             # If join() returned with the thread still running, the
488             # DumpRenderTree is completely hung and there's nothing
489             # more we can do with it.  We have to kill all the
490             # DumpRenderTrees to free it up. If we're running more than
491             # one DumpRenderTree thread, we'll end up killing the other
492             # DumpRenderTrees too, introducing spurious crashes. We accept
493             # that tradeoff in order to avoid losing the rest of this
494             # thread's results.
495             _log.error('Test thread hung: killing all DumpRenderTrees')
496             if worker._driver:
497                 worker._driver.stop()
498
499         try:
500             result = worker.get_test_result()
501         except AttributeError, e:
502             # This gets raised if the worker thread has already exited.
503             failures = []
504             _log.error('Cannot get results of test: %s' %
505                        test_input.filename)
506             result = test_results.TestResult(test_input.filename, failures=[],
507                 test_run_time=0, total_time_for_all_diffs=0, time_for_diffs={})
508
509         return result
510
511     def _run_test_in_this_thread(self, test_input):
512         """Run a single test file using a shared DumpRenderTree process.
513
514         Args:
515           test_input: Object containing the test filename, uri and timeout
516
517         Returns: a TestResult object.
518         """
519         self._ensure_dump_render_tree_is_running()
520         thread_timeout = _milliseconds_to_seconds(
521              _pad_timeout(int(test_input.timeout)))
522         self._next_timeout = time.time() + thread_timeout
523         test_result = _run_single_test(self._port, self._options, test_input,
524                                        self._test_types, self._test_args,
525                                        self._driver, self._name)
526         self._test_results.append(test_result)
527         return test_result
528
529     def _ensure_dump_render_tree_is_running(self):
530         """Start the shared DumpRenderTree, if it's not running.
531
532         This is not for use when running tests singly, since those each start
533         a separate DumpRenderTree in their own thread.
534
535         """
536         # poll() is not threadsafe and can throw OSError due to:
537         # http://bugs.python.org/issue1731717
538         if not self._driver or self._driver.poll() is not None:
539             self._driver = self._port.create_driver(self._worker_number)
540             self._driver.start()
541
542     def _start_servers_with_lock(self):
543         """Acquire http lock and start the servers."""
544         self._http_lock_wait_begin = time.time()
545         _log.debug('Acquire http lock ...')
546         self._port.acquire_http_lock()
547         _log.debug('Starting HTTP server ...')
548         self._port.start_http_server()
549         _log.debug('Starting WebSocket server ...')
550         self._port.start_websocket_server()
551         self._http_lock_wait_end = time.time()
552         self._have_http_lock = True
553
554     def _stop_servers_with_lock(self):
555         """Stop the servers and release http lock."""
556         if self._have_http_lock:
557             _log.debug('Stopping HTTP server ...')
558             self._port.stop_http_server()
559             _log.debug('Stopping WebSocket server ...')
560             self._port.stop_websocket_server()
561             _log.debug('Release http lock ...')
562             self._port.release_http_lock()
563             self._have_http_lock = False
564
565     def _kill_dump_render_tree(self):
566         """Kill the DumpRenderTree process if it's running."""
567         if self._driver:
568             self._driver.stop()
569             self._driver = None