050eefae1b18ec2b1628da75f261488d0f2ac3a5
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / layout_package / dump_render_tree_thread.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the name of Google Inc. nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """A Thread object for running DumpRenderTree and processing URLs from a
32 shared queue.
33
34 Each thread runs a separate instance of the DumpRenderTree binary and validates
35 the output.  When there are no more URLs to process in the shared queue, the
36 thread exits.
37 """
38
39 import copy
40 import logging
41 import os
42 import Queue
43 import signal
44 import sys
45 import thread
46 import threading
47 import time
48
49
50 from webkitpy.layout_tests.test_types import image_diff
51 from webkitpy.layout_tests.test_types import test_type_base
52 from webkitpy.layout_tests.test_types import text_diff
53
54 import test_failures
55 import test_output
56 import test_results
57
58 _log = logging.getLogger("webkitpy.layout_tests.layout_package."
59                          "dump_render_tree_thread")
60
61
62 def _expected_test_output(port, filename):
63     """Returns an expected TestOutput object."""
64     return test_output.TestOutput(port.expected_text(filename),
65                                   port.expected_image(filename),
66                                   port.expected_checksum(filename))
67
68 def _process_output(port, options, test_input, test_types, test_args,
69                     test_output, worker_name):
70     """Receives the output from a DumpRenderTree process, subjects it to a
71     number of tests, and returns a list of failure types the test produced.
72
73     Args:
74       port: port-specific hooks
75       options: command line options argument from optparse
76       proc: an active DumpRenderTree process
77       test_input: Object containing the test filename and timeout
78       test_types: list of test types to subject the output to
79       test_args: arguments to be passed to each test
80       test_output: a TestOutput object containing the output of the test
81       worker_name: worker name for logging
82
83     Returns: a TestResult object
84     """
85     failures = []
86     fs = port._filesystem
87
88     if test_output.crash:
89         failures.append(test_failures.FailureCrash())
90     if test_output.timeout:
91         failures.append(test_failures.FailureTimeout())
92
93     test_name = port.relative_test_filename(test_input.filename)
94     if test_output.crash:
95         _log.debug("%s Stacktrace for %s:\n%s" % (worker_name, test_name,
96                                                   test_output.error))
97         filename = fs.join(options.results_directory, test_name)
98         filename = fs.splitext(filename)[0] + "-stack.txt"
99         fs.maybe_make_directory(fs.dirname(filename))
100         fs.write_text_file(filename, test_output.error)
101     elif test_output.error:
102         _log.debug("%s %s output stderr lines:\n%s" % (worker_name, test_name,
103                                                        test_output.error))
104
105     expected_test_output = _expected_test_output(port, test_input.filename)
106
107     # Check the output and save the results.
108     start_time = time.time()
109     time_for_diffs = {}
110     for test_type in test_types:
111         start_diff_time = time.time()
112         new_failures = test_type.compare_output(port, test_input.filename,
113                                                 test_args, test_output,
114                                                 expected_test_output)
115         # Don't add any more failures if we already have a crash, so we don't
116         # double-report those tests. We do double-report for timeouts since
117         # we still want to see the text and image output.
118         if not test_output.crash:
119             failures.extend(new_failures)
120         time_for_diffs[test_type.__class__.__name__] = (
121             time.time() - start_diff_time)
122
123     total_time_for_all_diffs = time.time() - start_diff_time
124     return test_results.TestResult(test_input.filename, failures, test_output.test_time,
125                                    total_time_for_all_diffs, time_for_diffs)
126
127
128 def _pad_timeout(timeout):
129     """Returns a safe multiple of the per-test timeout value to use
130     to detect hung test threads.
131
132     """
133     # When we're running one test per DumpRenderTree process, we can
134     # enforce a hard timeout.  The DumpRenderTree watchdog uses 2.5x
135     # the timeout; we want to be larger than that.
136     return timeout * 3
137
138
139 def _milliseconds_to_seconds(msecs):
140     return float(msecs) / 1000.0
141
142
143 def _should_fetch_expected_checksum(options):
144     return options.pixel_tests and not (options.new_baseline or options.reset_results)
145
146
147 def _run_single_test(port, options, test_input, test_types, test_args, driver, worker_name):
148     # FIXME: Pull this into TestShellThread._run().
149
150     # The image hash is used to avoid doing an image dump if the
151     # checksums match, so it should be set to a blank value if we
152     # are generating a new baseline.  (Otherwise, an image from a
153     # previous run will be copied into the baseline."""
154     if _should_fetch_expected_checksum(options):
155         test_input.image_hash = port.expected_checksum(test_input.filename)
156     test_output = driver.run_test(test_input)
157     return _process_output(port, options, test_input, test_types, test_args,
158                            test_output, worker_name)
159
160
161 class SingleTestThread(threading.Thread):
162     """Thread wrapper for running a single test file."""
163
164     def __init__(self, port, options, worker_number, worker_name,
165                  test_input, test_types, test_args):
166         """
167         Args:
168           port: object implementing port-specific hooks
169           options: command line argument object from optparse
170           worker_number: worker number for tests
171           worker_name: for logging
172           test_input: Object containing the test filename and timeout
173           test_types: A list of TestType objects to run the test output
174               against.
175           test_args: A TestArguments object to pass to each TestType.
176         """
177
178         threading.Thread.__init__(self)
179         self._port = port
180         self._options = options
181         self._test_input = test_input
182         self._test_types = test_types
183         self._test_args = test_args
184         self._driver = None
185         self._worker_number = worker_number
186         self._name = worker_name
187
188     def run(self):
189         self._covered_run()
190
191     def _covered_run(self):
192         # FIXME: this is a separate routine to work around a bug
193         # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
194         self._driver = self._port.create_driver(self._worker_number)
195         self._driver.start()
196         self._test_result = _run_single_test(self._port, self._options,
197                                              self._test_input, self._test_types,
198                                              self._test_args, self._driver,
199                                              self._name)
200         self._driver.stop()
201
202     def get_test_result(self):
203         return self._test_result
204
205
206 class WatchableThread(threading.Thread):
207     """This class abstracts an interface used by
208     run_webkit_tests.TestRunner._wait_for_threads_to_finish for thread
209     management."""
210     def __init__(self):
211         threading.Thread.__init__(self)
212         self._canceled = False
213         self._exception_info = None
214         self._next_timeout = None
215         self._thread_id = None
216
217     def cancel(self):
218         """Set a flag telling this thread to quit."""
219         self._canceled = True
220
221     def clear_next_timeout(self):
222         """Mark a flag telling this thread to stop setting timeouts."""
223         self._timeout = 0
224
225     def exception_info(self):
226         """If run() terminated on an uncaught exception, return it here
227         ((type, value, traceback) tuple).
228         Returns None if run() terminated normally. Meant to be called after
229         joining this thread."""
230         return self._exception_info
231
232     def id(self):
233         """Return a thread identifier."""
234         return self._thread_id
235
236     def next_timeout(self):
237         """Return the time the test is supposed to finish by."""
238         return self._next_timeout
239
240
241 class TestShellThread(WatchableThread):
242     def __init__(self, port, options, worker_number, worker_name,
243                  filename_list_queue, result_queue):
244         """Initialize all the local state for this DumpRenderTree thread.
245
246         Args:
247           port: interface to port-specific hooks
248           options: command line options argument from optparse
249           worker_number: identifier for a particular worker thread.
250           worker_name: for logging.
251           filename_list_queue: A thread safe Queue class that contains lists
252               of tuples of (filename, uri) pairs.
253           result_queue: A thread safe Queue class that will contain
254               serialized TestResult objects.
255         """
256         WatchableThread.__init__(self)
257         self._port = port
258         self._options = options
259         self._worker_number = worker_number
260         self._name = worker_name
261         self._filename_list_queue = filename_list_queue
262         self._result_queue = result_queue
263         self._filename_list = []
264         self._driver = None
265         self._test_group_timing_stats = {}
266         self._test_results = []
267         self._num_tests = 0
268         self._start_time = 0
269         self._stop_time = 0
270         self._have_http_lock = False
271         self._http_lock_wait_begin = 0
272         self._http_lock_wait_end = 0
273
274         self._test_types = []
275         for cls in self._get_test_type_classes():
276             self._test_types.append(cls(self._port,
277                                         self._options.results_directory))
278         self._test_args = self._get_test_args(worker_number)
279
280         # Current group of tests we're running.
281         self._current_group = None
282         # Number of tests in self._current_group.
283         self._num_tests_in_current_group = None
284         # Time at which we started running tests from self._current_group.
285         self._current_group_start_time = None
286
287     def _get_test_args(self, worker_number):
288         """Returns the tuple of arguments for tests and for DumpRenderTree."""
289         test_args = test_type_base.TestArguments()
290         test_args.new_baseline = self._options.new_baseline
291         test_args.reset_results = self._options.reset_results
292
293         return test_args
294
295     def _get_test_type_classes(self):
296         classes = [text_diff.TestTextDiff]
297         if self._options.pixel_tests:
298             classes.append(image_diff.ImageDiff)
299         return classes
300
301     def get_test_group_timing_stats(self):
302         """Returns a dictionary mapping test group to a tuple of
303         (number of tests in that group, time to run the tests)"""
304         return self._test_group_timing_stats
305
306     def get_test_results(self):
307         """Return the list of all tests run on this thread.
308
309         This is used to calculate per-thread statistics.
310
311         """
312         return self._test_results
313
314     def get_total_time(self):
315         return max(self._stop_time - self._start_time -
316                    self._http_lock_wait_time(), 0.0)
317
318     def get_num_tests(self):
319         return self._num_tests
320
321     def run(self):
322         """Delegate main work to a helper method and watch for uncaught
323         exceptions."""
324         self._covered_run()
325
326     def _covered_run(self):
327         # FIXME: this is a separate routine to work around a bug
328         # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
329         self._thread_id = thread.get_ident()
330         self._start_time = time.time()
331         self._num_tests = 0
332         try:
333             _log.debug('%s starting' % (self.getName()))
334             self._run(test_runner=None, result_summary=None)
335             _log.debug('%s done (%d tests)' % (self.getName(),
336                        self.get_num_tests()))
337         except KeyboardInterrupt:
338             self._exception_info = sys.exc_info()
339             _log.debug("%s interrupted" % self.getName())
340         except:
341             # Save the exception for our caller to see.
342             self._exception_info = sys.exc_info()
343             self._stop_time = time.time()
344             _log.error('%s dying, exception raised' % self.getName())
345
346         self._stop_time = time.time()
347
348     def run_in_main_thread(self, test_runner, result_summary):
349         """This hook allows us to run the tests from the main thread if
350         --num-test-shells==1, instead of having to always run two or more
351         threads. This allows us to debug the test harness without having to
352         do multi-threaded debugging."""
353         self._run(test_runner, result_summary)
354
355     def cancel(self):
356         """Clean up http lock and set a flag telling this thread to quit."""
357         self._stop_servers_with_lock()
358         WatchableThread.cancel(self)
359
360     def next_timeout(self):
361         """Return the time the test is supposed to finish by."""
362         if self._next_timeout:
363             return self._next_timeout + self._http_lock_wait_time()
364         return self._next_timeout
365
366     def _http_lock_wait_time(self):
367         """Return the time what http locking takes."""
368         if self._http_lock_wait_begin == 0:
369             return 0
370         if self._http_lock_wait_end == 0:
371             return time.time() - self._http_lock_wait_begin
372         return self._http_lock_wait_end - self._http_lock_wait_begin
373
374     def _run(self, test_runner, result_summary):
375         """Main work entry point of the thread. Basically we pull urls from the
376         filename queue and run the tests until we run out of urls.
377
378         If test_runner is not None, then we call test_runner.UpdateSummary()
379         with the results of each test."""
380         batch_size = self._options.batch_size
381         batch_count = 0
382
383         # Append tests we're running to the existing tests_run.txt file.
384         # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput.
385         tests_run_filename = self._port._filesystem.join(self._options.results_directory,
386                                           "tests_run.txt")
387         tests_run_file = self._port._filesystem.open_text_file_for_writing(tests_run_filename, append=False)
388         while True:
389             if self._canceled:
390                 _log.debug('Testing cancelled')
391                 tests_run_file.close()
392                 return
393
394             if len(self._filename_list) is 0:
395                 if self._current_group is not None:
396                     self._test_group_timing_stats[self._current_group] = \
397                         (self._num_tests_in_current_group,
398                          time.time() - self._current_group_start_time)
399
400                 try:
401                     self._current_group, self._filename_list = \
402                         self._filename_list_queue.get_nowait()
403                 except Queue.Empty:
404                     self._stop_servers_with_lock()
405                     self._kill_dump_render_tree()
406                     tests_run_file.close()
407                     return
408
409                 if self._current_group == "tests_to_http_lock":
410                     self._start_servers_with_lock()
411                 elif self._have_http_lock:
412                     self._stop_servers_with_lock()
413
414                 self._num_tests_in_current_group = len(self._filename_list)
415                 self._current_group_start_time = time.time()
416
417             test_input = self._filename_list.pop()
418
419             # We have a url, run tests.
420             batch_count += 1
421             self._num_tests += 1
422             if self._options.run_singly:
423                 result = self._run_test_in_another_thread(test_input)
424             else:
425                 result = self._run_test_in_this_thread(test_input)
426
427             filename = test_input.filename
428             tests_run_file.write(filename + "\n")
429             if result.failures:
430                 # Check and kill DumpRenderTree if we need to.
431                 if len([1 for f in result.failures
432                         if f.should_kill_dump_render_tree()]):
433                     self._kill_dump_render_tree()
434                     # Reset the batch count since the shell just bounced.
435                     batch_count = 0
436                 # Print the error message(s).
437                 error_str = '\n'.join(['  ' + f.message() for
438                                        f in result.failures])
439                 _log.debug("%s %s failed:\n%s" % (self.getName(),
440                            self._port.relative_test_filename(filename),
441                            error_str))
442             else:
443                 _log.debug("%s %s passed" % (self.getName(),
444                            self._port.relative_test_filename(filename)))
445             self._result_queue.put(result.dumps())
446
447             if batch_size > 0 and batch_count >= batch_size:
448                 # Bounce the shell and reset count.
449                 self._kill_dump_render_tree()
450                 batch_count = 0
451
452             if test_runner:
453                 test_runner.update_summary(result_summary)
454
455     def _run_test_in_another_thread(self, test_input):
456         """Run a test in a separate thread, enforcing a hard time limit.
457
458         Since we can only detect the termination of a thread, not any internal
459         state or progress, we can only run per-test timeouts when running test
460         files singly.
461
462         Args:
463           test_input: Object containing the test filename and timeout
464
465         Returns:
466           A TestResult
467         """
468         worker = SingleTestThread(self._port,
469                                   self._options,
470                                   self._worker_number,
471                                   self._name,
472                                   test_input,
473                                   self._test_types,
474                                   self._test_args)
475
476         worker.start()
477
478         thread_timeout = _milliseconds_to_seconds(
479             _pad_timeout(int(test_input.timeout)))
480         thread._next_timeout = time.time() + thread_timeout
481         worker.join(thread_timeout)
482         if worker.isAlive():
483             # If join() returned with the thread still running, the
484             # DumpRenderTree is completely hung and there's nothing
485             # more we can do with it.  We have to kill all the
486             # DumpRenderTrees to free it up. If we're running more than
487             # one DumpRenderTree thread, we'll end up killing the other
488             # DumpRenderTrees too, introducing spurious crashes. We accept
489             # that tradeoff in order to avoid losing the rest of this
490             # thread's results.
491             _log.error('Test thread hung: killing all DumpRenderTrees')
492             if worker._driver:
493                 worker._driver.stop()
494
495         try:
496             result = worker.get_test_result()
497         except AttributeError, e:
498             # This gets raised if the worker thread has already exited.
499             _log.error('Cannot get results of test: %s' % test_input.filename)
500             # FIXME: Seems we want a unique failure type here.
501             result = test_results.TestResult(test_input.filename)
502
503         return result
504
505     def _run_test_in_this_thread(self, test_input):
506         """Run a single test file using a shared DumpRenderTree process.
507
508         Args:
509           test_input: Object containing the test filename, uri and timeout
510
511         Returns: a TestResult object.
512         """
513         self._ensure_dump_render_tree_is_running()
514         thread_timeout = _milliseconds_to_seconds(
515              _pad_timeout(int(test_input.timeout)))
516         self._next_timeout = time.time() + thread_timeout
517         test_result = _run_single_test(self._port, self._options, test_input,
518                                        self._test_types, self._test_args,
519                                        self._driver, self._name)
520         self._test_results.append(test_result)
521         return test_result
522
523     def _ensure_dump_render_tree_is_running(self):
524         """Start the shared DumpRenderTree, if it's not running.
525
526         This is not for use when running tests singly, since those each start
527         a separate DumpRenderTree in their own thread.
528
529         """
530         # poll() is not threadsafe and can throw OSError due to:
531         # http://bugs.python.org/issue1731717
532         if not self._driver or self._driver.poll() is not None:
533             self._driver = self._port.create_driver(self._worker_number)
534             self._driver.start()
535
536     def _start_servers_with_lock(self):
537         """Acquire http lock and start the servers."""
538         self._http_lock_wait_begin = time.time()
539         _log.debug('Acquire http lock ...')
540         self._port.acquire_http_lock()
541         _log.debug('Starting HTTP server ...')
542         self._port.start_http_server()
543         _log.debug('Starting WebSocket server ...')
544         self._port.start_websocket_server()
545         self._http_lock_wait_end = time.time()
546         self._have_http_lock = True
547
548     def _stop_servers_with_lock(self):
549         """Stop the servers and release http lock."""
550         if self._have_http_lock:
551             _log.debug('Stopping HTTP server ...')
552             self._port.stop_http_server()
553             _log.debug('Stopping WebSocket server ...')
554             self._port.stop_websocket_server()
555             _log.debug('Release http lock ...')
556             self._port.release_http_lock()
557             self._have_http_lock = False
558
559     def _kill_dump_render_tree(self):
560         """Kill the DumpRenderTree process if it's running."""
561         if self._driver:
562             self._driver.stop()
563             self._driver = None