WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the name of Google Inc. nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """A Thread object for running DumpRenderTree and processing URLs from a
  32 shared queue.
  33
  34 Each thread runs a separate instance of the DumpRenderTree binary and validates
  35 the output.  When there are no more URLs to process in the shared queue, the
  36 thread exits.
  37 """
  38
  39 from __future__ import with_statement
  40
  41 import codecs
  42 import copy
  43 import logging
  44 import os
  45 import Queue
  46 import signal
  47 import sys
  48 import thread
  49 import threading
  50 import time
  51
  52
  53 from webkitpy.layout_tests.test_types import image_diff
  54 from webkitpy.layout_tests.test_types import test_type_base
  55 from webkitpy.layout_tests.test_types import text_diff
  56
  57 import test_failures
  58 import test_output
  59 import test_results
  60
  61 _log = logging.getLogger("webkitpy.layout_tests.layout_package."
  62                          "dump_render_tree_thread")
  63
  64
  65 class WatchableThread(threading.Thread):
  66     """This class abstracts an interface used by
  67     run_webkit_tests.TestRunner._wait_for_threads_to_finish for thread
  68     management."""
  69     def __init__(self):
  70         threading.Thread.__init__(self)
  71         self._canceled = False
  72         self._exception_info = None
  73         self._next_timeout = None
  74         self._thread_id = None
  75
  76     def cancel(self):
  77         """Set a flag telling this thread to quit."""
  78         self._canceled = True
  79
  80     def clear_next_timeout(self):
  81         """Mark a flag telling this thread to stop setting timeouts."""
  82         self._timeout = 0
  83
  84     def exception_info(self):
  85         """If run() terminated on an uncaught exception, return it here
  86         ((type, value, traceback) tuple).
  87         Returns None if run() terminated normally. Meant to be called after
  88         joining this thread."""
  89         return self._exception_info
  90
  91     def id(self):
  92         """Return a thread identifier."""
  93         return self._thread_id
  94
  95     def next_timeout(self):
  96         """Return the time the test is supposed to finish by."""
  97         return self._next_timeout
  98
  99
 100 class TestShellThread(WatchableThread):
 101     def __init__(self, port, options, worker_number,
 102                  filename_list_queue, result_queue):
 103         """Initialize all the local state for this DumpRenderTree thread.
 104
 105         Args:
 106           port: interface to port-specific hooks
 107           options: command line options argument from optparse
 108           worker_number: identifier for a particular worker thread.
 109           filename_list_queue: A thread safe Queue class that contains lists
 110               of tuples of (filename, uri) pairs.
 111           result_queue: A thread safe Queue class that will contain
 112               serialized TestResult objects.
 113         """
 114         WatchableThread.__init__(self)
 115         self._port = port
 116         self._options = options
 117         self._worker_number = worker_number
 118         self._name = 'worker/%d' % worker_number
 119         self._filename_list_queue = filename_list_queue
 120         self._result_queue = result_queue
 121
 122         self._batch_count = 0
 123         self._batch_size = self._options.batch_size
 124         self._driver = None
 125         self._have_http_lock = False
 126
 127         self._test_runner = None
 128         self._result_summary = None
 129         self._test_list_timing_stats = {}
 130         self._test_results = []
 131         self._num_tests = 0
 132         self._start_time = 0
 133         self._stop_time = 0
 134         self._http_lock_wait_begin = 0
 135         self._http_lock_wait_end = 0
 136
 137         self._test_types = []
 138         for cls in self._get_test_type_classes():
 139             self._test_types.append(cls(self._port,
 140                                         self._options.results_directory))
 141         self._test_args = self._get_test_args(worker_number)
 142
 143         # Append tests we're running to the existing tests_run.txt file.
 144         # This is created in run_webkit_tests.py:_PrepareListsAndPrintOutput.
 145         tests_run_filename = os.path.join(self._options.results_directory,
 146                                           "tests_run.txt")
 147         self._tests_run_file = codecs.open(tests_run_filename, "a", "utf-8")
 148
 149     def __del__(self):
 150         self._cleanup()
 151
 152     def _get_test_args(self, worker_number):
 153         """Returns the tuple of arguments for tests and for DumpRenderTree."""
 154         test_args = test_type_base.TestArguments()
 155         test_args.new_baseline = self._options.new_baseline
 156         test_args.reset_results = self._options.reset_results
 157
 158         return test_args
 159
 160     def _get_test_type_classes(self):
 161         classes = [text_diff.TestTextDiff]
 162         if self._options.pixel_tests:
 163             classes.append(image_diff.ImageDiff)
 164         return classes
 165
 166     def get_test_group_timing_stats(self):
 167         """Returns a dictionary mapping test group to a tuple of
 168         (number of tests in that group, time to run the tests)"""
 169         return self._test_list_timing_stats
 170
 171     def get_test_results(self):
 172         """Return the list of all tests run on this thread.
 173
 174         This is used to calculate per-thread statistics.
 175
 176         """
 177         return self._test_results
 178
 179     def get_total_time(self):
 180         return max(self._stop_time - self._start_time -
 181                    self._http_lock_wait_time(), 0.0)
 182
 183     def get_num_tests(self):
 184         return self._num_tests
 185
 186     def name(self):
 187         return self._name
 188
 189     def next_timeout(self):
 190         """Return the time the test is supposed to finish by."""
 191         if self._next_timeout:
 192             return self._next_timeout + self._http_lock_wait_time()
 193         return self._next_timeout
 194
 195     def run(self):
 196         """Delegate main work to a helper method and watch for uncaught
 197         exceptions."""
 198         self._covered_run()
 199
 200     def _covered_run(self):
 201         # FIXME: this is a separate routine to work around a bug
 202         # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
 203         self._thread_id = thread.get_ident()
 204         self._start_time = time.time()
 205         try:
 206             _log.debug('%s starting' % (self._name))
 207             self._run(test_runner=None, result_summary=None)
 208             _log.debug('%s done (%d tests)' % (self._name, self._num_tests))
 209         except KeyboardInterrupt:
 210             self._exception_info = sys.exc_info()
 211             _log.debug("%s interrupted" % self._name)
 212         except:
 213             # Save the exception for our caller to see.
 214             self._exception_info = sys.exc_info()
 215             self._stop_time = time.time()
 216             _log.error('%s dying, exception raised' % self._name)
 217
 218         self._stop_time = time.time()
 219
 220     def run_in_main_thread(self, test_runner, result_summary):
 221         """This hook allows us to run the tests from the main thread if
 222         --num-test-shells==1, instead of having to always run two or more
 223         threads. This allows us to debug the test harness without having to
 224         do multi-threaded debugging."""
 225         self._run(test_runner, result_summary)
 226
 227     def _run(self, test_runner, result_summary):
 228         """Main work entry point of the thread. Basically we pull urls from the
 229         filename queue and run the tests until we run out of urls.
 230
 231         If test_runner is not None, then we call test_runner.UpdateSummary()
 232         with the results of each test during _tear_down_test(), below."""
 233         self._test_runner = test_runner
 234         self._result_summary = result_summary
 235
 236         while not self._canceled:
 237             try:
 238                 current_group, filename_list = \
 239                     self._filename_list_queue.get_nowait()
 240                 self.handle_test_list(current_group, filename_list)
 241             except Queue.Empty:
 242                 break
 243
 244         if self._canceled:
 245             _log.debug('Testing canceled')
 246
 247         self._cleanup()
 248
 249     def _cleanup(self):
 250         self._kill_dump_render_tree()
 251         if self._have_http_lock:
 252             self._stop_servers_with_lock()
 253         if self._tests_run_file:
 254             self._tests_run_file.close()
 255             self._tests_run_file = None
 256
 257     def handle_test_list(self, list_name, test_list):
 258         if list_name == "tests_to_http_lock":
 259             self._start_servers_with_lock()
 260
 261         start_time = time.time()
 262         num_tests = 0
 263         for test_input in test_list:
 264             self._run_test(test_input)
 265             if self._canceled:
 266                 break
 267             num_tests += 1
 268
 269         elapsed_time = time.time() - start_time
 270
 271         if self._have_http_lock:
 272             self._stop_servers_with_lock()
 273
 274         self._test_list_timing_stats[list_name] = \
 275            (num_tests, elapsed_time)
 276
 277     def _run_test(self, test_input):
 278         self._set_up_test(test_input)
 279
 280         # We calculate how long we expect the test to take.
 281         #
 282         # The DumpRenderTree watchdog uses 2.5x the timeout; we want to be
 283         # larger than that. We also add a little more padding if we're
 284         # running tests in a separate thread.
 285         #
 286         # Note that we need to convert the test timeout from a
 287         # string value in milliseconds to a float for Python.
 288         driver_timeout_sec = 3.0 * float(test_input.timeout) / 1000.0
 289         thread_padding_sec = 1.0
 290         thread_timeout_sec = driver_timeout_sec + thread_padding_sec
 291         if self._options.run_singly:
 292             test_timeout_sec = thread_timeout_sec
 293         else:
 294             test_timeout_sec = driver_timeout_sec
 295
 296         start = time.time()
 297         self._next_timeout = start + test_timeout_sec
 298
 299         if self._options.run_singly:
 300             result = self._run_test_in_another_thread(test_input,
 301                                                       thread_timeout_sec)
 302         else:
 303             result = self._run_test_in_this_thread(test_input)
 304
 305         self._tear_down_test(test_input, result)
 306
 307     def _set_up_test(self, test_input):
 308         test_input.uri = self._port.filename_to_uri(test_input.filename)
 309         if self._should_fetch_expected_checksum():
 310             test_input.image_checksum = self._port.expected_checksum(
 311                 test_input.filename)
 312
 313     def _should_fetch_expected_checksum(self):
 314         return (self._options.pixel_tests and not
 315                 (self._options.new_baseline or self._options.reset_results))
 316
 317     def _run_test_in_another_thread(self, test_input, thread_timeout_sec):
 318         """Run a test in a separate thread, enforcing a hard time limit.
 319
 320         Since we can only detect the termination of a thread, not any internal
 321         state or progress, we can only run per-test timeouts when running test
 322         files singly.
 323
 324         Args:
 325           test_input: Object containing the test filename and timeout
 326           thread_timeout_sec: time to wait before killing the driver process.
 327         Returns:
 328           A TestResult
 329         """
 330         worker = self
 331         result = None
 332
 333         driver = worker._port.create_driver(worker._worker_number)
 334         driver.start()
 335
 336         class SingleTestThread(threading.Thread):
 337             def run(self):
 338                 result = worker._run_single_test(test_input, driver)
 339
 340         thread = SingleTestThread()
 341         thread.start()
 342         thread.join(thread_timeout_sec)
 343         if thread.isAlive():
 344             # If join() returned with the thread still running, the
 345             # DumpRenderTree is completely hung and there's nothing
 346             # more we can do with it.  We have to kill all the
 347             # DumpRenderTrees to free it up. If we're running more than
 348             # one DumpRenderTree thread, we'll end up killing the other
 349             # DumpRenderTrees too, introducing spurious crashes. We accept
 350             # that tradeoff in order to avoid losing the rest of this
 351             # thread's results.
 352             _log.error('Test thread hung: killing all DumpRenderTrees')
 353
 354         driver.stop()
 355
 356         if not result:
 357             result = test_results.TestResult(test_input.filename, failures=[],
 358                 test_run_time=0, total_time_for_all_diffs=0, time_for_diffs={})
 359
 360         return result
 361
 362     def _run_test_in_this_thread(self, test_input):
 363         """Run a single test file using a shared DumpRenderTree process.
 364
 365         Args:
 366           test_input: Object containing the test filename, uri and timeout
 367
 368         Returns: a TestResult object.
 369         """
 370         # poll() is not threadsafe and can throw OSError due to:
 371         # http://bugs.python.org/issue1731717
 372         if not self._driver or self._driver.poll() is not None:
 373             self._driver = self._port.create_driver(self._worker_number)
 374             self._driver.start()
 375
 376         test_result = self._run_single_test(test_input, self._driver)
 377         self._test_results.append(test_result)
 378         return test_result
 379
 380     def _run_single_test(self, test_input, driver):
 381         # The image hash is used to avoid doing an image dump if the
 382         # checksums match, so it should be set to a blank value if we
 383         # are generating a new baseline.  (Otherwise, an image from a
 384         # previous run will be copied into the baseline."""
 385         if self._should_fetch_expected_checksum():
 386             test_input.image_hash = self._port.expected_checksum(
 387                 test_input.filename)
 388         test_output = driver.run_test(test_input)
 389         return self._process_output(test_input.filename, test_output)
 390
 391     def _process_output(self, test_filename, test_output):
 392         """Receives the output from a DumpRenderTree process, subjects it to a
 393         number of tests, and returns a list of failure types the test produced.
 394
 395         Args:
 396         test_filename: full path to the test in question.
 397         test_output: a TestOutput object containing the output of the test
 398
 399         Returns: a TestResult object
 400         """
 401         failures = []
 402
 403         if test_output.crash:
 404             failures.append(test_failures.FailureCrash())
 405         if test_output.timeout:
 406             failures.append(test_failures.FailureTimeout())
 407
 408         test_name = self._port.relative_test_filename(test_filename)
 409         if test_output.crash:
 410             _log.debug("%s Stacktrace for %s:\n%s" %
 411                        (self._name, test_name, test_output.error))
 412             filename = os.path.join(self._options.results_directory, test_name)
 413             filename = os.path.splitext(filename)[0] + "-stack.txt"
 414             self._port.maybe_make_directory(os.path.split(filename)[0])
 415             with codecs.open(filename, "wb", "utf-8") as file:
 416                 file.write(test_output.error)
 417         elif test_output.error:
 418             _log.debug("%s %s output stderr lines:\n%s" %
 419                        (self._name, test_name, test_output.error))
 420
 421         expected_test_output = self._expected_test_output(test_filename)
 422
 423         # Check the output and save the results.
 424         start_time = time.time()
 425         time_for_diffs = {}
 426         for test_type in self._test_types:
 427             start_diff_time = time.time()
 428             new_failures = test_type.compare_output(self._port,
 429                                                     test_filename,
 430                                                     self._test_args,
 431                                                     test_output,
 432                                                     expected_test_output)
 433             # Don't add any more failures if we already have a crash, so we
 434             # don't double-report those tests. We do double-report for timeouts
 435             # since we still want to see the text and image output.
 436             if not test_output.crash:
 437                 failures.extend(new_failures)
 438             time_for_diffs[test_type.__class__.__name__] = (
 439                 time.time() - start_diff_time)
 440
 441         total_time_for_all_diffs = time.time() - start_diff_time
 442         return test_results.TestResult(test_filename,
 443                                        failures,
 444                                        test_output.test_time,
 445                                        total_time_for_all_diffs,
 446                                        time_for_diffs)
 447
 448     def _expected_test_output(self, filename):
 449         """Returns an expected TestOutput object."""
 450         return test_output.TestOutput(self._port.expected_text(filename),
 451                                     self._port.expected_image(filename),
 452                                     self._port.expected_checksum(filename))
 453
 454     def _tear_down_test(self, test_input, result):
 455         self._num_tests += 1
 456         self._batch_count += 1
 457         self._tests_run_file.write(test_input.filename + "\n")
 458         test_name = self._port.relative_test_filename(test_input.filename)
 459
 460         if result.failures:
 461             # Check and kill DumpRenderTree if we need to.
 462             if any([f.should_kill_dump_render_tree() for f in result.failures]):
 463                 self._kill_dump_render_tree()
 464                 # Reset the batch count since the shell just bounced.
 465                 self._batch_count = 0
 466
 467             # Print the error message(s).
 468             _log.debug("%s %s failed:" % (self._name, test_name))
 469             for f in result.failures:
 470                 _log.debug("%s  %s" % (self._name, f.message()))
 471         else:
 472             _log.debug("%s %s passed" % (self._name, test_name))
 473
 474         self._result_queue.put(result.dumps())
 475
 476         if self._batch_size > 0 and self._batch_count >= self._batch_size:
 477             # Bounce the shell and reset count.
 478             self._kill_dump_render_tree()
 479             self._batch_count = 0
 480
 481         if self._test_runner:
 482             self._test_runner.update_summary(self._result_summary)
 483
 484     def _start_servers_with_lock(self):
 485         self._http_lock_wait_begin = time.time()
 486         _log.debug('Acquiring http lock ...')
 487         self._port.acquire_http_lock()
 488         _log.debug('Starting HTTP server ...')
 489         self._port.start_http_server()
 490         _log.debug('Starting WebSocket server ...')
 491         self._port.start_websocket_server()
 492         self._http_lock_wait_end = time.time()
 493         self._have_http_lock = True
 494
 495     def _http_lock_wait_time(self):
 496         """Return the time what http locking takes."""
 497         if self._http_lock_wait_begin == 0:
 498             return 0
 499         if self._http_lock_wait_end == 0:
 500             return time.time() - self._http_lock_wait_begin
 501         return self._http_lock_wait_end - self._http_lock_wait_begin
 502
 503     def _stop_servers_with_lock(self):
 504         """Stop the servers and release http lock."""
 505         if self._have_http_lock:
 506             _log.debug('Stopping HTTP server ...')
 507             self._port.stop_http_server()
 508             _log.debug('Stopping WebSocket server ...')
 509             self._port.stop_websocket_server()
 510             _log.debug('Release http lock ...')
 511             self._port.release_http_lock()
 512             self._have_http_lock = False
 513
 514     def _kill_dump_render_tree(self):
 515         """Kill the DumpRenderTree process if it's running."""
 516         if self._driver:
 517             self._driver.stop()
 518             self._driver = None