2011-02-03 Hayato Ito <hayato@chromium.org>
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / layout_package / single_test_runner.py
1 # Copyright (C) 2011 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29
30 import logging
31 import os
32 import time
33
34 from webkitpy.layout_tests.port import base
35 from webkitpy.layout_tests.layout_package import test_failures
36 from webkitpy.layout_tests.layout_package.test_results import TestResult
37
38
39 _log = logging.getLogger(__name__)
40
41
42 def run_single_test(port, options, test_input, driver, worker_name, test_types):
43     # FIXME: Pull this into TestShellThread._run().
44     runner = SingleTestRunner(options, port, driver, test_input, worker_name, test_types)
45     return runner.run()
46
47
48 class ExpectedDriverOutput:
49     """Groups information about an expected driver output."""
50     def __init__(self, text, image, image_hash):
51         self.text = text
52         self.image = image
53         self.image_hash = image_hash
54
55
56 class SingleTestRunner:
57
58     def __init__(self, options, port, driver, test_input, worker_name, test_types):
59         self._options = options
60         self._port = port
61         self._driver = driver
62         self._filename = test_input.filename
63         self._timeout = test_input.timeout
64         self._worker_name = worker_name
65         self._test_types = test_types
66         self._testname = port.relative_test_filename(test_input.filename)
67
68     def _expected_driver_output(self):
69         return ExpectedDriverOutput(self._port.expected_text(self._filename),
70                                     self._port.expected_image(self._filename),
71                                     self._port.expected_checksum(self._filename))
72
73     def _should_fetch_expected_checksum(self):
74         return (self._options.pixel_tests and
75                 not (self._options.new_baseline or self._options.reset_results))
76
77     def _driver_input(self):
78         # The image hash is used to avoid doing an image dump if the
79         # checksums match, so it should be set to a blank value if we
80         # are generating a new baseline.  (Otherwise, an image from a
81         # previous run will be copied into the baseline."""
82         image_hash = None
83         if self._should_fetch_expected_checksum():
84             image_hash = self._port.expected_checksum(self._filename)
85         return base.DriverInput(self._filename, self._timeout, image_hash)
86
87     def run(self):
88         driver_output = self._driver.run_test(self._driver_input())
89         return self._process_output(driver_output)
90
91     def _process_output(self, driver_output):
92         """Receives the output from a DumpRenderTree process, subjects it to a
93         number of tests, and returns a list of failure types the test produced.
94         Args:
95           driver_output: a DriverOutput object containing the output from the driver
96
97         Returns: a TestResult object
98         """
99         failures = []
100         fs = self._port._filesystem
101
102         if driver_output.crash:
103             failures.append(test_failures.FailureCrash())
104         if driver_output.timeout:
105             failures.append(test_failures.FailureTimeout())
106
107         if driver_output.crash:
108             _log.debug("%s Stacktrace for %s:\n%s" % (self._worker_name, self._testname,
109                                                       driver_output.error))
110             stack_filename = fs.join(self._options.results_directory, self._testname)
111             stack_filename = fs.splitext(stack_filename)[0] + "-stack.txt"
112             fs.maybe_make_directory(fs.dirname(stack_filename))
113             fs.write_text_file(stack_filename, driver_output.error)
114         elif driver_output.error:
115             _log.debug("%s %s output stderr lines:\n%s" % (self._worker_name, self._testname,
116                                                            driver_output.error))
117
118         expected_driver_output = self._expected_driver_output()
119
120         # Check the output and save the results.
121         start_time = time.time()
122         time_for_diffs = {}
123         for test_type in self._test_types:
124             start_diff_time = time.time()
125             new_failures = test_type.compare_output(
126                 self._port, self._filename, self._options, driver_output,
127                 expected_driver_output)
128             # Don't add any more failures if we already have a crash, so we don't
129             # double-report those tests. We do double-report for timeouts since
130             # we still want to see the text and image output.
131             if not driver_output.crash:
132                 failures.extend(new_failures)
133             time_for_diffs[test_type.__class__.__name__] = (
134                 time.time() - start_diff_time)
135
136         total_time_for_all_diffs = time.time() - start_diff_time
137         return TestResult(self._filename, failures, driver_output.test_time,
138                           total_time_for_all_diffs, time_for_diffs)