8592f1958bc12ddffebc7f788aa0a0dd22ec121c
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / layout_package / single_test_runner.py
1 # Copyright (C) 2011 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29
30 import logging
31 import os
32 import time
33
34 from webkitpy.layout_tests.port import base
35 from webkitpy.layout_tests.layout_package import test_failures
36 from webkitpy.layout_tests.layout_package.test_results import TestResult
37
38
39 _log = logging.getLogger(__name__)
40
41
42 def run_single_test(port, options, test_input, driver, worker_name, test_types, test_args):
43     # FIXME: Pull this into TestShellThread._run().
44     runner = SingleTestRunner(options, port, driver, test_input, worker_name, test_types, test_args)
45     return runner.run()
46
47
48 class ExpectedDriverOutput:
49     """Groups information about an expected driver output."""
50     def __init__(self, text, image, image_hash):
51         self.text = text
52         self.image = image
53         self.image_hash = image_hash
54
55
56 class SingleTestRunner:
57
58     def __init__(self, options, port, driver, test_input, worker_name, test_types, test_args):
59         self._options = options
60         self._port = port
61         self._driver = driver
62         self._filename = test_input.filename
63         self._timeout = test_input.timeout
64         self._worker_name = worker_name
65         self._test_types = test_types
66         self._test_args = test_args
67         self._testname = port.relative_test_filename(test_input.filename)
68
69     def _expected_driver_output(self):
70         return ExpectedDriverOutput(self._port.expected_text(self._filename),
71                                     self._port.expected_image(self._filename),
72                                     self._port.expected_checksum(self._filename))
73
74     def _should_fetch_expected_checksum(self):
75         return (self._options.pixel_tests and
76                 not (self._options.new_baseline or self._options.reset_results))
77
78     def _driver_input(self):
79         # The image hash is used to avoid doing an image dump if the
80         # checksums match, so it should be set to a blank value if we
81         # are generating a new baseline.  (Otherwise, an image from a
82         # previous run will be copied into the baseline."""
83         image_hash = None
84         if self._should_fetch_expected_checksum():
85             image_hash = self._port.expected_checksum(self._filename)
86         return base.DriverInput(self._filename, self._timeout, image_hash)
87
88     def run(self):
89         driver_output = self._driver.run_test(self._driver_input())
90         return self._process_output(driver_output)
91
92     def _process_output(self, driver_output):
93         """Receives the output from a DumpRenderTree process, subjects it to a
94         number of tests, and returns a list of failure types the test produced.
95         Args:
96           driver_output: a DriverOutput object containing the output from the driver
97
98         Returns: a TestResult object
99         """
100         failures = []
101         fs = self._port._filesystem
102
103         if driver_output.crash:
104             failures.append(test_failures.FailureCrash())
105         if driver_output.timeout:
106             failures.append(test_failures.FailureTimeout())
107
108         if driver_output.crash:
109             _log.debug("%s Stacktrace for %s:\n%s" % (self._worker_name, self._testname,
110                                                       driver_output.error))
111             stack_filename = fs.join(self._options.results_directory, self._testname)
112             stack_filename = fs.splitext(stack_filename)[0] + "-stack.txt"
113             fs.maybe_make_directory(fs.dirname(stack_filename))
114             fs.write_text_file(stack_filename, driver_output.error)
115         elif driver_output.error:
116             _log.debug("%s %s output stderr lines:\n%s" % (self._worker_name, self._testname,
117                                                            driver_output.error))
118
119         expected_driver_output = self._expected_driver_output()
120
121         # Check the output and save the results.
122         start_time = time.time()
123         time_for_diffs = {}
124         for test_type in self._test_types:
125             start_diff_time = time.time()
126             new_failures = test_type.compare_output(self._port, self._filename,
127                                                     self._test_args, driver_output,
128                                                     expected_driver_output)
129             # Don't add any more failures if we already have a crash, so we don't
130             # double-report those tests. We do double-report for timeouts since
131             # we still want to see the text and image output.
132             if not driver_output.crash:
133                 failures.extend(new_failures)
134             time_for_diffs[test_type.__class__.__name__] = (
135                 time.time() - start_diff_time)
136
137         total_time_for_all_diffs = time.time() - start_diff_time
138         return TestResult(self._filename, failures, driver_output.test_time,
139                           total_time_for_all_diffs, time_for_diffs)