Unreviewed, rolling out r115340.
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / controllers / test_result_writer.py
1 # Copyright (C) 2011 Google Inc. All rights reserved.
2 #
3 # Redistribution and use in source and binary forms, with or without
4 # modification, are permitted provided that the following conditions are
5 # met:
6 #
7 #     * Redistributions of source code must retain the above copyright
8 # notice, this list of conditions and the following disclaimer.
9 #     * Redistributions in binary form must reproduce the above
10 # copyright notice, this list of conditions and the following disclaimer
11 # in the documentation and/or other materials provided with the
12 # distribution.
13 #     * Neither the name of Google Inc. nor the names of its
14 # contributors may be used to endorse or promote products derived from
15 # this software without specific prior written permission.
16 #
17 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29
30 import logging
31
32 from webkitpy.layout_tests.models import test_failures
33
34
35 _log = logging.getLogger(__name__)
36
37
38 def write_test_result(filesystem, port, test_name, driver_output,
39                       expected_driver_output, failures):
40     """Write the test result to the result output directory."""
41     root_output_dir = port.results_directory()
42     writer = TestResultWriter(filesystem, port, root_output_dir, test_name)
43     if driver_output.error:
44         writer.write_stderr(driver_output.error)
45
46     for failure in failures:
47         # FIXME: Instead of this long 'if' block, each failure class might
48         # have a responsibility for writing a test result.
49         if isinstance(failure, (test_failures.FailureMissingResult,
50                                 test_failures.FailureTextMismatch)):
51             writer.write_text_files(driver_output.text, expected_driver_output.text)
52             writer.create_text_diff_and_write_result(driver_output.text, expected_driver_output.text)
53         elif isinstance(failure, test_failures.FailureMissingImage):
54             writer.write_image_files(driver_output.image, expected_image=None)
55         elif isinstance(failure, test_failures.FailureMissingImageHash):
56             writer.write_image_files(driver_output.image, expected_driver_output.image)
57         elif isinstance(failure, test_failures.FailureImageHashMismatch):
58             writer.write_image_files(driver_output.image, expected_driver_output.image)
59             writer.write_image_diff_files(driver_output.image_diff)
60         elif isinstance(failure, (test_failures.FailureAudioMismatch,
61                                   test_failures.FailureMissingAudio)):
62             writer.write_audio_files(driver_output.audio, expected_driver_output.audio)
63         elif isinstance(failure, test_failures.FailureCrash):
64             crashed_driver_output = expected_driver_output if failure.is_reftest else driver_output
65             writer.write_crash_log(crashed_driver_output.crash_log)
66         elif isinstance(failure, test_failures.FailureReftestMismatch):
67             writer.write_image_files(driver_output.image, expected_driver_output.image)
68             # FIXME: This work should be done earlier in the pipeline (e.g., when we compare images for non-ref tests).
69             # FIXME: We should always have 2 images here.
70             if driver_output.image and expected_driver_output.image:
71                 image_diff = port.diff_image(driver_output.image, expected_driver_output.image, tolerance=0)[0]
72                 if image_diff:
73                     writer.write_image_diff_files(image_diff)
74                 else:
75                     _log.warn('Can not get image diff. ImageDiff program might not work correctly.')
76             writer.copy_file(failure.reference_filename)
77         elif isinstance(failure, test_failures.FailureReftestMismatchDidNotOccur):
78             writer.write_image_files(driver_output.image, expected_image=None)
79             writer.copy_file(failure.reference_filename)
80         else:
81             assert isinstance(failure, (test_failures.FailureTimeout, test_failures.FailureReftestNoImagesGenerated))
82
83
84 class TestResultWriter(object):
85     """A class which handles all writing operations to the result directory."""
86
87     # Filename pieces when writing failures to the test results directory.
88     FILENAME_SUFFIX_ACTUAL = "-actual"
89     FILENAME_SUFFIX_EXPECTED = "-expected"
90     FILENAME_SUFFIX_DIFF = "-diff"
91     FILENAME_SUFFIX_STDERR = "-stderr"
92     FILENAME_SUFFIX_CRASH_LOG = "-crash-log"
93     FILENAME_SUFFIX_WDIFF = "-wdiff.html"
94     FILENAME_SUFFIX_PRETTY_PATCH = "-pretty-diff.html"
95     FILENAME_SUFFIX_IMAGE_DIFF = "-diff.png"
96     FILENAME_SUFFIX_IMAGE_DIFFS_HTML = "-diffs.html"
97
98     def __init__(self, filesystem, port, root_output_dir, test_name):
99         self._filesystem = filesystem
100         self._port = port
101         self._root_output_dir = root_output_dir
102         self._test_name = test_name
103
104     def _make_output_directory(self):
105         """Creates the output directory (if needed) for a given test filename."""
106         fs = self._filesystem
107         output_filename = fs.join(self._root_output_dir, self._test_name)
108         fs.maybe_make_directory(fs.dirname(output_filename))
109
110     def output_filename(self, modifier):
111         """Returns a filename inside the output dir that contains modifier.
112
113         For example, if test name is "fast/dom/foo.html" and modifier is "-expected.txt",
114         the return value is "/<path-to-root-output-dir>/fast/dom/foo-expected.txt".
115
116         Args:
117           modifier: a string to replace the extension of filename with
118
119         Return:
120           The absolute path to the output filename
121         """
122         fs = self._filesystem
123         output_filename = fs.join(self._root_output_dir, self._test_name)
124         return fs.splitext(output_filename)[0] + modifier
125
126     def _output_testname(self, modifier):
127         fs = self._filesystem
128         return fs.splitext(fs.basename(self._test_name))[0] + modifier
129
130     def write_output_files(self, file_type, output, expected):
131         """Writes the test output, the expected output in the results directory.
132
133         The full output filename of the actual, for example, will be
134           <filename>-actual<file_type>
135         For instance,
136           my_test-actual.txt
137
138         Args:
139           file_type: A string describing the test output file type, e.g. ".txt"
140           output: A string containing the test output
141           expected: A string containing the expected test output
142         """
143         self._make_output_directory()
144         actual_filename = self.output_filename(self.FILENAME_SUFFIX_ACTUAL + file_type)
145         expected_filename = self.output_filename(self.FILENAME_SUFFIX_EXPECTED + file_type)
146
147         fs = self._filesystem
148         if output is not None:
149             fs.write_binary_file(actual_filename, output)
150         if expected is not None:
151             fs.write_binary_file(expected_filename, expected)
152
153     def write_stderr(self, error):
154         fs = self._filesystem
155         filename = self.output_filename(self.FILENAME_SUFFIX_STDERR + ".txt")
156         fs.maybe_make_directory(fs.dirname(filename))
157         fs.write_binary_file(filename, error)
158
159     def write_crash_log(self, crash_log):
160         fs = self._filesystem
161         filename = self.output_filename(self.FILENAME_SUFFIX_CRASH_LOG + ".txt")
162         fs.maybe_make_directory(fs.dirname(filename))
163         if crash_log is not None:
164             fs.write_text_file(filename, crash_log)
165
166     def write_text_files(self, actual_text, expected_text):
167         self.write_output_files(".txt", actual_text, expected_text)
168
169     def create_text_diff_and_write_result(self, actual_text, expected_text):
170         # FIXME: This function is actually doing the diffs as well as writing results.
171         # It might be better to extract code which does 'diff' and make it a separate function.
172         if not actual_text or not expected_text:
173             return
174
175         self._make_output_directory()
176         file_type = '.txt'
177         actual_filename = self.output_filename(self.FILENAME_SUFFIX_ACTUAL + file_type)
178         expected_filename = self.output_filename(self.FILENAME_SUFFIX_EXPECTED + file_type)
179         fs = self._filesystem
180         # We treat diff output as binary. Diff output may contain multiple files
181         # in conflicting encodings.
182         diff = self._port.diff_text(expected_text, actual_text, expected_filename, actual_filename)
183         diff_filename = self.output_filename(self.FILENAME_SUFFIX_DIFF + file_type)
184         fs.write_binary_file(diff_filename, diff)
185
186         # Shell out to wdiff to get colored inline diffs.
187         wdiff = self._port.wdiff_text(expected_filename, actual_filename)
188         wdiff_filename = self.output_filename(self.FILENAME_SUFFIX_WDIFF)
189         fs.write_binary_file(wdiff_filename, wdiff)
190
191         # Use WebKit's PrettyPatch.rb to get an HTML diff.
192         pretty_patch = self._port.pretty_patch_text(diff_filename)
193         pretty_patch_filename = self.output_filename(self.FILENAME_SUFFIX_PRETTY_PATCH)
194         fs.write_binary_file(pretty_patch_filename, pretty_patch)
195
196     def write_audio_files(self, actual_audio, expected_audio):
197         self.write_output_files('.wav', actual_audio, expected_audio)
198
199     def write_image_files(self, actual_image, expected_image):
200         self.write_output_files('.png', actual_image, expected_image)
201
202     def write_image_diff_files(self, image_diff):
203         diff_filename = self.output_filename(self.FILENAME_SUFFIX_IMAGE_DIFF)
204         fs = self._filesystem
205         fs.write_binary_file(diff_filename, image_diff)
206
207         diffs_html_filename = self.output_filename(self.FILENAME_SUFFIX_IMAGE_DIFFS_HTML)
208         # FIXME: old-run-webkit-tests shows the diff percentage as the text contents of the "diff" link.
209         # FIXME: old-run-webkit-tests include a link to the test file.
210         html = """<!DOCTYPE HTML>
211 <html>
212 <head>
213 <title>%(title)s</title>
214 <style>.label{font-weight:bold}</style>
215 </head>
216 <body>
217 Difference between images: <a href="%(diff_filename)s">diff</a><br>
218 <div class=imageText></div>
219 <div class=imageContainer data-prefix="%(prefix)s">Loading...</div>
220 <script>
221 (function() {
222     var preloadedImageCount = 0;
223     function preloadComplete() {
224         ++preloadedImageCount;
225         if (preloadedImageCount < 2)
226             return;
227         toggleImages();
228         setInterval(toggleImages, 2000)
229     }
230
231     function preloadImage(url) {
232         image = new Image();
233         image.addEventListener('load', preloadComplete);
234         image.src = url;
235         return image;
236     }
237
238     function toggleImages() {
239         if (text.textContent == 'Expected Image') {
240             text.textContent = 'Actual Image';
241             container.replaceChild(actualImage, container.firstChild);
242         } else {
243             text.textContent = 'Expected Image';
244             container.replaceChild(expectedImage, container.firstChild);
245         }
246     }
247
248     var text = document.querySelector('.imageText');
249     var container = document.querySelector('.imageContainer');
250     var actualImage = preloadImage(container.getAttribute('data-prefix') + '-actual.png');
251     var expectedImage = preloadImage(container.getAttribute('data-prefix') + '-expected.png');
252 })();
253 </script>
254 </body>
255 </html>
256 """ % {
257             'title': self._test_name,
258             'diff_filename': self._output_testname(self.FILENAME_SUFFIX_IMAGE_DIFF),
259             'prefix': self._output_testname(''),
260         }
261         self._filesystem.write_text_file(diffs_html_filename, html)
262
263     def copy_file(self, src_filepath):
264         fs = self._filesystem
265         assert fs.exists(src_filepath), 'src_filepath: %s' % src_filepath
266         dst_filepath = fs.join(self._root_output_dir, self._port.relative_test_filename(src_filepath))
267         self._make_output_directory()
268         fs.copyfile(src_filepath, dst_filepath)