ff4086c74dc182ba847123e635500d14411d993e
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / layout_tests / port / test.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the Google name nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Dummy Port implementation used for testing."""
31 from __future__ import with_statement
32
33 import codecs
34 import fnmatch
35 import os
36 import sys
37 import time
38
39 import base
40
41
42 # This sets basic expectations for a test. Each individual expectation
43 # can be overridden by a keyword argument in TestList.add().
44 class TestInstance:
45     def __init__(self, name):
46         self.name = name
47         self.base = name[(name.rfind("/") + 1):name.rfind(".html")]
48         self.crash = False
49         self.exception = False
50         self.hang = False
51         self.keyboard = False
52         self.error = ''
53         self.timeout = False
54         self.actual_text = self.base + '-txt\n'
55         self.actual_checksum = self.base + '-checksum\n'
56         self.actual_image = self.base + '-png\n'
57         self.expected_text = self.actual_text
58         self.expected_checksum = self.actual_checksum
59         self.expected_image = self.actual_image
60
61
62 # This is an in-memory list of tests, what we want them to produce, and
63 # what we want to claim are the expected results.
64 class TestList:
65     def __init__(self, port):
66         self.port = port
67         self.tests = {}
68
69     def add(self, name, **kwargs):
70         test = TestInstance(name)
71         for key, value in kwargs.items():
72             test.__dict__[key] = value
73         self.tests[name] = test
74
75     def keys(self):
76         return self.tests.keys()
77
78     def __contains__(self, item):
79         return item in self.tests
80
81     def __getitem__(self, item):
82         return self.tests[item]
83
84
85 class TestPort(base.Port):
86     """Test implementation of the Port interface."""
87
88     def __init__(self, **kwargs):
89         base.Port.__init__(self, **kwargs)
90         tests = TestList(self)
91         tests.add('passes/image.html')
92         tests.add('passes/text.html')
93         tests.add('failures/expected/checksum.html',
94                   actual_checksum='checksum_fail-checksum')
95         tests.add('failures/expected/crash.html', crash=True)
96         tests.add('failures/expected/exception.html', exception=True)
97         tests.add('failures/expected/timeout.html', timeout=True)
98         tests.add('failures/expected/hang.html', hang=True)
99         tests.add('failures/expected/missing_text.html',
100                   expected_text=None)
101         tests.add('failures/expected/image.html',
102                   actual_image='image_fail-png',
103                   expected_image='image-png')
104         tests.add('failures/expected/image_checksum.html',
105                   actual_checksum='image_checksum_fail-checksum',
106                   actual_image='image_checksum_fail-png')
107         tests.add('failures/expected/keyboard.html',
108                   keyboard=True)
109         tests.add('failures/expected/missing_check.html',
110                   expected_checksum=None)
111         tests.add('failures/expected/missing_image.html',
112                   expected_image=None)
113         tests.add('failures/expected/missing_text.html',
114                   expected_text=None)
115         tests.add('failures/expected/text.html',
116                   actual_text='text_fail-png')
117         tests.add('failures/unexpected/text-image-checksum.html',
118                   actual_text='text-image-checksum_fail-txt',
119                   actual_checksum='text-image-checksum_fail-checksum')
120         tests.add('http/tests/passes/text.html')
121         tests.add('http/tests/ssl/text.html')
122         tests.add('passes/error.html', error='stuff going to stderr')
123         tests.add('passes/image.html')
124         tests.add('passes/platform_image.html')
125         tests.add('passes/text.html')
126         tests.add('websocket/tests/passes/text.html')
127         self._tests = tests
128
129     def baseline_path(self):
130         return os.path.join(self.layout_tests_dir(), 'platform',
131                             self.name() + self.version())
132
133     def baseline_search_path(self):
134         return [self.baseline_path()]
135
136     def check_build(self, needs_http):
137         return True
138
139     def diff_image(self, expected_contents, actual_contents,
140                    diff_filename=None):
141         diffed = actual_contents != expected_contents
142         if diffed and diff_filename:
143             with codecs.open(diff_filename, "w", "utf-8") as diff_fh:
144                 diff_fh.write("< %s\n---\n> %s\n" %
145                               (expected_contents, actual_contents))
146         return diffed
147
148     def expected_checksum(self, test):
149         test = self.relative_test_filename(test)
150         return self._tests[test].expected_checksum
151
152     def expected_image(self, test):
153         test = self.relative_test_filename(test)
154         return self._tests[test].expected_image
155
156     def expected_text(self, test):
157         test = self.relative_test_filename(test)
158         text = self._tests[test].expected_text
159         if not text:
160             text = ''
161         return text
162
163     def tests(self, paths):
164         # Test the idea of port-specific overrides for test lists. Also
165         # keep in memory to speed up the test harness.
166         if not paths:
167             paths = ['*']
168
169         matched_tests = []
170         for p in paths:
171             if self.path_isdir(p):
172                 matched_tests.extend(fnmatch.filter(self._tests.keys(), p + '*'))
173             else:
174                 matched_tests.extend(fnmatch.filter(self._tests.keys(), p))
175         layout_tests_dir = self.layout_tests_dir()
176         return set([os.path.join(layout_tests_dir, p) for p in matched_tests])
177
178     def path_exists(self, path):
179         # used by test_expectations.py and printing.py
180         rpath = self.relative_test_filename(path)
181         if rpath in self._tests:
182             return True
183         if self.path_isdir(rpath):
184             return True
185         if rpath.endswith('-expected.txt'):
186             test = rpath.replace('-expected.txt', '.html')
187             return (test in self._tests and
188                     self._tests[test].expected_text)
189         if rpath.endswith('-expected.checksum'):
190             test = rpath.replace('-expected.checksum', '.html')
191             return (test in self._tests and
192                     self._tests[test].expected_checksum)
193         if rpath.endswith('-expected.png'):
194             test = rpath.replace('-expected.png', '.html')
195             return (test in self._tests and
196                     self._tests[test].expected_image)
197         return False
198
199     def layout_tests_dir(self):
200         return self.path_from_webkit_base('WebKitTools', 'Scripts',
201                                           'webkitpy', 'layout_tests', 'data')
202
203     def path_isdir(self, path):
204         # Used by test_expectations.py
205         #
206         # We assume that a path is a directory if we have any tests that
207         # whose prefix matches the path plus a directory modifier.
208         if path[-1] != '/':
209             path += '/'
210         return any([t.startswith(path) for t in self._tests.keys()])
211
212     def test_dirs(self):
213         return ['passes', 'failures']
214
215     def name(self):
216         return self._name
217
218     def _path_to_wdiff(self):
219         return None
220
221     def results_directory(self):
222         return '/tmp/' + self.get_option('results_directory')
223
224     def setup_test_run(self):
225         pass
226
227     def create_driver(self, image_path, options):
228         return TestDriver(self, image_path, options, executive=None)
229
230     def start_http_server(self):
231         pass
232
233     def start_websocket_server(self):
234         pass
235
236     def stop_http_server(self):
237         pass
238
239     def stop_websocket_server(self):
240         pass
241
242     def test_expectations(self):
243         """Returns the test expectations for this port.
244
245         Basically this string should contain the equivalent of a
246         test_expectations file. See test_expectations.py for more details."""
247         return """
248 WONTFIX : failures/expected/checksum.html = IMAGE
249 WONTFIX : failures/expected/crash.html = CRASH
250 // This one actually passes because the checksums will match.
251 WONTFIX : failures/expected/image.html = PASS
252 WONTFIX : failures/expected/image_checksum.html = IMAGE
253 WONTFIX : failures/expected/missing_check.html = MISSING PASS
254 WONTFIX : failures/expected/missing_image.html = MISSING PASS
255 WONTFIX : failures/expected/missing_text.html = MISSING PASS
256 WONTFIX : failures/expected/text.html = TEXT
257 WONTFIX : failures/expected/timeout.html = TIMEOUT
258 WONTFIX SKIP : failures/expected/hang.html = TIMEOUT
259 WONTFIX SKIP : failures/expected/keyboard.html = CRASH
260 WONTFIX SKIP : failures/expected/exception.html = CRASH
261 """
262
263     def test_base_platform_names(self):
264         return ('mac', 'win')
265
266     def test_platform_name(self):
267         return 'mac'
268
269     def test_platform_names(self):
270         return self.test_base_platform_names()
271
272     def test_platform_name_to_name(self, test_platform_name):
273         return test_platform_name
274
275     def version(self):
276         return ''
277
278
279 class TestDriver(base.Driver):
280     """Test/Dummy implementation of the DumpRenderTree interface."""
281
282     def __init__(self, port, image_path, options, executive):
283         self._port = port
284         self._image_path = image_path
285         self._executive = executive
286         self._image_written = False
287
288     def poll(self):
289         return True
290
291     def run_test(self, uri, timeoutms, image_hash):
292         test_name = self._port.uri_to_test_name(uri)
293         test = self._port._tests[test_name]
294         if test.keyboard:
295             raise KeyboardInterrupt
296         if test.exception:
297             raise ValueError('exception from ' + test_name)
298         if test.hang:
299             time.sleep((float(timeoutms) * 4) / 1000.0)
300
301         if self._port.get_option('pixel_tests') and test.actual_image:
302             with open(self._image_path, 'w') as file:
303                 file.write(test.actual_image)
304
305         return (test.crash, test.timeout, test.actual_checksum,
306                 test.actual_text, test.error)
307
308     def start(self):
309         pass
310
311     def stop(self):
312         pass