Clean up ChunkedUpdateDrawingAreaProxy
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / layout_tests / port / test.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the Google name nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Dummy Port implementation used for testing."""
31 from __future__ import with_statement
32
33 import codecs
34 import fnmatch
35 import os
36 import sys
37 import time
38
39 from webkitpy.layout_tests.layout_package import test_output
40
41 import base
42
43
44 # This sets basic expectations for a test. Each individual expectation
45 # can be overridden by a keyword argument in TestList.add().
46 class TestInstance:
47     def __init__(self, name):
48         self.name = name
49         self.base = name[(name.rfind("/") + 1):name.rfind(".html")]
50         self.crash = False
51         self.exception = False
52         self.hang = False
53         self.keyboard = False
54         self.error = ''
55         self.timeout = False
56         self.actual_text = self.base + '-txt\n'
57         self.actual_checksum = self.base + '-checksum\n'
58         self.actual_image = self.base + '-png\n'
59         self.expected_text = self.actual_text
60         self.expected_checksum = self.actual_checksum
61         self.expected_image = self.actual_image
62
63
64 # This is an in-memory list of tests, what we want them to produce, and
65 # what we want to claim are the expected results.
66 class TestList:
67     def __init__(self, port):
68         self.port = port
69         self.tests = {}
70
71     def add(self, name, **kwargs):
72         test = TestInstance(name)
73         for key, value in kwargs.items():
74             test.__dict__[key] = value
75         self.tests[name] = test
76
77     def keys(self):
78         return self.tests.keys()
79
80     def __contains__(self, item):
81         return item in self.tests
82
83     def __getitem__(self, item):
84         return self.tests[item]
85
86
87 class TestPort(base.Port):
88     """Test implementation of the Port interface."""
89
90     def __init__(self, **kwargs):
91         base.Port.__init__(self, **kwargs)
92         tests = TestList(self)
93         tests.add('passes/image.html')
94         tests.add('passes/text.html')
95         tests.add('failures/expected/checksum.html',
96                   actual_checksum='checksum_fail-checksum')
97         tests.add('failures/expected/crash.html', crash=True)
98         tests.add('failures/expected/exception.html', exception=True)
99         tests.add('failures/expected/timeout.html', timeout=True)
100         tests.add('failures/expected/hang.html', hang=True)
101         tests.add('failures/expected/missing_text.html',
102                   expected_text=None)
103         tests.add('failures/expected/image.html',
104                   actual_image='image_fail-png',
105                   expected_image='image-png')
106         tests.add('failures/expected/image_checksum.html',
107                   actual_checksum='image_checksum_fail-checksum',
108                   actual_image='image_checksum_fail-png')
109         tests.add('failures/expected/keyboard.html',
110                   keyboard=True)
111         tests.add('failures/expected/missing_check.html',
112                   expected_checksum=None)
113         tests.add('failures/expected/missing_image.html',
114                   expected_image=None)
115         tests.add('failures/expected/missing_text.html',
116                   expected_text=None)
117         tests.add('failures/expected/text.html',
118                   actual_text='text_fail-png')
119         tests.add('failures/unexpected/crash.html', crash=True)
120         tests.add('failures/unexpected/text-image-checksum.html',
121                   actual_text='text-image-checksum_fail-txt',
122                   actual_checksum='text-image-checksum_fail-checksum')
123         tests.add('failures/unexpected/timeout.html', timeout=True)
124         tests.add('http/tests/passes/text.html')
125         tests.add('http/tests/ssl/text.html')
126         tests.add('passes/error.html', error='stuff going to stderr')
127         tests.add('passes/image.html')
128         tests.add('passes/platform_image.html')
129         tests.add('passes/text.html')
130         tests.add('websocket/tests/passes/text.html')
131         self._tests = tests
132
133     def baseline_path(self):
134         return os.path.join(self.layout_tests_dir(), 'platform',
135                             self.name() + self.version())
136
137     def baseline_search_path(self):
138         return [self.baseline_path()]
139
140     def check_build(self, needs_http):
141         return True
142
143     def diff_image(self, expected_contents, actual_contents,
144                    diff_filename=None):
145         diffed = actual_contents != expected_contents
146         if diffed and diff_filename:
147             with codecs.open(diff_filename, "w", "utf-8") as diff_fh:
148                 diff_fh.write("< %s\n---\n> %s\n" %
149                               (expected_contents, actual_contents))
150         return diffed
151
152     def expected_checksum(self, test):
153         test = self.relative_test_filename(test)
154         return self._tests[test].expected_checksum
155
156     def expected_image(self, test):
157         test = self.relative_test_filename(test)
158         return self._tests[test].expected_image
159
160     def expected_text(self, test):
161         test = self.relative_test_filename(test)
162         text = self._tests[test].expected_text
163         if not text:
164             text = ''
165         return text
166
167     def tests(self, paths):
168         # Test the idea of port-specific overrides for test lists. Also
169         # keep in memory to speed up the test harness.
170         if not paths:
171             paths = ['*']
172
173         matched_tests = []
174         for p in paths:
175             if self.path_isdir(p):
176                 matched_tests.extend(fnmatch.filter(self._tests.keys(), p + '*'))
177             else:
178                 matched_tests.extend(fnmatch.filter(self._tests.keys(), p))
179         layout_tests_dir = self.layout_tests_dir()
180         return set([os.path.join(layout_tests_dir, p) for p in matched_tests])
181
182     def path_exists(self, path):
183         # used by test_expectations.py and printing.py
184         rpath = self.relative_test_filename(path)
185         if rpath in self._tests:
186             return True
187         if self.path_isdir(rpath):
188             return True
189         if rpath.endswith('-expected.txt'):
190             test = rpath.replace('-expected.txt', '.html')
191             return (test in self._tests and
192                     self._tests[test].expected_text)
193         if rpath.endswith('-expected.checksum'):
194             test = rpath.replace('-expected.checksum', '.html')
195             return (test in self._tests and
196                     self._tests[test].expected_checksum)
197         if rpath.endswith('-expected.png'):
198             test = rpath.replace('-expected.png', '.html')
199             return (test in self._tests and
200                     self._tests[test].expected_image)
201         return False
202
203     def layout_tests_dir(self):
204         return self.path_from_webkit_base('WebKitTools', 'Scripts',
205                                           'webkitpy', 'layout_tests', 'data')
206
207     def path_isdir(self, path):
208         # Used by test_expectations.py
209         #
210         # We assume that a path is a directory if we have any tests
211         # that whose prefix matches the path plus a directory modifier
212         # and not a file extension.
213         if path[-1] != '/':
214             path += '/'
215
216         # FIXME: Directories can have a dot in the name. We should
217         # probably maintain a white list of known cases like CSS2.1
218         # and check it here in the future.
219         if path.find('.') != -1:
220             # extension separator found, assume this is a file
221             return False
222
223         # strip out layout tests directory path if found. The tests
224         # keys are relative to it.
225         tests_dir = self.layout_tests_dir()
226         if path.startswith(tests_dir):
227             path = path[len(tests_dir) + 1:]
228
229         return any([t.startswith(path) for t in self._tests.keys()])
230
231     def test_dirs(self):
232         return ['passes', 'failures']
233
234     def name(self):
235         return self._name
236
237     def _path_to_wdiff(self):
238         return None
239
240     def results_directory(self):
241         return '/tmp/' + self.get_option('results_directory')
242
243     def setup_test_run(self):
244         pass
245
246     def create_driver(self, worker_number):
247         return TestDriver(self, worker_number)
248
249     def start_http_server(self):
250         pass
251
252     def start_websocket_server(self):
253         pass
254
255     def stop_http_server(self):
256         pass
257
258     def stop_websocket_server(self):
259         pass
260
261     def test_expectations(self):
262         """Returns the test expectations for this port.
263
264         Basically this string should contain the equivalent of a
265         test_expectations file. See test_expectations.py for more details."""
266         return """
267 WONTFIX : failures/expected/checksum.html = IMAGE
268 WONTFIX : failures/expected/crash.html = CRASH
269 // This one actually passes because the checksums will match.
270 WONTFIX : failures/expected/image.html = PASS
271 WONTFIX : failures/expected/image_checksum.html = IMAGE
272 WONTFIX : failures/expected/missing_check.html = MISSING PASS
273 WONTFIX : failures/expected/missing_image.html = MISSING PASS
274 WONTFIX : failures/expected/missing_text.html = MISSING PASS
275 WONTFIX : failures/expected/text.html = TEXT
276 WONTFIX : failures/expected/timeout.html = TIMEOUT
277 WONTFIX SKIP : failures/expected/hang.html = TIMEOUT
278 WONTFIX SKIP : failures/expected/keyboard.html = CRASH
279 WONTFIX SKIP : failures/expected/exception.html = CRASH
280 """
281
282     def test_base_platform_names(self):
283         return ('mac', 'win')
284
285     def test_platform_name(self):
286         return 'mac'
287
288     def test_platform_names(self):
289         return self.test_base_platform_names()
290
291     def test_platform_name_to_name(self, test_platform_name):
292         return test_platform_name
293
294     def version(self):
295         return ''
296
297
298 class TestDriver(base.Driver):
299     """Test/Dummy implementation of the DumpRenderTree interface."""
300
301     def __init__(self, port, worker_number):
302         self._port = port
303
304     def cmd_line(self):
305         return ['None']
306
307     def poll(self):
308         return True
309
310     def run_test(self, test_input):
311         start_time = time.time()
312         test_name = self._port.relative_test_filename(test_input.filename)
313         test = self._port._tests[test_name]
314         if test.keyboard:
315             raise KeyboardInterrupt
316         if test.exception:
317             raise ValueError('exception from ' + test_name)
318         if test.hang:
319             time.sleep((float(test_input.timeout) * 4) / 1000.0)
320         return test_output.TestOutput(test.actual_text, test.actual_image,
321                                       test.actual_checksum, test.crash,
322                                       time.time() - start_time, test.timeout,
323                                       test.error)
324
325     def start(self):
326         pass
327
328     def stop(self):
329         pass