94ebb5b7d912d028c7664bcd2259d0250093b429
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / test.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the Google name nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Dummy Port implementation used for testing."""
31 from __future__ import with_statement
32
33 import time
34
35 from webkitpy.common.system import filesystem_mock
36 from webkitpy.tool import mocktool
37
38 from webkitpy.layout_tests.layout_package import test_output
39
40 import base
41
42
43 # This sets basic expectations for a test. Each individual expectation
44 # can be overridden by a keyword argument in TestList.add().
45 class TestInstance:
46     def __init__(self, name):
47         self.name = name
48         self.base = name[(name.rfind("/") + 1):name.rfind(".html")]
49         self.crash = False
50         self.exception = False
51         self.hang = False
52         self.keyboard = False
53         self.error = ''
54         self.timeout = False
55
56         # The values of each field are treated as raw byte strings. They
57         # will be converted to unicode strings where appropriate using
58         # MockFileSystem.read_text_file().
59         self.actual_text = self.base + '-txt'
60         self.actual_checksum = self.base + '-checksum'
61
62         # We add the '\x8a' for the image file to prevent the value from
63         # being treated as UTF-8 (the character is invalid)
64         self.actual_image = self.base + '\x8a' + '-png'
65
66         self.expected_text = self.actual_text
67         self.expected_checksum = self.actual_checksum
68         self.expected_image = self.actual_image
69
70
71 # This is an in-memory list of tests, what we want them to produce, and
72 # what we want to claim are the expected results.
73 class TestList:
74     def __init__(self):
75         self.tests = {}
76
77     def add(self, name, **kwargs):
78         test = TestInstance(name)
79         for key, value in kwargs.items():
80             test.__dict__[key] = value
81         self.tests[name] = test
82
83     def keys(self):
84         return self.tests.keys()
85
86     def __contains__(self, item):
87         return item in self.tests
88
89     def __getitem__(self, item):
90         return self.tests[item]
91
92
93 def unit_test_list():
94     tests = TestList()
95     tests.add('failures/expected/checksum.html',
96               actual_checksum='checksum_fail-checksum')
97     tests.add('failures/expected/crash.html', crash=True)
98     tests.add('failures/expected/exception.html', exception=True)
99     tests.add('failures/expected/timeout.html', timeout=True)
100     tests.add('failures/expected/hang.html', hang=True)
101     tests.add('failures/expected/missing_text.html', expected_text=None)
102     tests.add('failures/expected/image.html',
103               actual_image='image_fail-png',
104               expected_image='image-png')
105     tests.add('failures/expected/image_checksum.html',
106               actual_checksum='image_checksum_fail-checksum',
107               actual_image='image_checksum_fail-png')
108     tests.add('failures/expected/keyboard.html', keyboard=True)
109     tests.add('failures/expected/missing_check.html', expected_checksum=None)
110     tests.add('failures/expected/missing_image.html', expected_image=None)
111     tests.add('failures/expected/missing_text.html', expected_text=None)
112     tests.add('failures/expected/newlines_leading.html',
113               expected_text="\nfoo\n", actual_text="foo\n")
114     tests.add('failures/expected/newlines_trailing.html',
115               expected_text="foo\n\n", actual_text="foo\n")
116     tests.add('failures/expected/newlines_with_excess_CR.html',
117               expected_text="foo\r\r\r\n", actual_text="foo\n")
118     tests.add('failures/expected/text.html', actual_text='text_fail-png')
119     tests.add('failures/unexpected/crash.html', crash=True)
120     tests.add('failures/unexpected/text-image-checksum.html',
121               actual_text='text-image-checksum_fail-txt',
122               actual_checksum='text-image-checksum_fail-checksum')
123     tests.add('failures/unexpected/timeout.html', timeout=True)
124     tests.add('http/tests/passes/text.html')
125     tests.add('http/tests/ssl/text.html')
126     tests.add('passes/error.html', error='stuff going to stderr')
127     tests.add('passes/image.html')
128     tests.add('passes/platform_image.html')
129
130     # Text output files contain "\r\n" on Windows.  This may be
131     # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
132     tests.add('passes/text.html',
133               expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n')
134     tests.add('websocket/tests/passes/text.html')
135     return tests
136
137
138 # Here we use a non-standard location for the layout tests, to ensure that
139 # this works. The path contains a '.' in the name because we've seen bugs
140 # related to this before.
141
142 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests'
143
144
145 # Here we synthesize an in-memory filesystem from the test list
146 # in order to fully control the test output and to demonstrate that
147 # we don't need a real filesystem to run the tests.
148
149 def unit_test_filesystem(files=None):
150     """Return the FileSystem object used by the unit tests."""
151     test_list = unit_test_list()
152     files = files or {}
153
154     def add_file(files, test, suffix, contents):
155         dirname = test.name[0:test.name.rfind('/')]
156         base = test.base
157         path = LAYOUT_TEST_DIR + '/' + dirname + '/' + base + suffix
158         files[path] = contents
159
160     # Add each test and the expected output, if any.
161     for test in test_list.tests.values():
162         add_file(files, test, '.html', '')
163         add_file(files, test, '-expected.txt', test.expected_text)
164         add_file(files, test, '-expected.checksum', test.expected_checksum)
165         add_file(files, test, '-expected.png', test.expected_image)
166
167     # Add the test_expectations file.
168     files[LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt'] = """
169 WONTFIX : failures/expected/checksum.html = IMAGE
170 WONTFIX : failures/expected/crash.html = CRASH
171 // This one actually passes because the checksums will match.
172 WONTFIX : failures/expected/image.html = PASS
173 WONTFIX : failures/expected/image_checksum.html = IMAGE
174 WONTFIX : failures/expected/missing_check.html = MISSING PASS
175 WONTFIX : failures/expected/missing_image.html = MISSING PASS
176 WONTFIX : failures/expected/missing_text.html = MISSING PASS
177 WONTFIX : failures/expected/newlines_leading.html = TEXT
178 WONTFIX : failures/expected/newlines_trailing.html = TEXT
179 WONTFIX : failures/expected/newlines_with_excess_CR.html = TEXT
180 WONTFIX : failures/expected/text.html = TEXT
181 WONTFIX : failures/expected/timeout.html = TIMEOUT
182 WONTFIX SKIP : failures/expected/hang.html = TIMEOUT
183 WONTFIX SKIP : failures/expected/keyboard.html = CRASH
184 WONTFIX SKIP : failures/expected/exception.html = CRASH
185 """
186
187     # Add in a file should be ignored by test_files.find().
188     files[LAYOUT_TEST_DIR + 'userscripts/resources/iframe.html'] = 'iframe'
189
190     fs = filesystem_mock.MockFileSystem(files)
191     fs._tests = test_list
192     return fs
193
194
195 class TestPort(base.Port):
196     """Test implementation of the Port interface."""
197
198     def __init__(self, port_name=None, user=None, filesystem=None, **kwargs):
199         if not filesystem:
200             filesystem = unit_test_filesystem()
201
202         assert filesystem._tests
203         self._tests = filesystem._tests
204
205         if not user:
206             user = mocktool.MockUser()
207
208         if not port_name or port_name == 'test':
209             port_name = 'test-mac'
210
211         self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt'
212         base.Port.__init__(self, port_name=port_name, filesystem=filesystem, user=user,
213                            **kwargs)
214
215     def _path_to_driver(self):
216         # This routine shouldn't normally be called, but it is called by
217         # the mock_drt Driver. We return something, but make sure it's useless.
218         return 'junk'
219
220     def baseline_path(self):
221         # We don't bother with a fallback path.
222         return self._filesystem.join(self.layout_tests_dir(), 'platform', self.name())
223
224     def baseline_search_path(self):
225         return [self.baseline_path()]
226
227     def check_build(self, needs_http):
228         return True
229
230     def default_configuration(self):
231         return 'Release'
232
233     def diff_image(self, expected_contents, actual_contents,
234                    diff_filename=None):
235         diffed = actual_contents != expected_contents
236         if diffed and diff_filename:
237             self._filesystem.write_binary_file(diff_filename,
238                 "< %s\n---\n> %s\n" % (expected_contents, actual_contents))
239         return diffed
240
241     def layout_tests_dir(self):
242         return LAYOUT_TEST_DIR
243
244     def name(self):
245         return self._name
246
247     def _path_to_wdiff(self):
248         return None
249
250     def results_directory(self):
251         return '/tmp/' + self.get_option('results_directory')
252
253     def setup_test_run(self):
254         pass
255
256     def create_driver(self, worker_number):
257         return TestDriver(self, worker_number)
258
259     def start_http_server(self):
260         pass
261
262     def start_websocket_server(self):
263         pass
264
265     def stop_http_server(self):
266         pass
267
268     def stop_websocket_server(self):
269         pass
270
271     def path_to_test_expectations_file(self):
272         return self._expectations_path
273
274     def test_platform_name(self):
275         name_map = {
276             'test-mac': 'mac',
277             'test-win': 'win',
278             'test-win-xp': 'win-xp',
279         }
280         return name_map[self._name]
281
282     def test_platform_names(self):
283         return ('mac', 'win', 'win-xp')
284
285     def test_platform_name_to_name(self, test_platform_name):
286         name_map = {
287             'mac': 'test-mac',
288             'win': 'test-win',
289             'win-xp': 'test-win-xp',
290         }
291         return name_map[test_platform_name]
292
293     # FIXME: These next two routines are copied from base.py with
294     # the calls to path.abspath_to_uri() removed. We shouldn't have
295     # to do this.
296     def filename_to_uri(self, filename):
297         """Convert a test file (which is an absolute path) to a URI."""
298         LAYOUTTEST_HTTP_DIR = "http/tests/"
299         LAYOUTTEST_WEBSOCKET_DIR = "http/tests/websocket/tests/"
300
301         relative_path = self.relative_test_filename(filename)
302         port = None
303         use_ssl = False
304
305         if (relative_path.startswith(LAYOUTTEST_WEBSOCKET_DIR)
306             or relative_path.startswith(LAYOUTTEST_HTTP_DIR)):
307             relative_path = relative_path[len(LAYOUTTEST_HTTP_DIR):]
308             port = 8000
309
310         # Make http/tests/local run as local files. This is to mimic the
311         # logic in run-webkit-tests.
312         #
313         # TODO(dpranke): remove the media reference and the SSL reference?
314         if (port and not relative_path.startswith("local/") and
315             not relative_path.startswith("media/")):
316             if relative_path.startswith("ssl/"):
317                 port += 443
318                 protocol = "https"
319             else:
320                 protocol = "http"
321             return "%s://127.0.0.1:%u/%s" % (protocol, port, relative_path)
322
323         return "file://" + self._filesystem.abspath(filename)
324
325     def uri_to_test_name(self, uri):
326         """Return the base layout test name for a given URI.
327
328         This returns the test name for a given URI, e.g., if you passed in
329         "file:///src/LayoutTests/fast/html/keygen.html" it would return
330         "fast/html/keygen.html".
331
332         """
333         test = uri
334         if uri.startswith("file:///"):
335             prefix = "file://" + self.layout_tests_dir() + "/"
336             return test[len(prefix):]
337
338         if uri.startswith("http://127.0.0.1:8880/"):
339             # websocket tests
340             return test.replace('http://127.0.0.1:8880/', '')
341
342         if uri.startswith("http://"):
343             # regular HTTP test
344             return test.replace('http://127.0.0.1:8000/', 'http/tests/')
345
346         if uri.startswith("https://"):
347             return test.replace('https://127.0.0.1:8443/', 'http/tests/')
348
349         raise NotImplementedError('unknown url type: %s' % uri)
350
351     def version(self):
352         version_map = {
353             'test-win-xp': '-xp',
354             'test-win': '-7',
355             'test-mac': '-leopard',
356         }
357         return version_map[self._name]
358
359     def test_configuration(self):
360         if not self._test_configuration:
361             self._test_configuration = TestTestConfiguration(self)
362         return self._test_configuration
363
364
365 class TestDriver(base.Driver):
366     """Test/Dummy implementation of the DumpRenderTree interface."""
367
368     def __init__(self, port, worker_number):
369         self._port = port
370
371     def cmd_line(self):
372         return [self._port._path_to_driver()]
373
374     def poll(self):
375         return True
376
377     def run_test(self, test_input):
378         start_time = time.time()
379         test_name = self._port.relative_test_filename(test_input.filename)
380         test = self._port._tests[test_name]
381         if test.keyboard:
382             raise KeyboardInterrupt
383         if test.exception:
384             raise ValueError('exception from ' + test_name)
385         if test.hang:
386             time.sleep((float(test_input.timeout) * 4) / 1000.0)
387         return test_output.TestOutput(test.actual_text, test.actual_image,
388                                       test.actual_checksum, test.crash,
389                                       time.time() - start_time, test.timeout,
390                                       test.error)
391
392     def start(self):
393         pass
394
395     def stop(self):
396         pass
397
398
399 class TestTestConfiguration(base.TestConfiguration):
400     def all_systems(self):
401         return (('mac', 'leopard', 'x86'),
402                 ('win', 'xp', 'x86'),
403                 ('win', 'win7', 'x86'))