a4fc9244c3a5d9c524595c238b017bf5ead1e1e5
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / test.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the Google name nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Dummy Port implementation used for testing."""
31 from __future__ import with_statement
32
33 import base64
34 import time
35
36 from webkitpy.layout_tests.port import Port, Driver, DriverOutput
37 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
38 from webkitpy.common.host_mock import MockHost
39 from webkitpy.common.system.filesystem_mock import MockFileSystem
40
41
42 # This sets basic expectations for a test. Each individual expectation
43 # can be overridden by a keyword argument in TestList.add().
44 class TestInstance(object):
45     def __init__(self, name):
46         self.name = name
47         self.base = name[(name.rfind("/") + 1):name.rfind(".html")]
48         self.crash = False
49         self.web_process_crash = False
50         self.exception = False
51         self.hang = False
52         self.keyboard = False
53         self.error = ''
54         self.timeout = False
55         self.is_reftest = False
56
57         # The values of each field are treated as raw byte strings. They
58         # will be converted to unicode strings where appropriate using
59         # MockFileSystem.read_text_file().
60         self.actual_text = self.base + '-txt'
61         self.actual_checksum = self.base + '-checksum'
62
63         # We add the '\x8a' for the image file to prevent the value from
64         # being treated as UTF-8 (the character is invalid)
65         self.actual_image = self.base + '\x8a' + '-png' + 'tEXtchecksum\x00' + self.actual_checksum
66
67         self.expected_text = self.actual_text
68         self.expected_checksum = self.actual_checksum
69         self.expected_image = self.actual_image
70
71         self.actual_audio = None
72         self.expected_audio = None
73
74
75 # This is an in-memory list of tests, what we want them to produce, and
76 # what we want to claim are the expected results.
77 class TestList(object):
78     def __init__(self):
79         self.tests = {}
80
81     def add(self, name, **kwargs):
82         test = TestInstance(name)
83         for key, value in kwargs.items():
84             test.__dict__[key] = value
85         self.tests[name] = test
86
87     def add_reftest(self, name, reference_name, same_image):
88         self.add(name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
89         if same_image:
90             self.add(reference_name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
91         else:
92             self.add(reference_name, actual_checksum='yyy', actual_image='YYY', is_reftest=True)
93
94     def keys(self):
95         return self.tests.keys()
96
97     def __contains__(self, item):
98         return item in self.tests
99
100     def __getitem__(self, item):
101         return self.tests[item]
102
103
104 def unit_test_list():
105     tests = TestList()
106     tests.add('failures/expected/checksum.html',
107               actual_checksum='checksum_fail-checksum')
108     tests.add('failures/expected/crash.html', crash=True)
109     tests.add('failures/expected/exception.html', exception=True)
110     tests.add('failures/expected/timeout.html', timeout=True)
111     tests.add('failures/expected/hang.html', hang=True)
112     tests.add('failures/expected/missing_text.html', expected_text=None)
113     tests.add('failures/expected/image.html',
114               actual_image='image_fail-pngtEXtchecksum\x00checksum_fail',
115               expected_image='image-pngtEXtchecksum\x00checksum-png')
116     tests.add('failures/expected/image_checksum.html',
117               actual_checksum='image_checksum_fail-checksum',
118               actual_image='image_checksum_fail-png')
119     tests.add('failures/expected/audio.html',
120               actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='audio-wav',
121               actual_text=None, expected_text=None,
122               actual_image=None, expected_image=None,
123               actual_checksum=None, expected_checksum=None)
124     tests.add('failures/expected/keyboard.html', keyboard=True)
125     tests.add('failures/expected/missing_check.html',
126               expected_checksum=None,
127               expected_image=None)
128     tests.add('failures/expected/missing_image.html', expected_image=None)
129     tests.add('failures/expected/missing_audio.html', expected_audio=None,
130               actual_text=None, expected_text=None,
131               actual_image=None, expected_image=None,
132               actual_checksum=None, expected_checksum=None)
133     tests.add('failures/expected/missing_text.html', expected_text=None)
134     tests.add('failures/expected/newlines_leading.html',
135               expected_text="\nfoo\n", actual_text="foo\n")
136     tests.add('failures/expected/newlines_trailing.html',
137               expected_text="foo\n\n", actual_text="foo\n")
138     tests.add('failures/expected/newlines_with_excess_CR.html',
139               expected_text="foo\r\r\r\n", actual_text="foo\n")
140     tests.add('failures/expected/text.html', actual_text='text_fail-png')
141     tests.add('failures/unexpected/missing_text.html', expected_text=None)
142     tests.add('failures/unexpected/missing_image.html', expected_image=None)
143     tests.add('failures/unexpected/missing_render_tree_dump.html', actual_text="""layer at (0,0) size 800x600
144   RenderView at (0,0) size 800x600
145 layer at (0,0) size 800x34
146   RenderBlock {HTML} at (0,0) size 800x34
147     RenderBody {BODY} at (8,8) size 784x18
148       RenderText {#text} at (0,0) size 133x18
149         text run at (0,0) width 133: "This is an image test!"
150 """, expected_text=None)
151     tests.add('failures/unexpected/crash.html', crash=True)
152     tests.add('failures/unexpected/crash-with-stderr.html', crash=True,
153               error="mock-std-error-output")
154     tests.add('failures/unexpected/web-process-crash-with-stderr.html', web_process_crash=True,
155               error="mock-std-error-output")
156     tests.add('failures/unexpected/text-image-checksum.html',
157               actual_text='text-image-checksum_fail-txt',
158               actual_checksum='text-image-checksum_fail-checksum')
159     tests.add('failures/unexpected/checksum-with-matching-image.html',
160               actual_checksum='text-image-checksum_fail-checksum')
161     tests.add('failures/unexpected/timeout.html', timeout=True)
162     tests.add('http/tests/passes/text.html')
163     tests.add('http/tests/passes/image.html')
164     tests.add('http/tests/ssl/text.html')
165     tests.add('passes/error.html', error='stuff going to stderr')
166     tests.add('passes/image.html')
167     tests.add('passes/audio.html',
168               actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-wav',
169               actual_text=None, expected_text=None,
170               actual_image=None, expected_image=None,
171               actual_checksum=None, expected_checksum=None)
172     tests.add('passes/platform_image.html')
173     tests.add('passes/checksum_in_image.html',
174               expected_checksum=None,
175               expected_image='tEXtchecksum\x00checksum_in_image-checksum')
176
177     # Text output files contain "\r\n" on Windows.  This may be
178     # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
179     tests.add('passes/text.html',
180               expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n')
181
182     # For reftests.
183     tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True)
184     tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False)
185     tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False)
186     tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True)
187     tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
188     tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
189     # FIXME: Add a reftest which crashes.
190
191     tests.add('websocket/tests/passes/text.html')
192
193     # For --no-http tests, test that platform specific HTTP tests are properly skipped.
194     tests.add('platform/test-snow-leopard/http/test.html')
195     tests.add('platform/test-snow-leopard/websocket/test.html')
196
197     return tests
198
199
200 # Here we use a non-standard location for the layout tests, to ensure that
201 # this works. The path contains a '.' in the name because we've seen bugs
202 # related to this before.
203
204 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests'
205
206
207 # Here we synthesize an in-memory filesystem from the test list
208 # in order to fully control the test output and to demonstrate that
209 # we don't need a real filesystem to run the tests.
210
211 def unit_test_filesystem(files=None):
212     """Return the FileSystem object used by the unit tests."""
213     test_list = unit_test_list()
214     files = files or {}
215
216     def add_file(files, test, suffix, contents):
217         dirname = test.name[0:test.name.rfind('/')]
218         base = test.base
219         path = LAYOUT_TEST_DIR + '/' + dirname + '/' + base + suffix
220         files[path] = contents
221
222     # Add each test and the expected output, if any.
223     for test in test_list.tests.values():
224         add_file(files, test, '.html', '')
225         if test.is_reftest:
226             continue
227         if test.actual_audio:
228             add_file(files, test, '-expected.wav', test.expected_audio)
229             continue
230
231         add_file(files, test, '-expected.txt', test.expected_text)
232         add_file(files, test, '-expected.png', test.expected_image)
233
234
235     # Add the test_expectations file.
236     files[LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt'] = """
237 WONTFIX : failures/expected/checksum.html = IMAGE
238 WONTFIX : failures/expected/crash.html = CRASH
239 WONTFIX : failures/expected/image.html = IMAGE
240 WONTFIX : failures/expected/audio.html = AUDIO
241 WONTFIX : failures/expected/image_checksum.html = IMAGE
242 WONTFIX : failures/expected/mismatch.html = IMAGE
243 WONTFIX : failures/expected/missing_check.html = MISSING PASS
244 WONTFIX : failures/expected/missing_image.html = MISSING PASS
245 WONTFIX : failures/expected/missing_audio.html = MISSING PASS
246 WONTFIX : failures/expected/missing_text.html = MISSING PASS
247 WONTFIX : failures/expected/newlines_leading.html = TEXT
248 WONTFIX : failures/expected/newlines_trailing.html = TEXT
249 WONTFIX : failures/expected/newlines_with_excess_CR.html = TEXT
250 WONTFIX : failures/expected/reftest.html = IMAGE
251 WONTFIX : failures/expected/text.html = TEXT
252 WONTFIX : failures/expected/timeout.html = TIMEOUT
253 WONTFIX SKIP : failures/expected/hang.html = TIMEOUT
254 WONTFIX SKIP : failures/expected/keyboard.html = CRASH
255 WONTFIX SKIP : failures/expected/exception.html = CRASH
256 """
257
258     # FIXME: This test was only being ignored because of missing a leading '/'.
259     # Fixing the typo causes several tests to assert, so disabling the test entirely.
260     # Add in a file should be ignored by test_files.find().
261     #files[LAYOUT_TEST_DIR + '/userscripts/resources/iframe.html'] = 'iframe'
262
263     fs = MockFileSystem(files, dirs=set(['/mock-checkout']))  # Make sure at least the checkout_root exists as a directory.
264     fs._tests = test_list
265     return fs
266
267
268 class TestPort(Port):
269     """Test implementation of the Port interface."""
270     ALL_BASELINE_VARIANTS = (
271         'test-mac-snowleopard', 'test-mac-leopard',
272         'test-win-win7', 'test-win-vista', 'test-win-xp',
273         'test-linux-x86_64',
274     )
275
276     def _set_default_overriding_none(self, dictionary, key, default):
277         # dict.setdefault almost works, but won't actually override None values, which we want.
278         if not dictionary.get(key):
279             dictionary[key] = default
280         return dictionary[key]
281
282     def __init__(self, host=None, port_name=None, **kwargs):
283         if not port_name or port_name == 'test':
284             port_name = 'test-mac-leopard'
285
286         host = host or MockHost()
287         filesystem = self._set_default_overriding_none(kwargs, 'filesystem', unit_test_filesystem())
288
289         Port.__init__(self, host, port_name=port_name, **kwargs)
290         self._results_directory = None
291
292         assert filesystem._tests
293         self._tests = filesystem._tests
294
295         self._operating_system = 'mac'
296         if port_name.startswith('test-win'):
297             self._operating_system = 'win'
298         elif port_name.startswith('test-linux'):
299             self._operating_system = 'linux'
300
301         version_map = {
302             'test-win-xp': 'xp',
303             'test-win-win7': 'win7',
304             'test-win-vista': 'vista',
305             'test-mac-leopard': 'leopard',
306             'test-mac-snowleopard': 'snowleopard',
307             'test-linux-x86_64': 'lucid',
308         }
309         self._version = version_map[port_name]
310
311         self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt'
312
313     def _path_to_driver(self):
314         # This routine shouldn't normally be called, but it is called by
315         # the mock_drt Driver. We return something, but make sure it's useless.
316         return 'junk'
317
318     def baseline_search_path(self):
319         search_paths = {
320             'test-mac-snowleopard': ['test-mac-snowleopard'],
321             'test-mac-leopard': ['test-mac-leopard', 'test-mac-snowleopard'],
322             'test-win-win7': ['test-win-win7'],
323             'test-win-vista': ['test-win-vista', 'test-win-win7'],
324             'test-win-xp': ['test-win-xp', 'test-win-vista', 'test-win-win7'],
325             'test-linux-x86_64': ['test-linux', 'test-win-win7'],
326         }
327         return [self._webkit_baseline_path(d) for d in search_paths[self.name()]]
328
329     def default_child_processes(self):
330         return 1
331
332     def default_worker_model(self):
333         return 'inline'
334
335     def check_build(self, needs_http):
336         return True
337
338     def check_sys_deps(self, needs_http):
339         return True
340
341     def default_configuration(self):
342         return 'Release'
343
344     def diff_image(self, expected_contents, actual_contents):
345         diffed = actual_contents != expected_contents
346         if diffed:
347             return ["< %s\n---\n> %s\n" % (expected_contents, actual_contents), 1]
348         return (None, 0)
349
350     def layout_tests_dir(self):
351         return LAYOUT_TEST_DIR
352
353     def name(self):
354         return self._name
355
356     def _path_to_wdiff(self):
357         return None
358
359     def default_results_directory(self):
360         return '/tmp/layout-test-results'
361
362     def setup_test_run(self):
363         pass
364
365     def _driver_class(self):
366         return TestDriver
367
368     def start_http_server(self):
369         pass
370
371     def start_websocket_server(self):
372         pass
373
374     def acquire_http_lock(self):
375         pass
376
377     def stop_http_server(self):
378         pass
379
380     def stop_websocket_server(self):
381         pass
382
383     def release_http_lock(self):
384         pass
385
386     def path_to_test_expectations_file(self):
387         return self._expectations_path
388
389     def all_test_configurations(self):
390         """Returns a sequence of the TestConfigurations the port supports."""
391         # By default, we assume we want to test every graphics type in
392         # every configuration on every system.
393         test_configurations = []
394         for version, architecture in self._all_systems():
395             for build_type in self._all_build_types():
396                 for graphics_type in self._all_graphics_types():
397                     test_configurations.append(TestConfiguration(
398                         version=version,
399                         architecture=architecture,
400                         build_type=build_type,
401                         graphics_type=graphics_type))
402         return test_configurations
403
404     def _all_systems(self):
405         return (('leopard', 'x86'),
406                 ('snowleopard', 'x86'),
407                 ('xp', 'x86'),
408                 ('vista', 'x86'),
409                 ('win7', 'x86'),
410                 ('lucid', 'x86'),
411                 ('lucid', 'x86_64'))
412
413     def _all_build_types(self):
414         return ('debug', 'release')
415
416     def _all_graphics_types(self):
417         return ('cpu', 'gpu')
418
419     def configuration_specifier_macros(self):
420         """To avoid surprises when introducing new macros, these are intentionally fixed in time."""
421         return {'mac': ['leopard', 'snowleopard'], 'win': ['xp', 'vista', 'win7'], 'linux': ['lucid']}
422
423     def all_baseline_variants(self):
424         return self.ALL_BASELINE_VARIANTS
425
426     # FIXME: These next two routines are copied from base.py with
427     # the calls to path.abspath_to_uri() removed. We shouldn't have
428     # to do this.
429     def test_to_uri(self, test_name):
430         """Convert a test file (which is an absolute path) to a URI."""
431         LAYOUTTEST_HTTP_DIR = "http/tests/"
432         LAYOUTTEST_WEBSOCKET_DIR = "http/tests/websocket/tests/"
433
434         port = None
435         use_ssl = False
436
437         relative_path = test_name
438         if (relative_path.startswith(LAYOUTTEST_WEBSOCKET_DIR)
439             or relative_path.startswith(LAYOUTTEST_HTTP_DIR)):
440             relative_path = relative_path[len(LAYOUTTEST_HTTP_DIR):]
441             port = 8000
442
443         # Make http/tests/local run as local files. This is to mimic the
444         # logic in run-webkit-tests.
445         #
446         # TODO(dpranke): remove the media reference and the SSL reference?
447         if (port and not relative_path.startswith("local/") and
448             not relative_path.startswith("media/")):
449             if relative_path.startswith("ssl/"):
450                 port += 443
451                 protocol = "https"
452             else:
453                 protocol = "http"
454             return "%s://127.0.0.1:%u/%s" % (protocol, port, relative_path)
455
456         return "file://" + self.abspath_for_test(test_name)
457
458     def abspath_for_test(self, test_name):
459         return self.layout_tests_dir() + self.TEST_PATH_SEPARATOR + test_name
460
461     def uri_to_test_name(self, uri):
462         """Return the base layout test name for a given URI.
463
464         This returns the test name for a given URI, e.g., if you passed in
465         "file:///src/LayoutTests/fast/html/keygen.html" it would return
466         "fast/html/keygen.html".
467
468         """
469         test = uri
470         if uri.startswith("file:///"):
471             prefix = "file://" + self.layout_tests_dir() + "/"
472             return test[len(prefix):]
473
474         if uri.startswith("http://127.0.0.1:8880/"):
475             # websocket tests
476             return test.replace('http://127.0.0.1:8880/', '')
477
478         if uri.startswith("http://"):
479             # regular HTTP test
480             return test.replace('http://127.0.0.1:8000/', 'http/tests/')
481
482         if uri.startswith("https://"):
483             return test.replace('https://127.0.0.1:8443/', 'http/tests/')
484
485         raise NotImplementedError('unknown url type: %s' % uri)
486
487
488 class TestDriver(Driver):
489     """Test/Dummy implementation of the DumpRenderTree interface."""
490
491     def cmd_line(self):
492         return [self._port._path_to_driver()] + self._port.get_option('additional_drt_flag', [])
493
494     def run_test(self, test_input):
495         start_time = time.time()
496         test_name = test_input.test_name
497         test = self._port._tests[test_name]
498         if test.keyboard:
499             raise KeyboardInterrupt
500         if test.exception:
501             raise ValueError('exception from ' + test_name)
502         if test.hang:
503             time.sleep((float(test_input.timeout) * 4) / 1000.0)
504
505         audio = None
506         if test.actual_audio:
507             audio = base64.b64decode(test.actual_audio)
508         crashed_process_name = None
509         if test.crash:
510             crashed_process_name = self._port.driver_name()
511         elif test.web_process_crash:
512             crashed_process_name = 'WebProcess'
513         return DriverOutput(test.actual_text, test.actual_image,
514             test.actual_checksum, audio, crash=test.crash or test.web_process_crash,
515             crashed_process_name=crashed_process_name,
516             test_time=time.time() - start_time, timeout=test.timeout, error=test.error)
517
518     def stop(self):
519         pass