2011-02-03 Hayato Ito <hayato@chromium.org>
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / test.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 #
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions are
6 # met:
7 #
8 #     * Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 #     * Redistributions in binary form must reproduce the above
11 # copyright notice, this list of conditions and the following disclaimer
12 # in the documentation and/or other materials provided with the
13 # distribution.
14 #     * Neither the Google name nor the names of its
15 # contributors may be used to endorse or promote products derived from
16 # this software without specific prior written permission.
17 #
18 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30 """Dummy Port implementation used for testing."""
31 from __future__ import with_statement
32
33 import time
34
35 from webkitpy.common.system import filesystem_mock
36 from webkitpy.tool import mocktool
37
38 import base
39
40
41 # This sets basic expectations for a test. Each individual expectation
42 # can be overridden by a keyword argument in TestList.add().
43 class TestInstance:
44     def __init__(self, name):
45         self.name = name
46         self.base = name[(name.rfind("/") + 1):name.rfind(".html")]
47         self.crash = False
48         self.exception = False
49         self.hang = False
50         self.keyboard = False
51         self.error = ''
52         self.timeout = False
53
54         # The values of each field are treated as raw byte strings. They
55         # will be converted to unicode strings where appropriate using
56         # MockFileSystem.read_text_file().
57         self.actual_text = self.base + '-txt'
58         self.actual_checksum = self.base + '-checksum'
59
60         # We add the '\x8a' for the image file to prevent the value from
61         # being treated as UTF-8 (the character is invalid)
62         self.actual_image = self.base + '\x8a' + '-png'
63
64         self.expected_text = self.actual_text
65         self.expected_checksum = self.actual_checksum
66         self.expected_image = self.actual_image
67
68
69 # This is an in-memory list of tests, what we want them to produce, and
70 # what we want to claim are the expected results.
71 class TestList:
72     def __init__(self):
73         self.tests = {}
74
75     def add(self, name, **kwargs):
76         test = TestInstance(name)
77         for key, value in kwargs.items():
78             test.__dict__[key] = value
79         self.tests[name] = test
80
81     def keys(self):
82         return self.tests.keys()
83
84     def __contains__(self, item):
85         return item in self.tests
86
87     def __getitem__(self, item):
88         return self.tests[item]
89
90
91 def unit_test_list():
92     tests = TestList()
93     tests.add('failures/expected/checksum.html',
94               actual_checksum='checksum_fail-checksum')
95     tests.add('failures/expected/crash.html', crash=True)
96     tests.add('failures/expected/exception.html', exception=True)
97     tests.add('failures/expected/timeout.html', timeout=True)
98     tests.add('failures/expected/hang.html', hang=True)
99     tests.add('failures/expected/missing_text.html', expected_text=None)
100     tests.add('failures/expected/image.html',
101               actual_image='image_fail-png',
102               expected_image='image-png')
103     tests.add('failures/expected/image_checksum.html',
104               actual_checksum='image_checksum_fail-checksum',
105               actual_image='image_checksum_fail-png')
106     tests.add('failures/expected/keyboard.html', keyboard=True)
107     tests.add('failures/expected/missing_check.html', expected_checksum=None)
108     tests.add('failures/expected/missing_image.html', expected_image=None)
109     tests.add('failures/expected/missing_text.html', expected_text=None)
110     tests.add('failures/expected/newlines_leading.html',
111               expected_text="\nfoo\n", actual_text="foo\n")
112     tests.add('failures/expected/newlines_trailing.html',
113               expected_text="foo\n\n", actual_text="foo\n")
114     tests.add('failures/expected/newlines_with_excess_CR.html',
115               expected_text="foo\r\r\r\n", actual_text="foo\n")
116     tests.add('failures/expected/text.html', actual_text='text_fail-png')
117     tests.add('failures/unexpected/crash.html', crash=True)
118     tests.add('failures/unexpected/text-image-checksum.html',
119               actual_text='text-image-checksum_fail-txt',
120               actual_checksum='text-image-checksum_fail-checksum')
121     tests.add('failures/unexpected/timeout.html', timeout=True)
122     tests.add('http/tests/passes/text.html')
123     tests.add('http/tests/ssl/text.html')
124     tests.add('passes/error.html', error='stuff going to stderr')
125     tests.add('passes/image.html')
126     tests.add('passes/platform_image.html')
127
128     # Text output files contain "\r\n" on Windows.  This may be
129     # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
130     tests.add('passes/text.html',
131               expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n')
132     tests.add('websocket/tests/passes/text.html')
133     return tests
134
135
136 # Here we use a non-standard location for the layout tests, to ensure that
137 # this works. The path contains a '.' in the name because we've seen bugs
138 # related to this before.
139
140 LAYOUT_TEST_DIR = '/test.checkout/LayoutTests'
141
142
143 # Here we synthesize an in-memory filesystem from the test list
144 # in order to fully control the test output and to demonstrate that
145 # we don't need a real filesystem to run the tests.
146
147 def unit_test_filesystem(files=None):
148     """Return the FileSystem object used by the unit tests."""
149     test_list = unit_test_list()
150     files = files or {}
151
152     def add_file(files, test, suffix, contents):
153         dirname = test.name[0:test.name.rfind('/')]
154         base = test.base
155         path = LAYOUT_TEST_DIR + '/' + dirname + '/' + base + suffix
156         files[path] = contents
157
158     # Add each test and the expected output, if any.
159     for test in test_list.tests.values():
160         add_file(files, test, '.html', '')
161         add_file(files, test, '-expected.txt', test.expected_text)
162         add_file(files, test, '-expected.checksum', test.expected_checksum)
163         add_file(files, test, '-expected.png', test.expected_image)
164
165     # Add the test_expectations file.
166     files[LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt'] = """
167 WONTFIX : failures/expected/checksum.html = IMAGE
168 WONTFIX : failures/expected/crash.html = CRASH
169 // This one actually passes because the checksums will match.
170 WONTFIX : failures/expected/image.html = PASS
171 WONTFIX : failures/expected/image_checksum.html = IMAGE
172 WONTFIX : failures/expected/missing_check.html = MISSING PASS
173 WONTFIX : failures/expected/missing_image.html = MISSING PASS
174 WONTFIX : failures/expected/missing_text.html = MISSING PASS
175 WONTFIX : failures/expected/newlines_leading.html = TEXT
176 WONTFIX : failures/expected/newlines_trailing.html = TEXT
177 WONTFIX : failures/expected/newlines_with_excess_CR.html = TEXT
178 WONTFIX : failures/expected/text.html = TEXT
179 WONTFIX : failures/expected/timeout.html = TIMEOUT
180 WONTFIX SKIP : failures/expected/hang.html = TIMEOUT
181 WONTFIX SKIP : failures/expected/keyboard.html = CRASH
182 WONTFIX SKIP : failures/expected/exception.html = CRASH
183 """
184
185     # Add in a file should be ignored by test_files.find().
186     files[LAYOUT_TEST_DIR + 'userscripts/resources/iframe.html'] = 'iframe'
187
188     fs = filesystem_mock.MockFileSystem(files)
189     fs._tests = test_list
190     return fs
191
192
193 class TestPort(base.Port):
194     """Test implementation of the Port interface."""
195
196     def __init__(self, port_name=None, user=None, filesystem=None, **kwargs):
197         if not filesystem:
198             filesystem = unit_test_filesystem()
199
200         assert filesystem._tests
201         self._tests = filesystem._tests
202
203         if not user:
204             user = mocktool.MockUser()
205
206         if not port_name or port_name == 'test':
207             port_name = 'test-mac'
208
209         self._expectations_path = LAYOUT_TEST_DIR + '/platform/test/test_expectations.txt'
210         base.Port.__init__(self, port_name=port_name, filesystem=filesystem, user=user,
211                            **kwargs)
212
213     def _path_to_driver(self):
214         # This routine shouldn't normally be called, but it is called by
215         # the mock_drt Driver. We return something, but make sure it's useless.
216         return 'junk'
217
218     def baseline_path(self):
219         # We don't bother with a fallback path.
220         return self._filesystem.join(self.layout_tests_dir(), 'platform', self.name())
221
222     def baseline_search_path(self):
223         return [self.baseline_path()]
224
225     def check_build(self, needs_http):
226         return True
227
228     def default_configuration(self):
229         return 'Release'
230
231     def diff_image(self, expected_contents, actual_contents,
232                    diff_filename=None):
233         diffed = actual_contents != expected_contents
234         if diffed and diff_filename:
235             self._filesystem.write_binary_file(diff_filename,
236                 "< %s\n---\n> %s\n" % (expected_contents, actual_contents))
237         return diffed
238
239     def layout_tests_dir(self):
240         return LAYOUT_TEST_DIR
241
242     def name(self):
243         return self._name
244
245     def _path_to_wdiff(self):
246         return None
247
248     def results_directory(self):
249         return '/tmp/' + self.get_option('results_directory')
250
251     def setup_test_run(self):
252         pass
253
254     def create_driver(self, worker_number):
255         return TestDriver(self, worker_number)
256
257     def start_http_server(self):
258         pass
259
260     def start_websocket_server(self):
261         pass
262
263     def stop_http_server(self):
264         pass
265
266     def stop_websocket_server(self):
267         pass
268
269     def path_to_test_expectations_file(self):
270         return self._expectations_path
271
272     def test_platform_name(self):
273         name_map = {
274             'test-mac': 'mac',
275             'test-win': 'win',
276             'test-win-xp': 'win-xp',
277         }
278         return name_map[self._name]
279
280     def test_platform_names(self):
281         return ('mac', 'win', 'win-xp')
282
283     def test_platform_name_to_name(self, test_platform_name):
284         name_map = {
285             'mac': 'test-mac',
286             'win': 'test-win',
287             'win-xp': 'test-win-xp',
288         }
289         return name_map[test_platform_name]
290
291     # FIXME: These next two routines are copied from base.py with
292     # the calls to path.abspath_to_uri() removed. We shouldn't have
293     # to do this.
294     def filename_to_uri(self, filename):
295         """Convert a test file (which is an absolute path) to a URI."""
296         LAYOUTTEST_HTTP_DIR = "http/tests/"
297         LAYOUTTEST_WEBSOCKET_DIR = "http/tests/websocket/tests/"
298
299         relative_path = self.relative_test_filename(filename)
300         port = None
301         use_ssl = False
302
303         if (relative_path.startswith(LAYOUTTEST_WEBSOCKET_DIR)
304             or relative_path.startswith(LAYOUTTEST_HTTP_DIR)):
305             relative_path = relative_path[len(LAYOUTTEST_HTTP_DIR):]
306             port = 8000
307
308         # Make http/tests/local run as local files. This is to mimic the
309         # logic in run-webkit-tests.
310         #
311         # TODO(dpranke): remove the media reference and the SSL reference?
312         if (port and not relative_path.startswith("local/") and
313             not relative_path.startswith("media/")):
314             if relative_path.startswith("ssl/"):
315                 port += 443
316                 protocol = "https"
317             else:
318                 protocol = "http"
319             return "%s://127.0.0.1:%u/%s" % (protocol, port, relative_path)
320
321         return "file://" + self._filesystem.abspath(filename)
322
323     def uri_to_test_name(self, uri):
324         """Return the base layout test name for a given URI.
325
326         This returns the test name for a given URI, e.g., if you passed in
327         "file:///src/LayoutTests/fast/html/keygen.html" it would return
328         "fast/html/keygen.html".
329
330         """
331         test = uri
332         if uri.startswith("file:///"):
333             prefix = "file://" + self.layout_tests_dir() + "/"
334             return test[len(prefix):]
335
336         if uri.startswith("http://127.0.0.1:8880/"):
337             # websocket tests
338             return test.replace('http://127.0.0.1:8880/', '')
339
340         if uri.startswith("http://"):
341             # regular HTTP test
342             return test.replace('http://127.0.0.1:8000/', 'http/tests/')
343
344         if uri.startswith("https://"):
345             return test.replace('https://127.0.0.1:8443/', 'http/tests/')
346
347         raise NotImplementedError('unknown url type: %s' % uri)
348
349     def version(self):
350         version_map = {
351             'test-win-xp': '-xp',
352             'test-win': '-7',
353             'test-mac': '-leopard',
354         }
355         return version_map[self._name]
356
357     def test_configuration(self):
358         if not self._test_configuration:
359             self._test_configuration = TestTestConfiguration(self)
360         return self._test_configuration
361
362
363 class TestDriver(base.Driver):
364     """Test/Dummy implementation of the DumpRenderTree interface."""
365
366     def __init__(self, port, worker_number):
367         self._port = port
368
369     def cmd_line(self):
370         return [self._port._path_to_driver()]
371
372     def poll(self):
373         return True
374
375     def run_test(self, test_input):
376         start_time = time.time()
377         test_name = self._port.relative_test_filename(test_input.filename)
378         test = self._port._tests[test_name]
379         if test.keyboard:
380             raise KeyboardInterrupt
381         if test.exception:
382             raise ValueError('exception from ' + test_name)
383         if test.hang:
384             time.sleep((float(test_input.timeout) * 4) / 1000.0)
385         return base.DriverOutput(test.actual_text, test.actual_image,
386                                  test.actual_checksum, test.crash,
387                                  time.time() - start_time, test.timeout,
388                                  test.error)
389
390     def start(self):
391         pass
392
393     def stop(self):
394         pass
395
396
397 class TestTestConfiguration(base.TestConfiguration):
398     def all_systems(self):
399         return (('mac', 'leopard', 'x86'),
400                 ('win', 'xp', 'x86'),
401                 ('win', 'win7', 'x86'))