2011-01-21 Dirk Pranke <dpranke@chromium.org>
[WebKit.git] / Tools / Scripts / webkitpy / layout_tests / port / webkit.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the Google name nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """WebKit implementations of the Port interface."""
32
33
34 import logging
35 import operator
36 import os
37 import re
38 import signal
39 import sys
40 import time
41 import webbrowser
42
43 import webkitpy.common.system.ospath as ospath
44 import webkitpy.layout_tests.layout_package.test_output as test_output
45 import webkitpy.layout_tests.port.base as base
46 import webkitpy.layout_tests.port.server_process as server_process
47
48 _log = logging.getLogger("webkitpy.layout_tests.port.webkit")
49
50
51 class WebKitPort(base.Port):
52     """WebKit implementation of the Port class."""
53
54     def __init__(self, **kwargs):
55         base.Port.__init__(self, **kwargs)
56         self._cached_apache_path = None
57
58         # FIXME: disable pixel tests until they are run by default on the
59         # build machines.
60         self.set_option_default('pixel_tests', False)
61
62     def baseline_path(self):
63         return self._webkit_baseline_path(self._name)
64
65     def baseline_search_path(self):
66         return [self._webkit_baseline_path(self._name)]
67
68     def path_to_test_expectations_file(self):
69         return self._filesystem.join(self._webkit_baseline_path(self._name),
70                                      'test_expectations.txt')
71
72     # Only needed by ports which maintain versioned test expectations (like mac-tiger vs. mac-leopard)
73     def version(self):
74         return ''
75
76     def _build_driver(self):
77         configuration = self.get_option('configuration')
78         return self._config.build_dumprendertree(configuration)
79
80     def _check_driver(self):
81         driver_path = self._path_to_driver()
82         if not self._filesystem.exists(driver_path):
83             _log.error("DumpRenderTree was not found at %s" % driver_path)
84             return False
85         return True
86
87     def check_build(self, needs_http):
88         if self.get_option('build') and not self._build_driver():
89             return False
90         if not self._check_driver():
91             return False
92         if self.get_option('pixel_tests'):
93             if not self.check_image_diff():
94                 return False
95         if not self._check_port_build():
96             return False
97         return True
98
99     def _check_port_build(self):
100         # Ports can override this method to do additional checks.
101         return True
102
103     def check_image_diff(self, override_step=None, logging=True):
104         image_diff_path = self._path_to_image_diff()
105         if not self._filesystem.exists(image_diff_path):
106             _log.error("ImageDiff was not found at %s" % image_diff_path)
107             return False
108         return True
109
110     def diff_image(self, expected_contents, actual_contents,
111                    diff_filename=None):
112         """Return True if the two files are different. Also write a delta
113         image of the two images into |diff_filename| if it is not None."""
114
115         # Handle the case where the test didn't actually generate an image.
116         if not actual_contents:
117             return True
118
119         sp = self._diff_image_request(expected_contents, actual_contents)
120         return self._diff_image_reply(sp, diff_filename)
121
122     def _diff_image_request(self, expected_contents, actual_contents):
123         # FIXME: use self.get_option('tolerance') and
124         # self.set_option_default('tolerance', 0.1) once that behaves correctly
125         # with default values.
126         if self.get_option('tolerance') is not None:
127             tolerance = self.get_option('tolerance')
128         else:
129             tolerance = 0.1
130         command = [self._path_to_image_diff(), '--tolerance', str(tolerance)]
131         sp = server_process.ServerProcess(self, 'ImageDiff', command)
132
133         sp.write('Content-Length: %d\n%sContent-Length: %d\n%s' %
134                  (len(actual_contents), actual_contents,
135                   len(expected_contents), expected_contents))
136
137         return sp
138
139     def _diff_image_reply(self, sp, diff_filename):
140         timeout = 2.0
141         deadline = time.time() + timeout
142         output = sp.read_line(timeout)
143         while not sp.timed_out and not sp.crashed and output:
144             if output.startswith('Content-Length'):
145                 m = re.match('Content-Length: (\d+)', output)
146                 content_length = int(m.group(1))
147                 timeout = deadline - time.time()
148                 output = sp.read(timeout, content_length)
149                 break
150             elif output.startswith('diff'):
151                 break
152             else:
153                 timeout = deadline - time.time()
154                 output = sp.read_line(deadline)
155
156         result = True
157         if output.startswith('diff'):
158             m = re.match('diff: (.+)% (passed|failed)', output)
159             if m.group(2) == 'passed':
160                 result = False
161         elif output and diff_filename:
162             self._filesystem.write_text_file(diff_filename, output)
163         elif sp.timed_out:
164             _log.error("ImageDiff timed out")
165         elif sp.crashed:
166             _log.error("ImageDiff crashed")
167         sp.stop()
168         return result
169
170     def results_directory(self):
171         # Results are store relative to the built products to make it easy
172         # to have multiple copies of webkit checked out and built.
173         return self._build_path(self.get_option('results_directory'))
174
175     def setup_test_run(self):
176         # This port doesn't require any specific configuration.
177         pass
178
179     def create_driver(self, worker_number):
180         return WebKitDriver(self, worker_number)
181
182     def test_base_platform_names(self):
183         # At the moment we don't use test platform names, but we have
184         # to return something.
185         return ('mac', 'win')
186
187     def _tests_for_other_platforms(self):
188         raise NotImplementedError('WebKitPort._tests_for_other_platforms')
189         # The original run-webkit-tests builds up a "whitelist" of tests to
190         # run, and passes that to DumpRenderTree. new-run-webkit-tests assumes
191         # we run *all* tests and test_expectations.txt functions as a
192         # blacklist.
193         # FIXME: This list could be dynamic based on platform name and
194         # pushed into base.Port.
195         return [
196             "platform/chromium",
197             "platform/gtk",
198             "platform/qt",
199             "platform/win",
200         ]
201
202     def _runtime_feature_list(self):
203         """Return the supported features of DRT. If a port doesn't support
204         this DRT switch, it has to override this method to return None"""
205         driver_path = self._path_to_driver()
206         feature_list = ' '.join(os.popen(driver_path + " --print-supported-features 2>&1").readlines())
207         if "SupportedFeatures:" in feature_list:
208             return feature_list
209         return None
210
211     def _supported_symbol_list(self):
212         """Return the supported symbols of WebCore."""
213         webcore_library_path = self._path_to_webcore_library()
214         if not webcore_library_path:
215             return None
216         symbol_list = ' '.join(os.popen("nm " + webcore_library_path).readlines())
217         return symbol_list
218
219     def _directories_for_features(self):
220         """Return the supported feature dictionary. The keys are the
221         features and the values are the directories in lists."""
222         directories_for_features = {
223             "Accelerated Compositing": ["compositing"],
224             "3D Rendering": ["animations/3d", "transforms/3d"],
225         }
226         return directories_for_features
227
228     def _directories_for_symbols(self):
229         """Return the supported feature dictionary. The keys are the
230         symbols and the values are the directories in lists."""
231         directories_for_symbol = {
232             "MathMLElement": ["mathml"],
233             "GraphicsLayer": ["compositing"],
234             "WebCoreHas3DRendering": ["animations/3d", "transforms/3d"],
235             "WebGLShader": ["fast/canvas/webgl", "compositing/webgl", "http/tests/canvas/webgl"],
236             "WMLElement": ["http/tests/wml", "fast/wml", "wml"],
237             "parseWCSSInputProperty": ["fast/wcss"],
238             "isXHTMLMPDocument": ["fast/xhtmlmp"],
239         }
240         return directories_for_symbol
241
242     def _skipped_tests_for_unsupported_features(self):
243         """Return the directories of unsupported tests. Search for the
244         symbols in the symbol_list, if found add the corresponding
245         directories to the skipped directory list."""
246         feature_list = self._runtime_feature_list()
247         directories = self._directories_for_features()
248
249         # if DRT feature detection not supported
250         if not feature_list:
251             feature_list = self._supported_symbol_list()
252             directories = self._directories_for_symbols()
253
254         if not feature_list:
255             return []
256
257         skipped_directories = [directories[feature]
258                               for feature in directories.keys()
259                               if feature not in feature_list]
260         return reduce(operator.add, skipped_directories)
261
262     def _tests_for_disabled_features(self):
263         # FIXME: This should use the feature detection from
264         # webkitperl/features.pm to match run-webkit-tests.
265         # For now we hard-code a list of features known to be disabled on
266         # the Mac platform.
267         disabled_feature_tests = [
268             "fast/xhtmlmp",
269             "http/tests/wml",
270             "mathml",
271             "wml",
272         ]
273         # FIXME: webarchive tests expect to read-write from
274         # -expected.webarchive files instead of .txt files.
275         # This script doesn't know how to do that yet, so pretend they're
276         # just "disabled".
277         webarchive_tests = [
278             "webarchive",
279             "svg/webarchive",
280             "http/tests/webarchive",
281             "svg/custom/image-with-prefix-in-webarchive.svg",
282         ]
283         unsupported_feature_tests = self._skipped_tests_for_unsupported_features()
284         return disabled_feature_tests + webarchive_tests + unsupported_feature_tests
285
286     def _tests_from_skipped_file_contents(self, skipped_file_contents):
287         tests_to_skip = []
288         for line in skipped_file_contents.split('\n'):
289             line = line.strip()
290             if line.startswith('#') or not len(line):
291                 continue
292             tests_to_skip.append(line)
293         return tests_to_skip
294
295     def _skipped_file_paths(self):
296         return [self._filesystem.join(self._webkit_baseline_path(self._name), 'Skipped')]
297
298     def _expectations_from_skipped_files(self):
299         tests_to_skip = []
300         for filename in self._skipped_file_paths():
301             if not self._filesystem.exists(filename):
302                 _log.warn("Failed to open Skipped file: %s" % filename)
303                 continue
304             skipped_file_contents = self._filesystem.read_text_file(filename)
305             tests_to_skip.extend(self._tests_from_skipped_file_contents(skipped_file_contents))
306         return tests_to_skip
307
308     def test_expectations(self):
309         # The WebKit mac port uses a combination of a test_expectations file
310         # and 'Skipped' files.
311         expectations_path = self.path_to_test_expectations_file()
312         return self._filesystem.read_text_file(expectations_path) + self._skips()
313
314     def _skips(self):
315         # Each Skipped file contains a list of files
316         # or directories to be skipped during the test run. The total list
317         # of tests to skipped is given by the contents of the generic
318         # Skipped file found in platform/X plus a version-specific file
319         # found in platform/X-version. Duplicate entries are allowed.
320         # This routine reads those files and turns contents into the
321         # format expected by test_expectations.
322
323         tests_to_skip = self.skipped_layout_tests()
324         skip_lines = map(lambda test_path: "BUG_SKIPPED SKIP : %s = FAIL" %
325                                 test_path, tests_to_skip)
326         return "\n".join(skip_lines)
327
328     def skipped_layout_tests(self):
329         # Use a set to allow duplicates
330         tests_to_skip = set(self._expectations_from_skipped_files())
331         tests_to_skip.update(self._tests_for_other_platforms())
332         tests_to_skip.update(self._tests_for_disabled_features())
333         return tests_to_skip
334
335     def test_platform_name(self):
336         return self._name + self.version()
337
338     def test_platform_names(self):
339         return self.test_base_platform_names() + (
340             'mac-tiger', 'mac-leopard', 'mac-snowleopard')
341
342     def _build_path(self, *comps):
343         return self._filesystem.join(self._config.build_directory(
344             self.get_option('configuration')), *comps)
345
346     def _path_to_driver(self):
347         return self._build_path('DumpRenderTree')
348
349     def _path_to_webcore_library(self):
350         return None
351
352     def _path_to_helper(self):
353         return None
354
355     def _path_to_image_diff(self):
356         return self._build_path('ImageDiff')
357
358     def _path_to_wdiff(self):
359         # FIXME: This does not exist on a default Mac OS X Leopard install.
360         return 'wdiff'
361
362     def _path_to_apache(self):
363         if not self._cached_apache_path:
364             # The Apache binary path can vary depending on OS and distribution
365             # See http://wiki.apache.org/httpd/DistrosDefaultLayout
366             for path in ["/usr/sbin/httpd", "/usr/sbin/apache2"]:
367                 if self._filesystem.exists(path):
368                     self._cached_apache_path = path
369                     break
370
371             if not self._cached_apache_path:
372                 _log.error("Could not find apache. Not installed or unknown path.")
373
374         return self._cached_apache_path
375
376
377 class WebKitDriver(base.Driver):
378     """WebKit implementation of the DumpRenderTree interface."""
379
380     def __init__(self, port, worker_number):
381         self._worker_number = worker_number
382         self._port = port
383         self._driver_tempdir = port._filesystem.mkdtemp(prefix='DumpRenderTree-')
384
385     def __del__(self):
386         self._port._filesystem.rmtree(str(self._driver_tempdir))
387
388     def cmd_line(self):
389         cmd = self._command_wrapper(self._port.get_option('wrapper'))
390         cmd += [self._port._path_to_driver(), '-']
391
392         if self._port.get_option('pixel_tests'):
393             cmd.append('--pixel-tests')
394
395         return cmd
396
397     def start(self):
398         environment = self._port.setup_environ_for_server()
399         environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path()
400         environment['DUMPRENDERTREE_TEMP'] = str(self._driver_tempdir)
401         self._server_process = server_process.ServerProcess(self._port,
402             "DumpRenderTree", self.cmd_line(), environment)
403
404     def poll(self):
405         return self._server_process.poll()
406
407     def restart(self):
408         self._server_process.stop()
409         self._server_process.start()
410         return
411
412     # FIXME: This function is huge.
413     def run_test(self, test_input):
414         uri = self._port.filename_to_uri(test_input.filename)
415         if uri.startswith("file:///"):
416             command = uri[7:]
417         else:
418             command = uri
419
420         if test_input.image_hash:
421             command += "'" + test_input.image_hash
422         command += "\n"
423
424         start_time = time.time()
425         self._server_process.write(command)
426
427         have_seen_content_type = False
428         actual_image_hash = None
429         output = str()  # Use a byte array for output, even though it should be UTF-8.
430         image = str()
431
432         timeout = int(test_input.timeout) / 1000.0
433         deadline = time.time() + timeout
434         line = self._server_process.read_line(timeout)
435         while (not self._server_process.timed_out
436                and not self._server_process.crashed
437                and line.rstrip() != "#EOF"):
438             if (line.startswith('Content-Type:') and not
439                 have_seen_content_type):
440                 have_seen_content_type = True
441             else:
442                 # Note: Text output from DumpRenderTree is always UTF-8.
443                 # However, some tests (e.g. webarchives) spit out binary
444                 # data instead of text.  So to make things simple, we
445                 # always treat the output as binary.
446                 output += line
447             line = self._server_process.read_line(timeout)
448             timeout = deadline - time.time()
449
450         # Now read a second block of text for the optional image data
451         remaining_length = -1
452         HASH_HEADER = 'ActualHash: '
453         LENGTH_HEADER = 'Content-Length: '
454         line = self._server_process.read_line(timeout)
455         while (not self._server_process.timed_out
456                and not self._server_process.crashed
457                and line.rstrip() != "#EOF"):
458             if line.startswith(HASH_HEADER):
459                 actual_image_hash = line[len(HASH_HEADER):].strip()
460             elif line.startswith('Content-Type:'):
461                 pass
462             elif line.startswith(LENGTH_HEADER):
463                 timeout = deadline - time.time()
464                 content_length = int(line[len(LENGTH_HEADER):])
465                 image = self._server_process.read(timeout, content_length)
466             timeout = deadline - time.time()
467             line = self._server_process.read_line(timeout)
468
469         error_lines = self._server_process.error.splitlines()
470         # FIXME: This is a hack.  It is unclear why sometimes
471         # we do not get any error lines from the server_process
472         # probably we are not flushing stderr.
473         if error_lines and error_lines[-1] == "#EOF":
474             error_lines.pop()  # Remove the expected "#EOF"
475         error = "\n".join(error_lines)
476         # FIXME: This seems like the wrong section of code to be doing
477         # this reset in.
478         self._server_process.error = ""
479         return test_output.TestOutput(output, image, actual_image_hash,
480                                       self._server_process.crashed,
481                                       time.time() - start_time,
482                                       self._server_process.timed_out,
483                                       error)
484
485     def stop(self):
486         if self._server_process:
487             self._server_process.stop()
488             self._server_process = None