577acd45e983cc1c4cd8ad17da179f3db04ad8ef
[WebKit.git] / Tools / Scripts / webkitpy / layout_tests / port / webkit.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the Google name nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """WebKit implementations of the Port interface."""
32
33
34 import logging
35 import operator
36 import os
37 import re
38 import signal
39 import sys
40 import time
41 import webbrowser
42
43 import webkitpy.common.system.ospath as ospath
44 import webkitpy.layout_tests.layout_package.test_output as test_output
45 import webkitpy.layout_tests.port.base as base
46 import webkitpy.layout_tests.port.server_process as server_process
47
48 _log = logging.getLogger("webkitpy.layout_tests.port.webkit")
49
50
51 class WebKitPort(base.Port):
52     """WebKit implementation of the Port class."""
53
54     def __init__(self, **kwargs):
55         base.Port.__init__(self, **kwargs)
56         self._cached_apache_path = None
57
58         # FIXME: disable pixel tests until they are run by default on the
59         # build machines.
60         self.set_option_default('pixel_tests', False)
61
62     def baseline_path(self):
63         return self._webkit_baseline_path(self._name)
64
65     def baseline_search_path(self):
66         return [self._webkit_baseline_path(self._name)]
67
68     def path_to_test_expectations_file(self):
69         return self._filesystem.join(self._webkit_baseline_path(self._name),
70                                      'test_expectations.txt')
71
72     # Only needed by ports which maintain versioned test expectations (like mac-tiger vs. mac-leopard)
73     def version(self):
74         return ''
75
76     def _build_driver(self):
77         configuration = self.get_option('configuration')
78         return self._config.build_dumprendertree(configuration)
79
80     def _check_driver(self):
81         driver_path = self._path_to_driver()
82         if not self._filesystem.exists(driver_path):
83             _log.error("DumpRenderTree was not found at %s" % driver_path)
84             return False
85         return True
86
87     def check_build(self, needs_http):
88         if self.get_option('build') and not self._build_driver():
89             return False
90         if not self._check_driver():
91             return False
92         if self.get_option('pixel_tests'):
93             if not self.check_image_diff():
94                 return False
95         if not self._check_port_build():
96             return False
97         return True
98
99     def _check_port_build(self):
100         # Ports can override this method to do additional checks.
101         return True
102
103     def check_image_diff(self, override_step=None, logging=True):
104         image_diff_path = self._path_to_image_diff()
105         if not self._filesystem.exists(image_diff_path):
106             _log.error("ImageDiff was not found at %s" % image_diff_path)
107             return False
108         return True
109
110     def diff_image(self, expected_contents, actual_contents,
111                    diff_filename=None):
112         """Return True if the two files are different. Also write a delta
113         image of the two images into |diff_filename| if it is not None."""
114
115         # Handle the case where the test didn't actually generate an image.
116         if not actual_contents:
117             return True
118
119         sp = self._diff_image_request(expected_contents, actual_contents)
120         return self._diff_image_reply(sp, diff_filename)
121
122     def _diff_image_request(self, expected_contents, actual_contents):
123         # FIXME: use self.get_option('tolerance') and
124         # self.set_option_default('tolerance', 0.1) once that behaves correctly
125         # with default values.
126         if self.get_option('tolerance') is not None:
127             tolerance = self.get_option('tolerance')
128         else:
129             tolerance = 0.1
130         command = [self._path_to_image_diff(), '--tolerance', str(tolerance)]
131         sp = server_process.ServerProcess(self, 'ImageDiff', command)
132
133         sp.write('Content-Length: %d\n%sContent-Length: %d\n%s' %
134                  (len(actual_contents), actual_contents,
135                   len(expected_contents), expected_contents))
136
137         return sp
138
139     def _diff_image_reply(self, sp, diff_filename):
140         timeout = 2.0
141         deadline = time.time() + timeout
142         output = sp.read_line(timeout)
143         while not sp.timed_out and not sp.crashed and output:
144             if output.startswith('Content-Length'):
145                 m = re.match('Content-Length: (\d+)', output)
146                 content_length = int(m.group(1))
147                 timeout = deadline - time.time()
148                 output = sp.read(timeout, content_length)
149                 break
150             elif output.startswith('diff'):
151                 break
152             else:
153                 timeout = deadline - time.time()
154                 output = sp.read_line(deadline)
155
156         result = True
157         if output.startswith('diff'):
158             m = re.match('diff: (.+)% (passed|failed)', output)
159             if m.group(2) == 'passed':
160                 result = False
161         elif output and diff_filename:
162             self._filesystem.write_text_file(diff_filename, output)
163         elif sp.timed_out:
164             _log.error("ImageDiff timed out")
165         elif sp.crashed:
166             _log.error("ImageDiff crashed")
167         sp.stop()
168         return result
169
170     def results_directory(self):
171         # Results are store relative to the built products to make it easy
172         # to have multiple copies of webkit checked out and built.
173         return self._build_path(self.get_option('results_directory'))
174
175     def setup_test_run(self):
176         # This port doesn't require any specific configuration.
177         pass
178
179     def create_driver(self, worker_number):
180         return WebKitDriver(self, worker_number)
181
182     def test_base_platform_names(self):
183         # At the moment we don't use test platform names, but we have
184         # to return something.
185         return ('mac', 'win')
186
187     def _tests_for_other_platforms(self):
188         raise NotImplementedError('WebKitPort._tests_for_other_platforms')
189         # The original run-webkit-tests builds up a "whitelist" of tests to
190         # run, and passes that to DumpRenderTree. new-run-webkit-tests assumes
191         # we run *all* tests and test_expectations.txt functions as a
192         # blacklist.
193         # FIXME: This list could be dynamic based on platform name and
194         # pushed into base.Port.
195         return [
196             "platform/chromium",
197             "platform/gtk",
198             "platform/qt",
199             "platform/win",
200         ]
201
202     def _runtime_feature_list(self):
203         """Return the supported features of DRT. If a port doesn't support
204         this DRT switch, it has to override this method to return None"""
205         driver_path = self._path_to_driver()
206         feature_list = ' '.join(os.popen(driver_path + " --print-supported-features 2>&1").readlines())
207         if "SupportedFeatures:" in feature_list:
208             return feature_list
209         return None
210
211     def _supported_symbol_list(self):
212         """Return the supported symbols of WebCore."""
213         webcore_library_path = self._path_to_webcore_library()
214         if not webcore_library_path:
215             return None
216         symbol_list = ' '.join(os.popen("nm " + webcore_library_path).readlines())
217         return symbol_list
218
219     def _directories_for_features(self):
220         """Return the supported feature dictionary. The keys are the
221         features and the values are the directories in lists."""
222         directories_for_features = {
223             "Accelerated Compositing": ["compositing"],
224             "3D Rendering": ["animations/3d", "transforms/3d"],
225         }
226         return directories_for_features
227
228     def _directories_for_symbols(self):
229         """Return the supported feature dictionary. The keys are the
230         symbols and the values are the directories in lists."""
231         directories_for_symbol = {
232             "MathMLElement": ["mathml"],
233             "GraphicsLayer": ["compositing"],
234             "WebCoreHas3DRendering": ["animations/3d", "transforms/3d"],
235             "WebGLShader": ["fast/canvas/webgl", "compositing/webgl", "http/tests/canvas/webgl"],
236             "WMLElement": ["http/tests/wml", "fast/wml", "wml"],
237             "parseWCSSInputProperty": ["fast/wcss"],
238             "isXHTMLMPDocument": ["fast/xhtmlmp"],
239         }
240         return directories_for_symbol
241
242     def _skipped_tests_for_unsupported_features(self):
243         """Return the directories of unsupported tests. Search for the
244         symbols in the symbol_list, if found add the corresponding
245         directories to the skipped directory list."""
246         feature_list = self._runtime_feature_list()
247         directories = self._directories_for_features()
248
249         # if DRT feature detection not supported
250         if not feature_list:
251             feature_list = self._supported_symbol_list()
252             directories = self._directories_for_symbols()
253
254         if not feature_list:
255             return []
256
257         skipped_directories = [directories[feature]
258                               for feature in directories.keys()
259                               if feature not in feature_list]
260         return reduce(operator.add, skipped_directories)
261
262     def _tests_for_disabled_features(self):
263         # FIXME: This should use the feature detection from
264         # webkitperl/features.pm to match run-webkit-tests.
265         # For now we hard-code a list of features known to be disabled on
266         # the Mac platform.
267         disabled_feature_tests = [
268             "fast/xhtmlmp",
269             "http/tests/wml",
270             "mathml",
271             "wml",
272         ]
273         # FIXME: webarchive tests expect to read-write from
274         # -expected.webarchive files instead of .txt files.
275         # This script doesn't know how to do that yet, so pretend they're
276         # just "disabled".
277         webarchive_tests = [
278             "webarchive",
279             "svg/webarchive",
280             "http/tests/webarchive",
281             "svg/custom/image-with-prefix-in-webarchive.svg",
282         ]
283         unsupported_feature_tests = self._skipped_tests_for_unsupported_features()
284         return disabled_feature_tests + webarchive_tests + unsupported_feature_tests
285
286     def _tests_from_skipped_file(self, skipped_file):
287         tests_to_skip = []
288         for line in skipped_file.readlines():
289             line = line.strip()
290             if line.startswith('#') or not len(line):
291                 continue
292             tests_to_skip.append(line)
293         return tests_to_skip
294
295     def _skipped_file_paths(self):
296         return [self._filesystem.join(self._webkit_baseline_path(self._name), 'Skipped')]
297
298     def _expectations_from_skipped_files(self):
299         tests_to_skip = []
300         for filename in self._skipped_file_paths():
301             if not self._filesystem.exists(filename):
302                 _log.warn("Failed to open Skipped file: %s" % filename)
303                 continue
304             skipped_file = self._filesystem.read_text_file(filename)
305         return tests_to_skip
306
307     def test_expectations(self):
308         # The WebKit mac port uses a combination of a test_expectations file
309         # and 'Skipped' files.
310         expectations_path = self.path_to_test_expectations_file()
311         return self._filesystem.read_text_file(expectations_path) + self._skips()
312
313     def _skips(self):
314         # Each Skipped file contains a list of files
315         # or directories to be skipped during the test run. The total list
316         # of tests to skipped is given by the contents of the generic
317         # Skipped file found in platform/X plus a version-specific file
318         # found in platform/X-version. Duplicate entries are allowed.
319         # This routine reads those files and turns contents into the
320         # format expected by test_expectations.
321
322         tests_to_skip = self.skipped_layout_tests()
323         skip_lines = map(lambda test_path: "BUG_SKIPPED SKIP : %s = FAIL" %
324                                 test_path, tests_to_skip)
325         return "\n".join(skip_lines)
326
327     def skipped_layout_tests(self):
328         # Use a set to allow duplicates
329         tests_to_skip = set(self._expectations_from_skipped_files())
330         tests_to_skip.update(self._tests_for_other_platforms())
331         tests_to_skip.update(self._tests_for_disabled_features())
332         return tests_to_skip
333
334     def test_platform_name(self):
335         return self._name + self.version()
336
337     def test_platform_names(self):
338         return self.test_base_platform_names() + (
339             'mac-tiger', 'mac-leopard', 'mac-snowleopard')
340
341     def _build_path(self, *comps):
342         return self._filesystem.join(self._config.build_directory(
343             self.get_option('configuration')), *comps)
344
345     def _path_to_driver(self):
346         return self._build_path('DumpRenderTree')
347
348     def _path_to_webcore_library(self):
349         return None
350
351     def _path_to_helper(self):
352         return None
353
354     def _path_to_image_diff(self):
355         return self._build_path('ImageDiff')
356
357     def _path_to_wdiff(self):
358         # FIXME: This does not exist on a default Mac OS X Leopard install.
359         return 'wdiff'
360
361     def _path_to_apache(self):
362         if not self._cached_apache_path:
363             # The Apache binary path can vary depending on OS and distribution
364             # See http://wiki.apache.org/httpd/DistrosDefaultLayout
365             for path in ["/usr/sbin/httpd", "/usr/sbin/apache2"]:
366                 if self._filesystem.exists(path):
367                     self._cached_apache_path = path
368                     break
369
370             if not self._cached_apache_path:
371                 _log.error("Could not find apache. Not installed or unknown path.")
372
373         return self._cached_apache_path
374
375
376 class WebKitDriver(base.Driver):
377     """WebKit implementation of the DumpRenderTree interface."""
378
379     def __init__(self, port, worker_number):
380         self._worker_number = worker_number
381         self._port = port
382         self._driver_tempdir = port._filesystem.mkdtemp(prefix='DumpRenderTree-')
383
384     def __del__(self):
385         self._port._filesystem.rmtree(str(self._driver_tempdir))
386
387     def cmd_line(self):
388         cmd = self._command_wrapper(self._port.get_option('wrapper'))
389         cmd += [self._port._path_to_driver(), '-']
390
391         if self._port.get_option('pixel_tests'):
392             cmd.append('--pixel-tests')
393
394         return cmd
395
396     def start(self):
397         environment = self._port.setup_environ_for_server()
398         environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path()
399         environment['DUMPRENDERTREE_TEMP'] = str(self._driver_tempdir)
400         self._server_process = server_process.ServerProcess(self._port,
401             "DumpRenderTree", self.cmd_line(), environment)
402
403     def poll(self):
404         return self._server_process.poll()
405
406     def restart(self):
407         self._server_process.stop()
408         self._server_process.start()
409         return
410
411     # FIXME: This function is huge.
412     def run_test(self, test_input):
413         uri = self._port.filename_to_uri(test_input.filename)
414         if uri.startswith("file:///"):
415             command = uri[7:]
416         else:
417             command = uri
418
419         if test_input.image_hash:
420             command += "'" + test_input.image_hash
421         command += "\n"
422
423         start_time = time.time()
424         self._server_process.write(command)
425
426         have_seen_content_type = False
427         actual_image_hash = None
428         output = str()  # Use a byte array for output, even though it should be UTF-8.
429         image = str()
430
431         timeout = int(test_input.timeout) / 1000.0
432         deadline = time.time() + timeout
433         line = self._server_process.read_line(timeout)
434         while (not self._server_process.timed_out
435                and not self._server_process.crashed
436                and line.rstrip() != "#EOF"):
437             if (line.startswith('Content-Type:') and not
438                 have_seen_content_type):
439                 have_seen_content_type = True
440             else:
441                 # Note: Text output from DumpRenderTree is always UTF-8.
442                 # However, some tests (e.g. webarchives) spit out binary
443                 # data instead of text.  So to make things simple, we
444                 # always treat the output as binary.
445                 output += line
446             line = self._server_process.read_line(timeout)
447             timeout = deadline - time.time()
448
449         # Now read a second block of text for the optional image data
450         remaining_length = -1
451         HASH_HEADER = 'ActualHash: '
452         LENGTH_HEADER = 'Content-Length: '
453         line = self._server_process.read_line(timeout)
454         while (not self._server_process.timed_out
455                and not self._server_process.crashed
456                and line.rstrip() != "#EOF"):
457             if line.startswith(HASH_HEADER):
458                 actual_image_hash = line[len(HASH_HEADER):].strip()
459             elif line.startswith('Content-Type:'):
460                 pass
461             elif line.startswith(LENGTH_HEADER):
462                 timeout = deadline - time.time()
463                 content_length = int(line[len(LENGTH_HEADER):])
464                 image = self._server_process.read(timeout, content_length)
465             timeout = deadline - time.time()
466             line = self._server_process.read_line(timeout)
467
468         error_lines = self._server_process.error.splitlines()
469         # FIXME: This is a hack.  It is unclear why sometimes
470         # we do not get any error lines from the server_process
471         # probably we are not flushing stderr.
472         if error_lines and error_lines[-1] == "#EOF":
473             error_lines.pop()  # Remove the expected "#EOF"
474         error = "\n".join(error_lines)
475         # FIXME: This seems like the wrong section of code to be doing
476         # this reset in.
477         self._server_process.error = ""
478         return test_output.TestOutput(output, image, actual_image_hash,
479                                       self._server_process.crashed,
480                                       time.time() - start_time,
481                                       self._server_process.timed_out,
482                                       error)
483
484     def stop(self):
485         if self._server_process:
486             self._server_process.stop()
487             self._server_process = None