51f57c40e666167b18cdf52d64201875965247e8
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / layout_tests / port / webkit.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the Google name nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """WebKit implementations of the Port interface."""
32
33
34 from __future__ import with_statement
35
36 import codecs
37 import logging
38 import os
39 import re
40 import shutil
41 import signal
42 import sys
43 import time
44 import webbrowser
45 import operator
46 import tempfile
47 import shutil
48
49 from webkitpy.common.system.executive import Executive
50
51 import webkitpy.common.system.ospath as ospath
52 import webkitpy.layout_tests.port.base as base
53 import webkitpy.layout_tests.port.server_process as server_process
54
55 _log = logging.getLogger("webkitpy.layout_tests.port.webkit")
56
57
58 class WebKitPort(base.Port):
59     """WebKit implementation of the Port class."""
60
61     def __init__(self, **kwargs):
62         base.Port.__init__(self, **kwargs)
63         self._cached_apache_path = None
64
65         # FIXME: disable pixel tests until they are run by default on the
66         # build machines.
67         self.set_option_default('pixel_tests', False)
68
69     def baseline_path(self):
70         return self._webkit_baseline_path(self._name)
71
72     def baseline_search_path(self):
73         return [self._webkit_baseline_path(self._name)]
74
75     def path_to_test_expectations_file(self):
76         return os.path.join(self._webkit_baseline_path(self._name),
77                             'test_expectations.txt')
78
79     # Only needed by ports which maintain versioned test expectations (like mac-tiger vs. mac-leopard)
80     def version(self):
81         return ''
82
83     def _build_driver(self):
84         configuration = self.get_option('configuration')
85         return self._config.build_dumprendertree(configuration)
86
87     def _check_driver(self):
88         driver_path = self._path_to_driver()
89         if not os.path.exists(driver_path):
90             _log.error("DumpRenderTree was not found at %s" % driver_path)
91             return False
92         return True
93
94     def check_build(self, needs_http):
95         if self.get_option('build') and not self._build_driver():
96             return False
97         if not self._check_driver():
98             return False
99         if self.get_option('pixel_tests'):
100             if not self.check_image_diff():
101                 return False
102         if not self._check_port_build():
103             return False
104         return True
105
106     def _check_port_build(self):
107         # Ports can override this method to do additional checks.
108         return True
109
110     def check_image_diff(self, override_step=None, logging=True):
111         image_diff_path = self._path_to_image_diff()
112         if not os.path.exists(image_diff_path):
113             _log.error("ImageDiff was not found at %s" % image_diff_path)
114             return False
115         return True
116
117     def diff_image(self, expected_contents, actual_contents,
118                    diff_filename=None):
119         """Return True if the two files are different. Also write a delta
120         image of the two images into |diff_filename| if it is not None."""
121
122         # Handle the case where the test didn't actually generate an image.
123         if not actual_contents:
124             return True
125
126         sp = self._diff_image_request(expected_contents, actual_contents)
127         return self._diff_image_reply(sp, diff_filename)
128
129     def _diff_image_request(self, expected_contents, actual_contents):
130         # FIXME: use self.get_option('tolerance') and
131         # self.set_option_default('tolerance', 0.1) once that behaves correctly
132         # with default values.
133         if self.get_option('tolerance') is not None:
134             tolerance = self.get_option('tolerance')
135         else:
136             tolerance = 0.1
137         command = [self._path_to_image_diff(), '--tolerance', str(tolerance)]
138         sp = server_process.ServerProcess(self, 'ImageDiff', command)
139
140         sp.write('Content-Length: %d\n%sContent-Length: %d\n%s' %
141                  (len(actual_contents), actual_contents,
142                   len(expected_contents), expected_contents))
143
144         return sp
145
146     def _diff_image_reply(self, sp, diff_filename):
147         timeout = 2.0
148         deadline = time.time() + timeout
149         output = sp.read_line(timeout)
150         while not sp.timed_out and not sp.crashed and output:
151             if output.startswith('Content-Length'):
152                 m = re.match('Content-Length: (\d+)', output)
153                 content_length = int(m.group(1))
154                 timeout = deadline - time.time()
155                 output = sp.read(timeout, content_length)
156                 break
157             elif output.startswith('diff'):
158                 break
159             else:
160                 timeout = deadline - time.time()
161                 output = sp.read_line(deadline)
162
163         result = True
164         if output.startswith('diff'):
165             m = re.match('diff: (.+)% (passed|failed)', output)
166             if m.group(2) == 'passed':
167                 result = False
168         elif output and diff_filename:
169             with open(diff_filename, 'w') as file:
170                 file.write(output)
171         elif sp.timed_out:
172             _log.error("ImageDiff timed out")
173         elif sp.crashed:
174             _log.error("ImageDiff crashed")
175         sp.stop()
176         return result
177
178     def results_directory(self):
179         # Results are store relative to the built products to make it easy
180         # to have multiple copies of webkit checked out and built.
181         return self._build_path(self.get_option('results_directory'))
182
183     def setup_test_run(self):
184         # This port doesn't require any specific configuration.
185         pass
186
187     def create_driver(self, image_path, options):
188         return WebKitDriver(self, image_path, options,
189                             executive=self._executive)
190
191     def test_base_platform_names(self):
192         # At the moment we don't use test platform names, but we have
193         # to return something.
194         return ('mac', 'win')
195
196     def _tests_for_other_platforms(self):
197         raise NotImplementedError('WebKitPort._tests_for_other_platforms')
198         # The original run-webkit-tests builds up a "whitelist" of tests to
199         # run, and passes that to DumpRenderTree. new-run-webkit-tests assumes
200         # we run *all* tests and test_expectations.txt functions as a
201         # blacklist.
202         # FIXME: This list could be dynamic based on platform name and
203         # pushed into base.Port.
204         return [
205             "platform/chromium",
206             "platform/gtk",
207             "platform/qt",
208             "platform/win",
209         ]
210
211     def _runtime_feature_list(self):
212         """Return the supported features of DRT. If a port doesn't support
213         this DRT switch, it has to override this method to return None"""
214         driver_path = self._path_to_driver()
215         feature_list = ' '.join(os.popen(driver_path + " --print-supported-features 2>&1").readlines())
216         if "SupportedFeatures:" in feature_list:
217             return feature_list
218         return None
219
220     def _supported_symbol_list(self):
221         """Return the supported symbols of WebCore."""
222         webcore_library_path = self._path_to_webcore_library()
223         if not webcore_library_path:
224             return None
225         symbol_list = ' '.join(os.popen("nm " + webcore_library_path).readlines())
226         return symbol_list
227
228     def _directories_for_features(self):
229         """Return the supported feature dictionary. The keys are the
230         features and the values are the directories in lists."""
231         directories_for_features = {
232             "Accelerated Compositing": ["compositing"],
233             "3D Rendering": ["animations/3d", "transforms/3d"],
234         }
235         return directories_for_features
236
237     def _directories_for_symbols(self):
238         """Return the supported feature dictionary. The keys are the
239         symbols and the values are the directories in lists."""
240         directories_for_symbol = {
241             "MathMLElement": ["mathml"],
242             "GraphicsLayer": ["compositing"],
243             "WebCoreHas3DRendering": ["animations/3d", "transforms/3d"],
244             "WebGLShader": ["fast/canvas/webgl", "compositing/webgl", "http/tests/canvas/webgl"],
245             "WMLElement": ["http/tests/wml", "fast/wml", "wml"],
246             "parseWCSSInputProperty": ["fast/wcss"],
247             "isXHTMLMPDocument": ["fast/xhtmlmp"],
248         }
249         return directories_for_symbol
250
251     def _skipped_tests_for_unsupported_features(self):
252         """Return the directories of unsupported tests. Search for the
253         symbols in the symbol_list, if found add the corresponding
254         directories to the skipped directory list."""
255         feature_list = self._runtime_feature_list()
256         directories = self._directories_for_features()
257
258         # if DRT feature detection not supported
259         if not feature_list:
260             feature_list = self._supported_symbol_list()
261             directories = self._directories_for_symbols()
262
263         if not feature_list:
264             return []
265
266         skipped_directories = [directories[feature]
267                               for feature in directories.keys()
268                               if feature not in feature_list]
269         return reduce(operator.add, skipped_directories)
270
271     def _tests_for_disabled_features(self):
272         # FIXME: This should use the feature detection from
273         # webkitperl/features.pm to match run-webkit-tests.
274         # For now we hard-code a list of features known to be disabled on
275         # the Mac platform.
276         disabled_feature_tests = [
277             "fast/xhtmlmp",
278             "http/tests/wml",
279             "mathml",
280             "wml",
281         ]
282         # FIXME: webarchive tests expect to read-write from
283         # -expected.webarchive files instead of .txt files.
284         # This script doesn't know how to do that yet, so pretend they're
285         # just "disabled".
286         webarchive_tests = [
287             "webarchive",
288             "svg/webarchive",
289             "http/tests/webarchive",
290             "svg/custom/image-with-prefix-in-webarchive.svg",
291         ]
292         unsupported_feature_tests = self._skipped_tests_for_unsupported_features()
293         return disabled_feature_tests + webarchive_tests + unsupported_feature_tests
294
295     def _tests_from_skipped_file(self, skipped_file):
296         tests_to_skip = []
297         for line in skipped_file.readlines():
298             line = line.strip()
299             if line.startswith('#') or not len(line):
300                 continue
301             tests_to_skip.append(line)
302         return tests_to_skip
303
304     def _skipped_file_paths(self):
305         return [os.path.join(self._webkit_baseline_path(self._name),
306                                                         'Skipped')]
307
308     def _expectations_from_skipped_files(self):
309         tests_to_skip = []
310         for filename in self._skipped_file_paths():
311             if not os.path.exists(filename):
312                 _log.warn("Failed to open Skipped file: %s" % filename)
313                 continue
314             with codecs.open(filename, "r", "utf-8") as skipped_file:
315                 tests_to_skip.extend(self._tests_from_skipped_file(skipped_file))
316         return tests_to_skip
317
318     def test_expectations(self):
319         # The WebKit mac port uses a combination of a test_expectations file
320         # and 'Skipped' files.
321         expectations_path = self.path_to_test_expectations_file()
322         with codecs.open(expectations_path, "r", "utf-8") as file:
323             return file.read() + self._skips()
324
325     def _skips(self):
326         # Each Skipped file contains a list of files
327         # or directories to be skipped during the test run. The total list
328         # of tests to skipped is given by the contents of the generic
329         # Skipped file found in platform/X plus a version-specific file
330         # found in platform/X-version. Duplicate entries are allowed.
331         # This routine reads those files and turns contents into the
332         # format expected by test_expectations.
333
334         tests_to_skip = self.skipped_layout_tests()
335         skip_lines = map(lambda test_path: "BUG_SKIPPED SKIP : %s = FAIL" %
336                                 test_path, tests_to_skip)
337         return "\n".join(skip_lines)
338
339     def skipped_layout_tests(self):
340         # Use a set to allow duplicates
341         tests_to_skip = set(self._expectations_from_skipped_files())
342         tests_to_skip.update(self._tests_for_other_platforms())
343         tests_to_skip.update(self._tests_for_disabled_features())
344         return tests_to_skip
345
346     def test_platform_name(self):
347         return self._name + self.version()
348
349     def test_platform_names(self):
350         return self.test_base_platform_names() + (
351             'mac-tiger', 'mac-leopard', 'mac-snowleopard')
352
353     def _build_path(self, *comps):
354         return self._filesystem.join(self._config.build_directory(
355             self.get_option('configuration')), *comps)
356
357     def _path_to_driver(self):
358         return self._build_path('DumpRenderTree')
359
360     def _path_to_webcore_library(self):
361         return None
362
363     def _path_to_helper(self):
364         return None
365
366     def _path_to_image_diff(self):
367         return self._build_path('ImageDiff')
368
369     def _path_to_wdiff(self):
370         # FIXME: This does not exist on a default Mac OS X Leopard install.
371         return 'wdiff'
372
373     def _path_to_apache(self):
374         if not self._cached_apache_path:
375             # The Apache binary path can vary depending on OS and distribution
376             # See http://wiki.apache.org/httpd/DistrosDefaultLayout
377             for path in ["/usr/sbin/httpd", "/usr/sbin/apache2"]:
378                 if os.path.exists(path):
379                     self._cached_apache_path = path
380                     break
381
382             if not self._cached_apache_path:
383                 _log.error("Could not find apache. Not installed or unknown path.")
384
385         return self._cached_apache_path
386
387
388 class WebKitDriver(base.Driver):
389     """WebKit implementation of the DumpRenderTree interface."""
390
391     def __init__(self, port, image_path, options, executive=Executive()):
392         self._port = port
393         self._image_path = image_path
394         self._executive = executive
395         self._driver_tempdir = tempfile.mkdtemp(prefix='DumpRenderTree-')
396
397     def __del__(self):
398         shutil.rmtree(self._driver_tempdir)
399
400     def _driver_args(self):
401         driver_args = []
402
403         if self._image_path:
404             driver_args.append('--pixel-tests')
405
406         if self._port.get_option('use_drt'):
407             if self._port.get_option('accelerated_compositing'):
408                 driver_args.append('--enable-accelerated-compositing')
409
410             if self._port.get_option('accelerated_2d_canvas'):
411                 driver_args.append('--enable-accelerated-2d-canvas')
412
413         return driver_args
414
415     def start(self):
416         command = self._command_wrapper(self._port.get_option('wrapper'))
417         command += [self._port._path_to_driver(), '-']
418         command += self._driver_args()
419
420         environment = self._port.setup_environ_for_server()
421         environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path()
422         environment['DUMPRENDERTREE_TEMP'] = self._driver_tempdir
423         self._server_process = server_process.ServerProcess(self._port,
424             "DumpRenderTree", command, environment)
425
426     def poll(self):
427         return self._server_process.poll()
428
429     def restart(self):
430         self._server_process.stop()
431         self._server_process.start()
432         return
433
434     # FIXME: This function is huge.
435     def run_test(self, uri, timeoutms, image_hash):
436         if uri.startswith("file:///"):
437             command = uri[7:]
438         else:
439             command = uri
440
441         if image_hash:
442             command += "'" + image_hash
443         command += "\n"
444
445         self._server_process.write(command)
446
447         have_seen_content_type = False
448         actual_image_hash = None
449         output = str()  # Use a byte array for output, even though it should be UTF-8.
450         image = str()
451
452         timeout = int(timeoutms) / 1000.0
453         deadline = time.time() + timeout
454         line = self._server_process.read_line(timeout)
455         while (not self._server_process.timed_out
456                and not self._server_process.crashed
457                and line.rstrip() != "#EOF"):
458             if (line.startswith('Content-Type:') and not
459                 have_seen_content_type):
460                 have_seen_content_type = True
461             else:
462                 # Note: Text output from DumpRenderTree is always UTF-8.
463                 # However, some tests (e.g. webarchives) spit out binary
464                 # data instead of text.  So to make things simple, we
465                 # always treat the output as binary.
466                 output += line
467             line = self._server_process.read_line(timeout)
468             timeout = deadline - time.time()
469
470         # Now read a second block of text for the optional image data
471         remaining_length = -1
472         HASH_HEADER = 'ActualHash: '
473         LENGTH_HEADER = 'Content-Length: '
474         line = self._server_process.read_line(timeout)
475         while (not self._server_process.timed_out
476                and not self._server_process.crashed
477                and line.rstrip() != "#EOF"):
478             if line.startswith(HASH_HEADER):
479                 actual_image_hash = line[len(HASH_HEADER):].strip()
480             elif line.startswith('Content-Type:'):
481                 pass
482             elif line.startswith(LENGTH_HEADER):
483                 timeout = deadline - time.time()
484                 content_length = int(line[len(LENGTH_HEADER):])
485                 image = self._server_process.read(timeout, content_length)
486             timeout = deadline - time.time()
487             line = self._server_process.read_line(timeout)
488
489         if self._image_path and len(self._image_path):
490             with open(self._image_path, "wb") as image_file:
491                 image_file.write(image)
492
493         error_lines = self._server_process.error.splitlines()
494         # FIXME: This is a hack.  It is unclear why sometimes
495         # we do not get any error lines from the server_process
496         # probably we are not flushing stderr.
497         if error_lines and error_lines[-1] == "#EOF":
498             error_lines.pop()  # Remove the expected "#EOF"
499         error = "\n".join(error_lines)
500         # FIXME: This seems like the wrong section of code to be doing
501         # this reset in.
502         self._server_process.error = ""
503         return (self._server_process.crashed,
504                 self._server_process.timed_out,
505                 actual_image_hash,
506                 output,
507                 error)
508
509     def stop(self):
510         if self._server_process:
511             self._server_process.stop()
512             self._server_process = None