2010-11-17 Hayato Ito <hayato@chromium.org>
[WebKit-https.git] / WebKitTools / Scripts / webkitpy / layout_tests / port / webkit.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the Google name nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """WebKit implementations of the Port interface."""
32
33
34 from __future__ import with_statement
35
36 import codecs
37 import logging
38 import os
39 import re
40 import shutil
41 import signal
42 import sys
43 import time
44 import webbrowser
45 import operator
46 import tempfile
47 import shutil
48
49 from webkitpy.common.system.executive import Executive
50
51 import webkitpy.common.system.ospath as ospath
52 import webkitpy.layout_tests.layout_package.test_output as test_output
53 import webkitpy.layout_tests.port.base as base
54 import webkitpy.layout_tests.port.server_process as server_process
55
56 _log = logging.getLogger("webkitpy.layout_tests.port.webkit")
57
58
59 class WebKitPort(base.Port):
60     """WebKit implementation of the Port class."""
61
62     def __init__(self, **kwargs):
63         base.Port.__init__(self, **kwargs)
64         self._cached_apache_path = None
65
66         # FIXME: disable pixel tests until they are run by default on the
67         # build machines.
68         self.set_option_default('pixel_tests', False)
69
70     def baseline_path(self):
71         return self._webkit_baseline_path(self._name)
72
73     def baseline_search_path(self):
74         return [self._webkit_baseline_path(self._name)]
75
76     def path_to_test_expectations_file(self):
77         return os.path.join(self._webkit_baseline_path(self._name),
78                             'test_expectations.txt')
79
80     # Only needed by ports which maintain versioned test expectations (like mac-tiger vs. mac-leopard)
81     def version(self):
82         return ''
83
84     def _build_driver(self):
85         configuration = self.get_option('configuration')
86         return self._config.build_dumprendertree(configuration)
87
88     def _check_driver(self):
89         driver_path = self._path_to_driver()
90         if not os.path.exists(driver_path):
91             _log.error("DumpRenderTree was not found at %s" % driver_path)
92             return False
93         return True
94
95     def check_build(self, needs_http):
96         if self.get_option('build') and not self._build_driver():
97             return False
98         if not self._check_driver():
99             return False
100         if self.get_option('pixel_tests'):
101             if not self.check_image_diff():
102                 return False
103         if not self._check_port_build():
104             return False
105         return True
106
107     def _check_port_build(self):
108         # Ports can override this method to do additional checks.
109         return True
110
111     def check_image_diff(self, override_step=None, logging=True):
112         image_diff_path = self._path_to_image_diff()
113         if not os.path.exists(image_diff_path):
114             _log.error("ImageDiff was not found at %s" % image_diff_path)
115             return False
116         return True
117
118     def diff_image(self, expected_contents, actual_contents,
119                    diff_filename=None):
120         """Return True if the two files are different. Also write a delta
121         image of the two images into |diff_filename| if it is not None."""
122
123         # Handle the case where the test didn't actually generate an image.
124         if not actual_contents:
125             return True
126
127         sp = self._diff_image_request(expected_contents, actual_contents)
128         return self._diff_image_reply(sp, diff_filename)
129
130     def _diff_image_request(self, expected_contents, actual_contents):
131         # FIXME: use self.get_option('tolerance') and
132         # self.set_option_default('tolerance', 0.1) once that behaves correctly
133         # with default values.
134         if self.get_option('tolerance') is not None:
135             tolerance = self.get_option('tolerance')
136         else:
137             tolerance = 0.1
138         command = [self._path_to_image_diff(), '--tolerance', str(tolerance)]
139         sp = server_process.ServerProcess(self, 'ImageDiff', command)
140
141         sp.write('Content-Length: %d\n%sContent-Length: %d\n%s' %
142                  (len(actual_contents), actual_contents,
143                   len(expected_contents), expected_contents))
144
145         return sp
146
147     def _diff_image_reply(self, sp, diff_filename):
148         timeout = 2.0
149         deadline = time.time() + timeout
150         output = sp.read_line(timeout)
151         while not sp.timed_out and not sp.crashed and output:
152             if output.startswith('Content-Length'):
153                 m = re.match('Content-Length: (\d+)', output)
154                 content_length = int(m.group(1))
155                 timeout = deadline - time.time()
156                 output = sp.read(timeout, content_length)
157                 break
158             elif output.startswith('diff'):
159                 break
160             else:
161                 timeout = deadline - time.time()
162                 output = sp.read_line(deadline)
163
164         result = True
165         if output.startswith('diff'):
166             m = re.match('diff: (.+)% (passed|failed)', output)
167             if m.group(2) == 'passed':
168                 result = False
169         elif output and diff_filename:
170             with open(diff_filename, 'w') as file:
171                 file.write(output)
172         elif sp.timed_out:
173             _log.error("ImageDiff timed out")
174         elif sp.crashed:
175             _log.error("ImageDiff crashed")
176         sp.stop()
177         return result
178
179     def results_directory(self):
180         # Results are store relative to the built products to make it easy
181         # to have multiple copies of webkit checked out and built.
182         return self._build_path(self.get_option('results_directory'))
183
184     def setup_test_run(self):
185         # This port doesn't require any specific configuration.
186         pass
187
188     def create_driver(self, image_path, options):
189         return WebKitDriver(self, image_path, options,
190                             executive=self._executive)
191
192     def test_base_platform_names(self):
193         # At the moment we don't use test platform names, but we have
194         # to return something.
195         return ('mac', 'win')
196
197     def _tests_for_other_platforms(self):
198         raise NotImplementedError('WebKitPort._tests_for_other_platforms')
199         # The original run-webkit-tests builds up a "whitelist" of tests to
200         # run, and passes that to DumpRenderTree. new-run-webkit-tests assumes
201         # we run *all* tests and test_expectations.txt functions as a
202         # blacklist.
203         # FIXME: This list could be dynamic based on platform name and
204         # pushed into base.Port.
205         return [
206             "platform/chromium",
207             "platform/gtk",
208             "platform/qt",
209             "platform/win",
210         ]
211
212     def _runtime_feature_list(self):
213         """Return the supported features of DRT. If a port doesn't support
214         this DRT switch, it has to override this method to return None"""
215         driver_path = self._path_to_driver()
216         feature_list = ' '.join(os.popen(driver_path + " --print-supported-features 2>&1").readlines())
217         if "SupportedFeatures:" in feature_list:
218             return feature_list
219         return None
220
221     def _supported_symbol_list(self):
222         """Return the supported symbols of WebCore."""
223         webcore_library_path = self._path_to_webcore_library()
224         if not webcore_library_path:
225             return None
226         symbol_list = ' '.join(os.popen("nm " + webcore_library_path).readlines())
227         return symbol_list
228
229     def _directories_for_features(self):
230         """Return the supported feature dictionary. The keys are the
231         features and the values are the directories in lists."""
232         directories_for_features = {
233             "Accelerated Compositing": ["compositing"],
234             "3D Rendering": ["animations/3d", "transforms/3d"],
235         }
236         return directories_for_features
237
238     def _directories_for_symbols(self):
239         """Return the supported feature dictionary. The keys are the
240         symbols and the values are the directories in lists."""
241         directories_for_symbol = {
242             "MathMLElement": ["mathml"],
243             "GraphicsLayer": ["compositing"],
244             "WebCoreHas3DRendering": ["animations/3d", "transforms/3d"],
245             "WebGLShader": ["fast/canvas/webgl", "compositing/webgl", "http/tests/canvas/webgl"],
246             "WMLElement": ["http/tests/wml", "fast/wml", "wml"],
247             "parseWCSSInputProperty": ["fast/wcss"],
248             "isXHTMLMPDocument": ["fast/xhtmlmp"],
249         }
250         return directories_for_symbol
251
252     def _skipped_tests_for_unsupported_features(self):
253         """Return the directories of unsupported tests. Search for the
254         symbols in the symbol_list, if found add the corresponding
255         directories to the skipped directory list."""
256         feature_list = self._runtime_feature_list()
257         directories = self._directories_for_features()
258
259         # if DRT feature detection not supported
260         if not feature_list:
261             feature_list = self._supported_symbol_list()
262             directories = self._directories_for_symbols()
263
264         if not feature_list:
265             return []
266
267         skipped_directories = [directories[feature]
268                               for feature in directories.keys()
269                               if feature not in feature_list]
270         return reduce(operator.add, skipped_directories)
271
272     def _tests_for_disabled_features(self):
273         # FIXME: This should use the feature detection from
274         # webkitperl/features.pm to match run-webkit-tests.
275         # For now we hard-code a list of features known to be disabled on
276         # the Mac platform.
277         disabled_feature_tests = [
278             "fast/xhtmlmp",
279             "http/tests/wml",
280             "mathml",
281             "wml",
282         ]
283         # FIXME: webarchive tests expect to read-write from
284         # -expected.webarchive files instead of .txt files.
285         # This script doesn't know how to do that yet, so pretend they're
286         # just "disabled".
287         webarchive_tests = [
288             "webarchive",
289             "svg/webarchive",
290             "http/tests/webarchive",
291             "svg/custom/image-with-prefix-in-webarchive.svg",
292         ]
293         unsupported_feature_tests = self._skipped_tests_for_unsupported_features()
294         return disabled_feature_tests + webarchive_tests + unsupported_feature_tests
295
296     def _tests_from_skipped_file(self, skipped_file):
297         tests_to_skip = []
298         for line in skipped_file.readlines():
299             line = line.strip()
300             if line.startswith('#') or not len(line):
301                 continue
302             tests_to_skip.append(line)
303         return tests_to_skip
304
305     def _skipped_file_paths(self):
306         return [os.path.join(self._webkit_baseline_path(self._name),
307                                                         'Skipped')]
308
309     def _expectations_from_skipped_files(self):
310         tests_to_skip = []
311         for filename in self._skipped_file_paths():
312             if not os.path.exists(filename):
313                 _log.warn("Failed to open Skipped file: %s" % filename)
314                 continue
315             with codecs.open(filename, "r", "utf-8") as skipped_file:
316                 tests_to_skip.extend(self._tests_from_skipped_file(skipped_file))
317         return tests_to_skip
318
319     def test_expectations(self):
320         # The WebKit mac port uses a combination of a test_expectations file
321         # and 'Skipped' files.
322         expectations_path = self.path_to_test_expectations_file()
323         with codecs.open(expectations_path, "r", "utf-8") as file:
324             return file.read() + self._skips()
325
326     def _skips(self):
327         # Each Skipped file contains a list of files
328         # or directories to be skipped during the test run. The total list
329         # of tests to skipped is given by the contents of the generic
330         # Skipped file found in platform/X plus a version-specific file
331         # found in platform/X-version. Duplicate entries are allowed.
332         # This routine reads those files and turns contents into the
333         # format expected by test_expectations.
334
335         tests_to_skip = self.skipped_layout_tests()
336         skip_lines = map(lambda test_path: "BUG_SKIPPED SKIP : %s = FAIL" %
337                                 test_path, tests_to_skip)
338         return "\n".join(skip_lines)
339
340     def skipped_layout_tests(self):
341         # Use a set to allow duplicates
342         tests_to_skip = set(self._expectations_from_skipped_files())
343         tests_to_skip.update(self._tests_for_other_platforms())
344         tests_to_skip.update(self._tests_for_disabled_features())
345         return tests_to_skip
346
347     def test_platform_name(self):
348         return self._name + self.version()
349
350     def test_platform_names(self):
351         return self.test_base_platform_names() + (
352             'mac-tiger', 'mac-leopard', 'mac-snowleopard')
353
354     def _build_path(self, *comps):
355         return self._filesystem.join(self._config.build_directory(
356             self.get_option('configuration')), *comps)
357
358     def _path_to_driver(self):
359         return self._build_path('DumpRenderTree')
360
361     def _path_to_webcore_library(self):
362         return None
363
364     def _path_to_helper(self):
365         return None
366
367     def _path_to_image_diff(self):
368         return self._build_path('ImageDiff')
369
370     def _path_to_wdiff(self):
371         # FIXME: This does not exist on a default Mac OS X Leopard install.
372         return 'wdiff'
373
374     def _path_to_apache(self):
375         if not self._cached_apache_path:
376             # The Apache binary path can vary depending on OS and distribution
377             # See http://wiki.apache.org/httpd/DistrosDefaultLayout
378             for path in ["/usr/sbin/httpd", "/usr/sbin/apache2"]:
379                 if os.path.exists(path):
380                     self._cached_apache_path = path
381                     break
382
383             if not self._cached_apache_path:
384                 _log.error("Could not find apache. Not installed or unknown path.")
385
386         return self._cached_apache_path
387
388
389 class WebKitDriver(base.Driver):
390     """WebKit implementation of the DumpRenderTree interface."""
391
392     def __init__(self, port, image_path, options, executive=Executive()):
393         self._port = port
394         self._image_path = image_path
395         self._executive = executive
396         self._driver_tempdir = tempfile.mkdtemp(prefix='DumpRenderTree-')
397
398     def __del__(self):
399         shutil.rmtree(self._driver_tempdir)
400
401     def _driver_args(self):
402         driver_args = []
403
404         if self._image_path:
405             driver_args.append('--pixel-tests')
406
407         if self._port.get_option('use_drt'):
408             if self._port.get_option('accelerated_compositing'):
409                 driver_args.append('--enable-accelerated-compositing')
410
411             if self._port.get_option('accelerated_2d_canvas'):
412                 driver_args.append('--enable-accelerated-2d-canvas')
413
414         return driver_args
415
416     def start(self):
417         command = self._command_wrapper(self._port.get_option('wrapper'))
418         command += [self._port._path_to_driver(), '-']
419         command += self._driver_args()
420
421         environment = self._port.setup_environ_for_server()
422         environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path()
423         environment['DUMPRENDERTREE_TEMP'] = self._driver_tempdir
424         self._server_process = server_process.ServerProcess(self._port,
425             "DumpRenderTree", command, environment)
426
427     def poll(self):
428         return self._server_process.poll()
429
430     def restart(self):
431         self._server_process.stop()
432         self._server_process.start()
433         return
434
435     # FIXME: This function is huge.
436     def run_test(self, uri, timeoutms, image_hash):
437         if uri.startswith("file:///"):
438             command = uri[7:]
439         else:
440             command = uri
441
442         if image_hash:
443             command += "'" + image_hash
444         command += "\n"
445
446         start_time = time.time()
447         self._server_process.write(command)
448
449         have_seen_content_type = False
450         actual_image_hash = None
451         output = str()  # Use a byte array for output, even though it should be UTF-8.
452         image = str()
453
454         timeout = int(timeoutms) / 1000.0
455         deadline = time.time() + timeout
456         line = self._server_process.read_line(timeout)
457         while (not self._server_process.timed_out
458                and not self._server_process.crashed
459                and line.rstrip() != "#EOF"):
460             if (line.startswith('Content-Type:') and not
461                 have_seen_content_type):
462                 have_seen_content_type = True
463             else:
464                 # Note: Text output from DumpRenderTree is always UTF-8.
465                 # However, some tests (e.g. webarchives) spit out binary
466                 # data instead of text.  So to make things simple, we
467                 # always treat the output as binary.
468                 output += line
469             line = self._server_process.read_line(timeout)
470             timeout = deadline - time.time()
471
472         # Now read a second block of text for the optional image data
473         remaining_length = -1
474         HASH_HEADER = 'ActualHash: '
475         LENGTH_HEADER = 'Content-Length: '
476         line = self._server_process.read_line(timeout)
477         while (not self._server_process.timed_out
478                and not self._server_process.crashed
479                and line.rstrip() != "#EOF"):
480             if line.startswith(HASH_HEADER):
481                 actual_image_hash = line[len(HASH_HEADER):].strip()
482             elif line.startswith('Content-Type:'):
483                 pass
484             elif line.startswith(LENGTH_HEADER):
485                 timeout = deadline - time.time()
486                 content_length = int(line[len(LENGTH_HEADER):])
487                 image = self._server_process.read(timeout, content_length)
488             timeout = deadline - time.time()
489             line = self._server_process.read_line(timeout)
490
491         error_lines = self._server_process.error.splitlines()
492         # FIXME: This is a hack.  It is unclear why sometimes
493         # we do not get any error lines from the server_process
494         # probably we are not flushing stderr.
495         if error_lines and error_lines[-1] == "#EOF":
496             error_lines.pop()  # Remove the expected "#EOF"
497         error = "\n".join(error_lines)
498         # FIXME: This seems like the wrong section of code to be doing
499         # this reset in.
500         self._server_process.error = ""
501         return test_output.TestOutput(output, image, actual_image_hash,
502                                       self._server_process.crashed,
503                                       time.time() - start_time,
504                                       self._server_process.timed_out,
505                                       error)
506
507     def stop(self):
508         if self._server_process:
509             self._server_process.stop()
510             self._server_process = None