2011-02-03 Hayato Ito <hayato@chromium.org>
[WebKit-https.git] / Tools / Scripts / webkitpy / layout_tests / port / webkit.py
1 #!/usr/bin/env python
2 # Copyright (C) 2010 Google Inc. All rights reserved.
3 # Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged
4 #
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions are
7 # met:
8 #
9 #     * Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 #     * Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following disclaimer
13 # in the documentation and/or other materials provided with the
14 # distribution.
15 #     * Neither the Google name nor the names of its
16 # contributors may be used to endorse or promote products derived from
17 # this software without specific prior written permission.
18 #
19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31 """WebKit implementations of the Port interface."""
32
33
34 import logging
35 import operator
36 import os
37 import re
38 import signal
39 import sys
40 import time
41 import webbrowser
42
43 import webkitpy.common.system.ospath as ospath
44 import webkitpy.layout_tests.port.base as base
45 import webkitpy.layout_tests.port.server_process as server_process
46
47 _log = logging.getLogger("webkitpy.layout_tests.port.webkit")
48
49
50 class WebKitPort(base.Port):
51     """WebKit implementation of the Port class."""
52
53     def __init__(self, **kwargs):
54         base.Port.__init__(self, **kwargs)
55         self._cached_apache_path = None
56
57         # FIXME: disable pixel tests until they are run by default on the
58         # build machines.
59         if not hasattr(self._options, "pixel_tests") or self._options.pixel_tests == None:
60             self._options.pixel_tests = False
61
62     def baseline_path(self):
63         return self._webkit_baseline_path(self._name)
64
65     def baseline_search_path(self):
66         return [self._webkit_baseline_path(self._name)]
67
68     def path_to_test_expectations_file(self):
69         return self._filesystem.join(self._webkit_baseline_path(self._name),
70                                      'test_expectations.txt')
71
72     # Only needed by ports which maintain versioned test expectations (like mac-tiger vs. mac-leopard)
73     def version(self):
74         return ''
75
76     def _build_driver(self):
77         configuration = self.get_option('configuration')
78         return self._config.build_dumprendertree(configuration)
79
80     def _check_driver(self):
81         driver_path = self._path_to_driver()
82         if not self._filesystem.exists(driver_path):
83             _log.error("DumpRenderTree was not found at %s" % driver_path)
84             return False
85         return True
86
87     def check_build(self, needs_http):
88         if self.get_option('build') and not self._build_driver():
89             return False
90         if not self._check_driver():
91             return False
92         if self.get_option('pixel_tests'):
93             if not self.check_image_diff():
94                 return False
95         if not self._check_port_build():
96             return False
97         return True
98
99     def _check_port_build(self):
100         # Ports can override this method to do additional checks.
101         return True
102
103     def check_image_diff(self, override_step=None, logging=True):
104         image_diff_path = self._path_to_image_diff()
105         if not self._filesystem.exists(image_diff_path):
106             _log.error("ImageDiff was not found at %s" % image_diff_path)
107             return False
108         return True
109
110     def diff_image(self, expected_contents, actual_contents,
111                    diff_filename=None):
112         """Return True if the two files are different. Also write a delta
113         image of the two images into |diff_filename| if it is not None."""
114
115         # Handle the case where the test didn't actually generate an image.
116         if not actual_contents:
117             return True
118
119         sp = self._diff_image_request(expected_contents, actual_contents)
120         return self._diff_image_reply(sp, diff_filename)
121
122     def _diff_image_request(self, expected_contents, actual_contents):
123         # FIXME: There needs to be a more sane way of handling default
124         # values for options so that you can distinguish between a default
125         # value of None and a default value that wasn't set.
126         if self.get_option('tolerance') is not None:
127             tolerance = self.get_option('tolerance')
128         else:
129             tolerance = 0.1
130         command = [self._path_to_image_diff(), '--tolerance', str(tolerance)]
131         sp = server_process.ServerProcess(self, 'ImageDiff', command)
132
133         sp.write('Content-Length: %d\n%sContent-Length: %d\n%s' %
134                  (len(actual_contents), actual_contents,
135                   len(expected_contents), expected_contents))
136
137         return sp
138
139     def _diff_image_reply(self, sp, diff_filename):
140         timeout = 2.0
141         deadline = time.time() + timeout
142         output = sp.read_line(timeout)
143         while not sp.timed_out and not sp.crashed and output:
144             if output.startswith('Content-Length'):
145                 m = re.match('Content-Length: (\d+)', output)
146                 content_length = int(m.group(1))
147                 timeout = deadline - time.time()
148                 output = sp.read(timeout, content_length)
149                 break
150             elif output.startswith('diff'):
151                 break
152             else:
153                 timeout = deadline - time.time()
154                 output = sp.read_line(deadline)
155
156         result = True
157         if output.startswith('diff'):
158             m = re.match('diff: (.+)% (passed|failed)', output)
159             if m.group(2) == 'passed':
160                 result = False
161         elif output and diff_filename:
162             self._filesystem.write_binary_file(diff_filename, output)
163         elif sp.timed_out:
164             _log.error("ImageDiff timed out")
165         elif sp.crashed:
166             _log.error("ImageDiff crashed")
167         sp.stop()
168         return result
169
170     def results_directory(self):
171         # Results are store relative to the built products to make it easy
172         # to have multiple copies of webkit checked out and built.
173         return self._build_path(self.get_option('results_directory'))
174
175     def setup_test_run(self):
176         # This port doesn't require any specific configuration.
177         pass
178
179     def create_driver(self, worker_number):
180         return WebKitDriver(self, worker_number)
181
182     def _tests_for_other_platforms(self):
183         raise NotImplementedError('WebKitPort._tests_for_other_platforms')
184         # The original run-webkit-tests builds up a "whitelist" of tests to
185         # run, and passes that to DumpRenderTree. new-run-webkit-tests assumes
186         # we run *all* tests and test_expectations.txt functions as a
187         # blacklist.
188         # FIXME: This list could be dynamic based on platform name and
189         # pushed into base.Port.
190         return [
191             "platform/chromium",
192             "platform/gtk",
193             "platform/qt",
194             "platform/win",
195         ]
196
197     def _runtime_feature_list(self):
198         """Return the supported features of DRT. If a port doesn't support
199         this DRT switch, it has to override this method to return None"""
200         driver_path = self._path_to_driver()
201         feature_list = ' '.join(os.popen(driver_path + " --print-supported-features 2>&1").readlines())
202         if "SupportedFeatures:" in feature_list:
203             return feature_list
204         return None
205
206     def _supported_symbol_list(self):
207         """Return the supported symbols of WebCore."""
208         webcore_library_path = self._path_to_webcore_library()
209         if not webcore_library_path:
210             return None
211         symbol_list = ' '.join(os.popen("nm " + webcore_library_path).readlines())
212         return symbol_list
213
214     def _directories_for_features(self):
215         """Return the supported feature dictionary. The keys are the
216         features and the values are the directories in lists."""
217         directories_for_features = {
218             "Accelerated Compositing": ["compositing"],
219             "3D Rendering": ["animations/3d", "transforms/3d"],
220         }
221         return directories_for_features
222
223     def _directories_for_symbols(self):
224         """Return the supported feature dictionary. The keys are the
225         symbols and the values are the directories in lists."""
226         directories_for_symbol = {
227             "MathMLElement": ["mathml"],
228             "GraphicsLayer": ["compositing"],
229             "WebCoreHas3DRendering": ["animations/3d", "transforms/3d"],
230             "WebGLShader": ["fast/canvas/webgl", "compositing/webgl", "http/tests/canvas/webgl"],
231             "WMLElement": ["http/tests/wml", "fast/wml", "wml"],
232             "parseWCSSInputProperty": ["fast/wcss"],
233             "isXHTMLMPDocument": ["fast/xhtmlmp"],
234         }
235         return directories_for_symbol
236
237     def _skipped_tests_for_unsupported_features(self):
238         """Return the directories of unsupported tests. Search for the
239         symbols in the symbol_list, if found add the corresponding
240         directories to the skipped directory list."""
241         feature_list = self._runtime_feature_list()
242         directories = self._directories_for_features()
243
244         # if DRT feature detection not supported
245         if not feature_list:
246             feature_list = self._supported_symbol_list()
247             directories = self._directories_for_symbols()
248
249         if not feature_list:
250             return []
251
252         skipped_directories = [directories[feature]
253                               for feature in directories.keys()
254                               if feature not in feature_list]
255         return reduce(operator.add, skipped_directories)
256
257     def _tests_for_disabled_features(self):
258         # FIXME: This should use the feature detection from
259         # webkitperl/features.pm to match run-webkit-tests.
260         # For now we hard-code a list of features known to be disabled on
261         # the Mac platform.
262         disabled_feature_tests = [
263             "fast/xhtmlmp",
264             "http/tests/wml",
265             "mathml",
266             "wml",
267         ]
268         # FIXME: webarchive tests expect to read-write from
269         # -expected.webarchive files instead of .txt files.
270         # This script doesn't know how to do that yet, so pretend they're
271         # just "disabled".
272         webarchive_tests = [
273             "webarchive",
274             "svg/webarchive",
275             "http/tests/webarchive",
276             "svg/custom/image-with-prefix-in-webarchive.svg",
277         ]
278         unsupported_feature_tests = self._skipped_tests_for_unsupported_features()
279         return disabled_feature_tests + webarchive_tests + unsupported_feature_tests
280
281     def _tests_from_skipped_file_contents(self, skipped_file_contents):
282         tests_to_skip = []
283         for line in skipped_file_contents.split('\n'):
284             line = line.strip()
285             if line.startswith('#') or not len(line):
286                 continue
287             tests_to_skip.append(line)
288         return tests_to_skip
289
290     def _skipped_file_paths(self):
291         return [self._filesystem.join(self._webkit_baseline_path(self._name), 'Skipped')]
292
293     def _expectations_from_skipped_files(self):
294         tests_to_skip = []
295         for filename in self._skipped_file_paths():
296             if not self._filesystem.exists(filename):
297                 _log.warn("Failed to open Skipped file: %s" % filename)
298                 continue
299             skipped_file_contents = self._filesystem.read_text_file(filename)
300             tests_to_skip.extend(self._tests_from_skipped_file_contents(skipped_file_contents))
301         return tests_to_skip
302
303     def test_expectations(self):
304         # The WebKit mac port uses a combination of a test_expectations file
305         # and 'Skipped' files.
306         expectations_path = self.path_to_test_expectations_file()
307         return self._filesystem.read_text_file(expectations_path) + self._skips()
308
309     def _skips(self):
310         # Each Skipped file contains a list of files
311         # or directories to be skipped during the test run. The total list
312         # of tests to skipped is given by the contents of the generic
313         # Skipped file found in platform/X plus a version-specific file
314         # found in platform/X-version. Duplicate entries are allowed.
315         # This routine reads those files and turns contents into the
316         # format expected by test_expectations.
317
318         tests_to_skip = self.skipped_layout_tests()
319         skip_lines = map(lambda test_path: "BUG_SKIPPED SKIP : %s = FAIL" %
320                                 test_path, tests_to_skip)
321         return "\n".join(skip_lines)
322
323     def skipped_layout_tests(self):
324         # Use a set to allow duplicates
325         tests_to_skip = set(self._expectations_from_skipped_files())
326         tests_to_skip.update(self._tests_for_other_platforms())
327         tests_to_skip.update(self._tests_for_disabled_features())
328         return tests_to_skip
329
330     def test_platform_name(self):
331         return self._name + self.version()
332
333     def test_platform_names(self):
334         return ('mac', 'win', 'mac-tiger', 'mac-leopard', 'mac-snowleopard')
335
336     def _build_path(self, *comps):
337         return self._filesystem.join(self._config.build_directory(
338             self.get_option('configuration')), *comps)
339
340     def _path_to_driver(self):
341         return self._build_path('DumpRenderTree')
342
343     def _path_to_webcore_library(self):
344         return None
345
346     def _path_to_helper(self):
347         return None
348
349     def _path_to_image_diff(self):
350         return self._build_path('ImageDiff')
351
352     def _path_to_wdiff(self):
353         # FIXME: This does not exist on a default Mac OS X Leopard install.
354         return 'wdiff'
355
356     def _path_to_apache(self):
357         if not self._cached_apache_path:
358             # The Apache binary path can vary depending on OS and distribution
359             # See http://wiki.apache.org/httpd/DistrosDefaultLayout
360             for path in ["/usr/sbin/httpd", "/usr/sbin/apache2"]:
361                 if self._filesystem.exists(path):
362                     self._cached_apache_path = path
363                     break
364
365             if not self._cached_apache_path:
366                 _log.error("Could not find apache. Not installed or unknown path.")
367
368         return self._cached_apache_path
369
370
371 class WebKitDriver(base.Driver):
372     """WebKit implementation of the DumpRenderTree interface."""
373
374     def __init__(self, port, worker_number):
375         self._worker_number = worker_number
376         self._port = port
377         self._driver_tempdir = port._filesystem.mkdtemp(prefix='DumpRenderTree-')
378
379     def __del__(self):
380         self._port._filesystem.rmtree(str(self._driver_tempdir))
381
382     def cmd_line(self):
383         cmd = self._command_wrapper(self._port.get_option('wrapper'))
384         cmd += [self._port._path_to_driver(), '-']
385
386         if self._port.get_option('pixel_tests'):
387             cmd.append('--pixel-tests')
388
389         return cmd
390
391     def start(self):
392         environment = self._port.setup_environ_for_server()
393         environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path()
394         environment['DUMPRENDERTREE_TEMP'] = str(self._driver_tempdir)
395         self._server_process = server_process.ServerProcess(self._port,
396             "DumpRenderTree", self.cmd_line(), environment)
397
398     def poll(self):
399         return self._server_process.poll()
400
401     def restart(self):
402         self._server_process.stop()
403         self._server_process.start()
404         return
405
406     # FIXME: This function is huge.
407     def run_test(self, driver_input):
408         uri = self._port.filename_to_uri(driver_input.filename)
409         if uri.startswith("file:///"):
410             command = uri[7:]
411         else:
412             command = uri
413
414         if driver_input.image_hash:
415             command += "'" + driver_input.image_hash
416         command += "\n"
417
418         start_time = time.time()
419         self._server_process.write(command)
420
421         have_seen_content_type = False
422         actual_image_hash = None
423         output = str()  # Use a byte array for output, even though it should be UTF-8.
424         image = str()
425
426         timeout = int(driver_input.timeout) / 1000.0
427         deadline = time.time() + timeout
428         line = self._server_process.read_line(timeout)
429         while (not self._server_process.timed_out
430                and not self._server_process.crashed
431                and line.rstrip() != "#EOF"):
432             if (line.startswith('Content-Type:') and not
433                 have_seen_content_type):
434                 have_seen_content_type = True
435             else:
436                 # Note: Text output from DumpRenderTree is always UTF-8.
437                 # However, some tests (e.g. webarchives) spit out binary
438                 # data instead of text.  So to make things simple, we
439                 # always treat the output as binary.
440                 output += line
441             line = self._server_process.read_line(timeout)
442             timeout = deadline - time.time()
443
444         # Now read a second block of text for the optional image data
445         remaining_length = -1
446         HASH_HEADER = 'ActualHash: '
447         LENGTH_HEADER = 'Content-Length: '
448         line = self._server_process.read_line(timeout)
449         while (not self._server_process.timed_out
450                and not self._server_process.crashed
451                and line.rstrip() != "#EOF"):
452             if line.startswith(HASH_HEADER):
453                 actual_image_hash = line[len(HASH_HEADER):].strip()
454             elif line.startswith('Content-Type:'):
455                 pass
456             elif line.startswith(LENGTH_HEADER):
457                 timeout = deadline - time.time()
458                 content_length = int(line[len(LENGTH_HEADER):])
459                 image = self._server_process.read(timeout, content_length)
460             timeout = deadline - time.time()
461             line = self._server_process.read_line(timeout)
462
463         error_lines = self._server_process.error.splitlines()
464         # FIXME: This is a hack.  It is unclear why sometimes
465         # we do not get any error lines from the server_process
466         # probably we are not flushing stderr.
467         if error_lines and error_lines[-1] == "#EOF":
468             error_lines.pop()  # Remove the expected "#EOF"
469         error = "\n".join(error_lines)
470         # FIXME: This seems like the wrong section of code to be doing
471         # this reset in.
472         self._server_process.error = ""
473         return base.DriverOutput(output, image, actual_image_hash,
474                                  self._server_process.crashed,
475                                  time.time() - start_time,
476                                  self._server_process.timed_out,
477                                  error)
478
479     def stop(self):
480         if self._server_process:
481             self._server_process.stop()
482             self._server_process = None