Tools/Scripts/webkitpy/layout_tests/port/webkit.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 # Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the Google name nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """WebKit implementations of the Port interface."""
  32
  33
  34 import logging
  35 import operator
  36 import os
  37 import re
  38 import signal
  39 import sys
  40 import time
  41 import webbrowser
  42
  43 import webkitpy.common.system.ospath as ospath
  44 import webkitpy.layout_tests.layout_package.test_output as test_output
  45 import webkitpy.layout_tests.port.base as base
  46 import webkitpy.layout_tests.port.server_process as server_process
  47
  48 _log = logging.getLogger("webkitpy.layout_tests.port.webkit")
  49
  50
  51 class WebKitPort(base.Port):
  52     """WebKit implementation of the Port class."""
  53
  54     def __init__(self, **kwargs):
  55         base.Port.__init__(self, **kwargs)
  56         self._cached_apache_path = None
  57
  58         # FIXME: disable pixel tests until they are run by default on the
  59         # build machines.
  60         self.set_option_default('pixel_tests', False)
  61
  62     def baseline_path(self):
  63         return self._webkit_baseline_path(self._name)
  64
  65     def baseline_search_path(self):
  66         return [self._webkit_baseline_path(self._name)]
  67
  68     def path_to_test_expectations_file(self):
  69         return self._filesystem.join(self._webkit_baseline_path(self._name),
  70                                      'test_expectations.txt')
  71
  72     # Only needed by ports which maintain versioned test expectations (like mac-tiger vs. mac-leopard)
  73     def version(self):
  74         return ''
  75
  76     def _build_driver(self):
  77         configuration = self.get_option('configuration')
  78         return self._config.build_dumprendertree(configuration)
  79
  80     def _check_driver(self):
  81         driver_path = self._path_to_driver()
  82         if not self._filesystem.exists(driver_path):
  83             _log.error("DumpRenderTree was not found at %s" % driver_path)
  84             return False
  85         return True
  86
  87     def check_build(self, needs_http):
  88         if self.get_option('build') and not self._build_driver():
  89             return False
  90         if not self._check_driver():
  91             return False
  92         if self.get_option('pixel_tests'):
  93             if not self.check_image_diff():
  94                 return False
  95         if not self._check_port_build():
  96             return False
  97         return True
  98
  99     def _check_port_build(self):
 100         # Ports can override this method to do additional checks.
 101         return True
 102
 103     def check_image_diff(self, override_step=None, logging=True):
 104         image_diff_path = self._path_to_image_diff()
 105         if not self._filesystem.exists(image_diff_path):
 106             _log.error("ImageDiff was not found at %s" % image_diff_path)
 107             return False
 108         return True
 109
 110     def diff_image(self, expected_contents, actual_contents,
 111                    diff_filename=None):
 112         """Return True if the two files are different. Also write a delta
 113         image of the two images into |diff_filename| if it is not None."""
 114
 115         # Handle the case where the test didn't actually generate an image.
 116         if not actual_contents:
 117             return True
 118
 119         sp = self._diff_image_request(expected_contents, actual_contents)
 120         return self._diff_image_reply(sp, diff_filename)
 121
 122     def _diff_image_request(self, expected_contents, actual_contents):
 123         # FIXME: use self.get_option('tolerance') and
 124         # self.set_option_default('tolerance', 0.1) once that behaves correctly
 125         # with default values.
 126         if self.get_option('tolerance') is not None:
 127             tolerance = self.get_option('tolerance')
 128         else:
 129             tolerance = 0.1
 130         command = [self._path_to_image_diff(), '--tolerance', str(tolerance)]
 131         sp = server_process.ServerProcess(self, 'ImageDiff', command)
 132
 133         sp.write('Content-Length: %d\n%sContent-Length: %d\n%s' %
 134                  (len(actual_contents), actual_contents,
 135                   len(expected_contents), expected_contents))
 136
 137         return sp
 138
 139     def _diff_image_reply(self, sp, diff_filename):
 140         timeout = 2.0
 141         deadline = time.time() + timeout
 142         output = sp.read_line(timeout)
 143         while not sp.timed_out and not sp.crashed and output:
 144             if output.startswith('Content-Length'):
 145                 m = re.match('Content-Length: (\d+)', output)
 146                 content_length = int(m.group(1))
 147                 timeout = deadline - time.time()
 148                 output = sp.read(timeout, content_length)
 149                 break
 150             elif output.startswith('diff'):
 151                 break
 152             else:
 153                 timeout = deadline - time.time()
 154                 output = sp.read_line(deadline)
 155
 156         result = True
 157         if output.startswith('diff'):
 158             m = re.match('diff: (.+)% (passed|failed)', output)
 159             if m.group(2) == 'passed':
 160                 result = False
 161         elif output and diff_filename:
 162             self._filesystem.write_text_file(diff_filename, output)
 163         elif sp.timed_out:
 164             _log.error("ImageDiff timed out")
 165         elif sp.crashed:
 166             _log.error("ImageDiff crashed")
 167         sp.stop()
 168         return result
 169
 170     def results_directory(self):
 171         # Results are store relative to the built products to make it easy
 172         # to have multiple copies of webkit checked out and built.
 173         return self._build_path(self.get_option('results_directory'))
 174
 175     def setup_test_run(self):
 176         # This port doesn't require any specific configuration.
 177         pass
 178
 179     def create_driver(self, worker_number):
 180         return WebKitDriver(self, worker_number)
 181
 182     def test_base_platform_names(self):
 183         # At the moment we don't use test platform names, but we have
 184         # to return something.
 185         return ('mac', 'win')
 186
 187     def _tests_for_other_platforms(self):
 188         raise NotImplementedError('WebKitPort._tests_for_other_platforms')
 189         # The original run-webkit-tests builds up a "whitelist" of tests to
 190         # run, and passes that to DumpRenderTree. new-run-webkit-tests assumes
 191         # we run *all* tests and test_expectations.txt functions as a
 192         # blacklist.
 193         # FIXME: This list could be dynamic based on platform name and
 194         # pushed into base.Port.
 195         return [
 196             "platform/chromium",
 197             "platform/gtk",
 198             "platform/qt",
 199             "platform/win",
 200         ]
 201
 202     def _runtime_feature_list(self):
 203         """Return the supported features of DRT. If a port doesn't support
 204         this DRT switch, it has to override this method to return None"""
 205         driver_path = self._path_to_driver()
 206         feature_list = ' '.join(os.popen(driver_path + " --print-supported-features 2>&1").readlines())
 207         if "SupportedFeatures:" in feature_list:
 208             return feature_list
 209         return None
 210
 211     def _supported_symbol_list(self):
 212         """Return the supported symbols of WebCore."""
 213         webcore_library_path = self._path_to_webcore_library()
 214         if not webcore_library_path:
 215             return None
 216         symbol_list = ' '.join(os.popen("nm " + webcore_library_path).readlines())
 217         return symbol_list
 218
 219     def _directories_for_features(self):
 220         """Return the supported feature dictionary. The keys are the
 221         features and the values are the directories in lists."""
 222         directories_for_features = {
 223             "Accelerated Compositing": ["compositing"],
 224             "3D Rendering": ["animations/3d", "transforms/3d"],
 225         }
 226         return directories_for_features
 227
 228     def _directories_for_symbols(self):
 229         """Return the supported feature dictionary. The keys are the
 230         symbols and the values are the directories in lists."""
 231         directories_for_symbol = {
 232             "MathMLElement": ["mathml"],
 233             "GraphicsLayer": ["compositing"],
 234             "WebCoreHas3DRendering": ["animations/3d", "transforms/3d"],
 235             "WebGLShader": ["fast/canvas/webgl", "compositing/webgl", "http/tests/canvas/webgl"],
 236             "WMLElement": ["http/tests/wml", "fast/wml", "wml"],
 237             "parseWCSSInputProperty": ["fast/wcss"],
 238             "isXHTMLMPDocument": ["fast/xhtmlmp"],
 239         }
 240         return directories_for_symbol
 241
 242     def _skipped_tests_for_unsupported_features(self):
 243         """Return the directories of unsupported tests. Search for the
 244         symbols in the symbol_list, if found add the corresponding
 245         directories to the skipped directory list."""
 246         feature_list = self._runtime_feature_list()
 247         directories = self._directories_for_features()
 248
 249         # if DRT feature detection not supported
 250         if not feature_list:
 251             feature_list = self._supported_symbol_list()
 252             directories = self._directories_for_symbols()
 253
 254         if not feature_list:
 255             return []
 256
 257         skipped_directories = [directories[feature]
 258                               for feature in directories.keys()
 259                               if feature not in feature_list]
 260         return reduce(operator.add, skipped_directories)
 261
 262     def _tests_for_disabled_features(self):
 263         # FIXME: This should use the feature detection from
 264         # webkitperl/features.pm to match run-webkit-tests.
 265         # For now we hard-code a list of features known to be disabled on
 266         # the Mac platform.
 267         disabled_feature_tests = [
 268             "fast/xhtmlmp",
 269             "http/tests/wml",
 270             "mathml",
 271             "wml",
 272         ]
 273         # FIXME: webarchive tests expect to read-write from
 274         # -expected.webarchive files instead of .txt files.
 275         # This script doesn't know how to do that yet, so pretend they're
 276         # just "disabled".
 277         webarchive_tests = [
 278             "webarchive",
 279             "svg/webarchive",
 280             "http/tests/webarchive",
 281             "svg/custom/image-with-prefix-in-webarchive.svg",
 282         ]
 283         unsupported_feature_tests = self._skipped_tests_for_unsupported_features()
 284         return disabled_feature_tests + webarchive_tests + unsupported_feature_tests
 285
 286     def _tests_from_skipped_file_contents(self, skipped_file_contents):
 287         tests_to_skip = []
 288         for line in skipped_file_contents.split('\n'):
 289             line = line.strip()
 290             if line.startswith('#') or not len(line):
 291                 continue
 292             tests_to_skip.append(line)
 293         return tests_to_skip
 294
 295     def _skipped_file_paths(self):
 296         return [self._filesystem.join(self._webkit_baseline_path(self._name), 'Skipped')]
 297
 298     def _expectations_from_skipped_files(self):
 299         tests_to_skip = []
 300         for filename in self._skipped_file_paths():
 301             if not self._filesystem.exists(filename):
 302                 _log.warn("Failed to open Skipped file: %s" % filename)
 303                 continue
 304             skipped_file_contents = self._filesystem.read_text_file(filename)
 305             tests_to_skip.extend(self._tests_from_skipped_file_contents(skipped_file_contents))
 306         return tests_to_skip
 307
 308     def test_expectations(self):
 309         # The WebKit mac port uses a combination of a test_expectations file
 310         # and 'Skipped' files.
 311         expectations_path = self.path_to_test_expectations_file()
 312         return self._filesystem.read_text_file(expectations_path) + self._skips()
 313
 314     def _skips(self):
 315         # Each Skipped file contains a list of files
 316         # or directories to be skipped during the test run. The total list
 317         # of tests to skipped is given by the contents of the generic
 318         # Skipped file found in platform/X plus a version-specific file
 319         # found in platform/X-version. Duplicate entries are allowed.
 320         # This routine reads those files and turns contents into the
 321         # format expected by test_expectations.
 322
 323         tests_to_skip = self.skipped_layout_tests()
 324         skip_lines = map(lambda test_path: "BUG_SKIPPED SKIP : %s = FAIL" %
 325                                 test_path, tests_to_skip)
 326         return "\n".join(skip_lines)
 327
 328     def skipped_layout_tests(self):
 329         # Use a set to allow duplicates
 330         tests_to_skip = set(self._expectations_from_skipped_files())
 331         tests_to_skip.update(self._tests_for_other_platforms())
 332         tests_to_skip.update(self._tests_for_disabled_features())
 333         return tests_to_skip
 334
 335     def test_platform_name(self):
 336         return self._name + self.version()
 337
 338     def test_platform_names(self):
 339         return self.test_base_platform_names() + (
 340             'mac-tiger', 'mac-leopard', 'mac-snowleopard')
 341
 342     def _build_path(self, *comps):
 343         return self._filesystem.join(self._config.build_directory(
 344             self.get_option('configuration')), *comps)
 345
 346     def _path_to_driver(self):
 347         return self._build_path('DumpRenderTree')
 348
 349     def _path_to_webcore_library(self):
 350         return None
 351
 352     def _path_to_helper(self):
 353         return None
 354
 355     def _path_to_image_diff(self):
 356         return self._build_path('ImageDiff')
 357
 358     def _path_to_wdiff(self):
 359         # FIXME: This does not exist on a default Mac OS X Leopard install.
 360         return 'wdiff'
 361
 362     def _path_to_apache(self):
 363         if not self._cached_apache_path:
 364             # The Apache binary path can vary depending on OS and distribution
 365             # See http://wiki.apache.org/httpd/DistrosDefaultLayout
 366             for path in ["/usr/sbin/httpd", "/usr/sbin/apache2"]:
 367                 if self._filesystem.exists(path):
 368                     self._cached_apache_path = path
 369                     break
 370
 371             if not self._cached_apache_path:
 372                 _log.error("Could not find apache. Not installed or unknown path.")
 373
 374         return self._cached_apache_path
 375
 376
 377 class WebKitDriver(base.Driver):
 378     """WebKit implementation of the DumpRenderTree interface."""
 379
 380     def __init__(self, port, worker_number):
 381         self._worker_number = worker_number
 382         self._port = port
 383         self._driver_tempdir = port._filesystem.mkdtemp(prefix='DumpRenderTree-')
 384
 385     def __del__(self):
 386         self._port._filesystem.rmtree(str(self._driver_tempdir))
 387
 388     def cmd_line(self):
 389         cmd = self._command_wrapper(self._port.get_option('wrapper'))
 390         cmd += [self._port._path_to_driver(), '-']
 391
 392         if self._port.get_option('pixel_tests'):
 393             cmd.append('--pixel-tests')
 394
 395         return cmd
 396
 397     def start(self):
 398         environment = self._port.setup_environ_for_server()
 399         environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path()
 400         environment['DUMPRENDERTREE_TEMP'] = str(self._driver_tempdir)
 401         self._server_process = server_process.ServerProcess(self._port,
 402             "DumpRenderTree", self.cmd_line(), environment)
 403
 404     def poll(self):
 405         return self._server_process.poll()
 406
 407     def restart(self):
 408         self._server_process.stop()
 409         self._server_process.start()
 410         return
 411
 412     # FIXME: This function is huge.
 413     def run_test(self, test_input):
 414         uri = self._port.filename_to_uri(test_input.filename)
 415         if uri.startswith("file:///"):
 416             command = uri[7:]
 417         else:
 418             command = uri
 419
 420         if test_input.image_hash:
 421             command += "'" + test_input.image_hash
 422         command += "\n"
 423
 424         start_time = time.time()
 425         self._server_process.write(command)
 426
 427         have_seen_content_type = False
 428         actual_image_hash = None
 429         output = str()  # Use a byte array for output, even though it should be UTF-8.
 430         image = str()
 431
 432         timeout = int(test_input.timeout) / 1000.0
 433         deadline = time.time() + timeout
 434         line = self._server_process.read_line(timeout)
 435         while (not self._server_process.timed_out
 436                and not self._server_process.crashed
 437                and line.rstrip() != "#EOF"):
 438             if (line.startswith('Content-Type:') and not
 439                 have_seen_content_type):
 440                 have_seen_content_type = True
 441             else:
 442                 # Note: Text output from DumpRenderTree is always UTF-8.
 443                 # However, some tests (e.g. webarchives) spit out binary
 444                 # data instead of text.  So to make things simple, we
 445                 # always treat the output as binary.
 446                 output += line
 447             line = self._server_process.read_line(timeout)
 448             timeout = deadline - time.time()
 449
 450         # Now read a second block of text for the optional image data
 451         remaining_length = -1
 452         HASH_HEADER = 'ActualHash: '
 453         LENGTH_HEADER = 'Content-Length: '
 454         line = self._server_process.read_line(timeout)
 455         while (not self._server_process.timed_out
 456                and not self._server_process.crashed
 457                and line.rstrip() != "#EOF"):
 458             if line.startswith(HASH_HEADER):
 459                 actual_image_hash = line[len(HASH_HEADER):].strip()
 460             elif line.startswith('Content-Type:'):
 461                 pass
 462             elif line.startswith(LENGTH_HEADER):
 463                 timeout = deadline - time.time()
 464                 content_length = int(line[len(LENGTH_HEADER):])
 465                 image = self._server_process.read(timeout, content_length)
 466             timeout = deadline - time.time()
 467             line = self._server_process.read_line(timeout)
 468
 469         error_lines = self._server_process.error.splitlines()
 470         # FIXME: This is a hack.  It is unclear why sometimes
 471         # we do not get any error lines from the server_process
 472         # probably we are not flushing stderr.
 473         if error_lines and error_lines[-1] == "#EOF":
 474             error_lines.pop()  # Remove the expected "#EOF"
 475         error = "\n".join(error_lines)
 476         # FIXME: This seems like the wrong section of code to be doing
 477         # this reset in.
 478         self._server_process.error = ""
 479         return test_output.TestOutput(output, image, actual_image_hash,
 480                                       self._server_process.crashed,
 481                                       time.time() - start_time,
 482                                       self._server_process.timed_out,
 483                                       error)
 484
 485     def stop(self):
 486         if self._server_process:
 487             self._server_process.stop()
 488             self._server_process = None