Tools/Scripts/webkitpy/layout_tests/port/webkit.py

   1 #!/usr/bin/env python
   2 # Copyright (C) 2010 Google Inc. All rights reserved.
   3 # Copyright (C) 2010 Gabor Rapcsanyi <rgabor@inf.u-szeged.hu>, University of Szeged
   4 #
   5 # Redistribution and use in source and binary forms, with or without
   6 # modification, are permitted provided that the following conditions are
   7 # met:
   8 #
   9 #     * Redistributions of source code must retain the above copyright
  10 # notice, this list of conditions and the following disclaimer.
  11 #     * Redistributions in binary form must reproduce the above
  12 # copyright notice, this list of conditions and the following disclaimer
  13 # in the documentation and/or other materials provided with the
  14 # distribution.
  15 #     * Neither the Google name nor the names of its
  16 # contributors may be used to endorse or promote products derived from
  17 # this software without specific prior written permission.
  18 #
  19 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  20 # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  21 # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  22 # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  23 # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  24 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  25 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  26 # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  27 # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  28 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  29 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  30
  31 """WebKit implementations of the Port interface."""
  32
  33
  34 import logging
  35 import operator
  36 import os
  37 import re
  38 import signal
  39 import sys
  40 import time
  41 import webbrowser
  42
  43 import webkitpy.common.system.ospath as ospath
  44 import webkitpy.layout_tests.layout_package.test_output as test_output
  45 import webkitpy.layout_tests.port.base as base
  46 import webkitpy.layout_tests.port.server_process as server_process
  47
  48 _log = logging.getLogger("webkitpy.layout_tests.port.webkit")
  49
  50
  51 class WebKitPort(base.Port):
  52     """WebKit implementation of the Port class."""
  53
  54     def __init__(self, **kwargs):
  55         base.Port.__init__(self, **kwargs)
  56         self._cached_apache_path = None
  57
  58         # FIXME: disable pixel tests until they are run by default on the
  59         # build machines.
  60         self.set_option_default('pixel_tests', False)
  61
  62     def baseline_path(self):
  63         return self._webkit_baseline_path(self._name)
  64
  65     def baseline_search_path(self):
  66         return [self._webkit_baseline_path(self._name)]
  67
  68     def path_to_test_expectations_file(self):
  69         return self._filesystem.join(self._webkit_baseline_path(self._name),
  70                                      'test_expectations.txt')
  71
  72     # Only needed by ports which maintain versioned test expectations (like mac-tiger vs. mac-leopard)
  73     def version(self):
  74         return ''
  75
  76     def _build_driver(self):
  77         configuration = self.get_option('configuration')
  78         return self._config.build_dumprendertree(configuration)
  79
  80     def _check_driver(self):
  81         driver_path = self._path_to_driver()
  82         if not self._filesystem.exists(driver_path):
  83             _log.error("DumpRenderTree was not found at %s" % driver_path)
  84             return False
  85         return True
  86
  87     def check_build(self, needs_http):
  88         if self.get_option('build') and not self._build_driver():
  89             return False
  90         if not self._check_driver():
  91             return False
  92         if self.get_option('pixel_tests'):
  93             if not self.check_image_diff():
  94                 return False
  95         if not self._check_port_build():
  96             return False
  97         return True
  98
  99     def _check_port_build(self):
 100         # Ports can override this method to do additional checks.
 101         return True
 102
 103     def check_image_diff(self, override_step=None, logging=True):
 104         image_diff_path = self._path_to_image_diff()
 105         if not self._filesystem.exists(image_diff_path):
 106             _log.error("ImageDiff was not found at %s" % image_diff_path)
 107             return False
 108         return True
 109
 110     def diff_image(self, expected_contents, actual_contents,
 111                    diff_filename=None):
 112         """Return True if the two files are different. Also write a delta
 113         image of the two images into |diff_filename| if it is not None."""
 114
 115         # Handle the case where the test didn't actually generate an image.
 116         if not actual_contents:
 117             return True
 118
 119         sp = self._diff_image_request(expected_contents, actual_contents)
 120         return self._diff_image_reply(sp, diff_filename)
 121
 122     def _diff_image_request(self, expected_contents, actual_contents):
 123         # FIXME: use self.get_option('tolerance') and
 124         # self.set_option_default('tolerance', 0.1) once that behaves correctly
 125         # with default values.
 126         if self.get_option('tolerance') is not None:
 127             tolerance = self.get_option('tolerance')
 128         else:
 129             tolerance = 0.1
 130         command = [self._path_to_image_diff(), '--tolerance', str(tolerance)]
 131         sp = server_process.ServerProcess(self, 'ImageDiff', command)
 132
 133         sp.write('Content-Length: %d\n%sContent-Length: %d\n%s' %
 134                  (len(actual_contents), actual_contents,
 135                   len(expected_contents), expected_contents))
 136
 137         return sp
 138
 139     def _diff_image_reply(self, sp, diff_filename):
 140         timeout = 2.0
 141         deadline = time.time() + timeout
 142         output = sp.read_line(timeout)
 143         while not sp.timed_out and not sp.crashed and output:
 144             if output.startswith('Content-Length'):
 145                 m = re.match('Content-Length: (\d+)', output)
 146                 content_length = int(m.group(1))
 147                 timeout = deadline - time.time()
 148                 output = sp.read(timeout, content_length)
 149                 break
 150             elif output.startswith('diff'):
 151                 break
 152             else:
 153                 timeout = deadline - time.time()
 154                 output = sp.read_line(deadline)
 155
 156         result = True
 157         if output.startswith('diff'):
 158             m = re.match('diff: (.+)% (passed|failed)', output)
 159             if m.group(2) == 'passed':
 160                 result = False
 161         elif output and diff_filename:
 162             self._filesystem.write_text_file(diff_filename, output)
 163         elif sp.timed_out:
 164             _log.error("ImageDiff timed out")
 165         elif sp.crashed:
 166             _log.error("ImageDiff crashed")
 167         sp.stop()
 168         return result
 169
 170     def results_directory(self):
 171         # Results are store relative to the built products to make it easy
 172         # to have multiple copies of webkit checked out and built.
 173         return self._build_path(self.get_option('results_directory'))
 174
 175     def setup_test_run(self):
 176         # This port doesn't require any specific configuration.
 177         pass
 178
 179     def create_driver(self, worker_number):
 180         return WebKitDriver(self, worker_number)
 181
 182     def test_base_platform_names(self):
 183         # At the moment we don't use test platform names, but we have
 184         # to return something.
 185         return ('mac', 'win')
 186
 187     def _tests_for_other_platforms(self):
 188         raise NotImplementedError('WebKitPort._tests_for_other_platforms')
 189         # The original run-webkit-tests builds up a "whitelist" of tests to
 190         # run, and passes that to DumpRenderTree. new-run-webkit-tests assumes
 191         # we run *all* tests and test_expectations.txt functions as a
 192         # blacklist.
 193         # FIXME: This list could be dynamic based on platform name and
 194         # pushed into base.Port.
 195         return [
 196             "platform/chromium",
 197             "platform/gtk",
 198             "platform/qt",
 199             "platform/win",
 200         ]
 201
 202     def _runtime_feature_list(self):
 203         """Return the supported features of DRT. If a port doesn't support
 204         this DRT switch, it has to override this method to return None"""
 205         driver_path = self._path_to_driver()
 206         feature_list = ' '.join(os.popen(driver_path + " --print-supported-features 2>&1").readlines())
 207         if "SupportedFeatures:" in feature_list:
 208             return feature_list
 209         return None
 210
 211     def _supported_symbol_list(self):
 212         """Return the supported symbols of WebCore."""
 213         webcore_library_path = self._path_to_webcore_library()
 214         if not webcore_library_path:
 215             return None
 216         symbol_list = ' '.join(os.popen("nm " + webcore_library_path).readlines())
 217         return symbol_list
 218
 219     def _directories_for_features(self):
 220         """Return the supported feature dictionary. The keys are the
 221         features and the values are the directories in lists."""
 222         directories_for_features = {
 223             "Accelerated Compositing": ["compositing"],
 224             "3D Rendering": ["animations/3d", "transforms/3d"],
 225         }
 226         return directories_for_features
 227
 228     def _directories_for_symbols(self):
 229         """Return the supported feature dictionary. The keys are the
 230         symbols and the values are the directories in lists."""
 231         directories_for_symbol = {
 232             "MathMLElement": ["mathml"],
 233             "GraphicsLayer": ["compositing"],
 234             "WebCoreHas3DRendering": ["animations/3d", "transforms/3d"],
 235             "WebGLShader": ["fast/canvas/webgl", "compositing/webgl", "http/tests/canvas/webgl"],
 236             "WMLElement": ["http/tests/wml", "fast/wml", "wml"],
 237             "parseWCSSInputProperty": ["fast/wcss"],
 238             "isXHTMLMPDocument": ["fast/xhtmlmp"],
 239         }
 240         return directories_for_symbol
 241
 242     def _skipped_tests_for_unsupported_features(self):
 243         """Return the directories of unsupported tests. Search for the
 244         symbols in the symbol_list, if found add the corresponding
 245         directories to the skipped directory list."""
 246         feature_list = self._runtime_feature_list()
 247         directories = self._directories_for_features()
 248
 249         # if DRT feature detection not supported
 250         if not feature_list:
 251             feature_list = self._supported_symbol_list()
 252             directories = self._directories_for_symbols()
 253
 254         if not feature_list:
 255             return []
 256
 257         skipped_directories = [directories[feature]
 258                               for feature in directories.keys()
 259                               if feature not in feature_list]
 260         return reduce(operator.add, skipped_directories)
 261
 262     def _tests_for_disabled_features(self):
 263         # FIXME: This should use the feature detection from
 264         # webkitperl/features.pm to match run-webkit-tests.
 265         # For now we hard-code a list of features known to be disabled on
 266         # the Mac platform.
 267         disabled_feature_tests = [
 268             "fast/xhtmlmp",
 269             "http/tests/wml",
 270             "mathml",
 271             "wml",
 272         ]
 273         # FIXME: webarchive tests expect to read-write from
 274         # -expected.webarchive files instead of .txt files.
 275         # This script doesn't know how to do that yet, so pretend they're
 276         # just "disabled".
 277         webarchive_tests = [
 278             "webarchive",
 279             "svg/webarchive",
 280             "http/tests/webarchive",
 281             "svg/custom/image-with-prefix-in-webarchive.svg",
 282         ]
 283         unsupported_feature_tests = self._skipped_tests_for_unsupported_features()
 284         return disabled_feature_tests + webarchive_tests + unsupported_feature_tests
 285
 286     def _tests_from_skipped_file(self, skipped_file):
 287         tests_to_skip = []
 288         for line in skipped_file.readlines():
 289             line = line.strip()
 290             if line.startswith('#') or not len(line):
 291                 continue
 292             tests_to_skip.append(line)
 293         return tests_to_skip
 294
 295     def _skipped_file_paths(self):
 296         return [self._filesystem.join(self._webkit_baseline_path(self._name), 'Skipped')]
 297
 298     def _expectations_from_skipped_files(self):
 299         tests_to_skip = []
 300         for filename in self._skipped_file_paths():
 301             if not self._filesystem.exists(filename):
 302                 _log.warn("Failed to open Skipped file: %s" % filename)
 303                 continue
 304             skipped_file = self._filesystem.read_text_file(filename)
 305         return tests_to_skip
 306
 307     def test_expectations(self):
 308         # The WebKit mac port uses a combination of a test_expectations file
 309         # and 'Skipped' files.
 310         expectations_path = self.path_to_test_expectations_file()
 311         return self._filesystem.read_text_file(expectations_path) + self._skips()
 312
 313     def _skips(self):
 314         # Each Skipped file contains a list of files
 315         # or directories to be skipped during the test run. The total list
 316         # of tests to skipped is given by the contents of the generic
 317         # Skipped file found in platform/X plus a version-specific file
 318         # found in platform/X-version. Duplicate entries are allowed.
 319         # This routine reads those files and turns contents into the
 320         # format expected by test_expectations.
 321
 322         tests_to_skip = self.skipped_layout_tests()
 323         skip_lines = map(lambda test_path: "BUG_SKIPPED SKIP : %s = FAIL" %
 324                                 test_path, tests_to_skip)
 325         return "\n".join(skip_lines)
 326
 327     def skipped_layout_tests(self):
 328         # Use a set to allow duplicates
 329         tests_to_skip = set(self._expectations_from_skipped_files())
 330         tests_to_skip.update(self._tests_for_other_platforms())
 331         tests_to_skip.update(self._tests_for_disabled_features())
 332         return tests_to_skip
 333
 334     def test_platform_name(self):
 335         return self._name + self.version()
 336
 337     def test_platform_names(self):
 338         return self.test_base_platform_names() + (
 339             'mac-tiger', 'mac-leopard', 'mac-snowleopard')
 340
 341     def _build_path(self, *comps):
 342         return self._filesystem.join(self._config.build_directory(
 343             self.get_option('configuration')), *comps)
 344
 345     def _path_to_driver(self):
 346         return self._build_path('DumpRenderTree')
 347
 348     def _path_to_webcore_library(self):
 349         return None
 350
 351     def _path_to_helper(self):
 352         return None
 353
 354     def _path_to_image_diff(self):
 355         return self._build_path('ImageDiff')
 356
 357     def _path_to_wdiff(self):
 358         # FIXME: This does not exist on a default Mac OS X Leopard install.
 359         return 'wdiff'
 360
 361     def _path_to_apache(self):
 362         if not self._cached_apache_path:
 363             # The Apache binary path can vary depending on OS and distribution
 364             # See http://wiki.apache.org/httpd/DistrosDefaultLayout
 365             for path in ["/usr/sbin/httpd", "/usr/sbin/apache2"]:
 366                 if self._filesystem.exists(path):
 367                     self._cached_apache_path = path
 368                     break
 369
 370             if not self._cached_apache_path:
 371                 _log.error("Could not find apache. Not installed or unknown path.")
 372
 373         return self._cached_apache_path
 374
 375
 376 class WebKitDriver(base.Driver):
 377     """WebKit implementation of the DumpRenderTree interface."""
 378
 379     def __init__(self, port, worker_number):
 380         self._worker_number = worker_number
 381         self._port = port
 382         self._driver_tempdir = port._filesystem.mkdtemp(prefix='DumpRenderTree-')
 383
 384     def __del__(self):
 385         self._port._filesystem.rmtree(str(self._driver_tempdir))
 386
 387     def cmd_line(self):
 388         cmd = self._command_wrapper(self._port.get_option('wrapper'))
 389         cmd += [self._port._path_to_driver(), '-']
 390
 391         if self._port.get_option('pixel_tests'):
 392             cmd.append('--pixel-tests')
 393
 394         return cmd
 395
 396     def start(self):
 397         environment = self._port.setup_environ_for_server()
 398         environment['DYLD_FRAMEWORK_PATH'] = self._port._build_path()
 399         environment['DUMPRENDERTREE_TEMP'] = str(self._driver_tempdir)
 400         self._server_process = server_process.ServerProcess(self._port,
 401             "DumpRenderTree", self.cmd_line(), environment)
 402
 403     def poll(self):
 404         return self._server_process.poll()
 405
 406     def restart(self):
 407         self._server_process.stop()
 408         self._server_process.start()
 409         return
 410
 411     # FIXME: This function is huge.
 412     def run_test(self, test_input):
 413         uri = self._port.filename_to_uri(test_input.filename)
 414         if uri.startswith("file:///"):
 415             command = uri[7:]
 416         else:
 417             command = uri
 418
 419         if test_input.image_hash:
 420             command += "'" + test_input.image_hash
 421         command += "\n"
 422
 423         start_time = time.time()
 424         self._server_process.write(command)
 425
 426         have_seen_content_type = False
 427         actual_image_hash = None
 428         output = str()  # Use a byte array for output, even though it should be UTF-8.
 429         image = str()
 430
 431         timeout = int(test_input.timeout) / 1000.0
 432         deadline = time.time() + timeout
 433         line = self._server_process.read_line(timeout)
 434         while (not self._server_process.timed_out
 435                and not self._server_process.crashed
 436                and line.rstrip() != "#EOF"):
 437             if (line.startswith('Content-Type:') and not
 438                 have_seen_content_type):
 439                 have_seen_content_type = True
 440             else:
 441                 # Note: Text output from DumpRenderTree is always UTF-8.
 442                 # However, some tests (e.g. webarchives) spit out binary
 443                 # data instead of text.  So to make things simple, we
 444                 # always treat the output as binary.
 445                 output += line
 446             line = self._server_process.read_line(timeout)
 447             timeout = deadline - time.time()
 448
 449         # Now read a second block of text for the optional image data
 450         remaining_length = -1
 451         HASH_HEADER = 'ActualHash: '
 452         LENGTH_HEADER = 'Content-Length: '
 453         line = self._server_process.read_line(timeout)
 454         while (not self._server_process.timed_out
 455                and not self._server_process.crashed
 456                and line.rstrip() != "#EOF"):
 457             if line.startswith(HASH_HEADER):
 458                 actual_image_hash = line[len(HASH_HEADER):].strip()
 459             elif line.startswith('Content-Type:'):
 460                 pass
 461             elif line.startswith(LENGTH_HEADER):
 462                 timeout = deadline - time.time()
 463                 content_length = int(line[len(LENGTH_HEADER):])
 464                 image = self._server_process.read(timeout, content_length)
 465             timeout = deadline - time.time()
 466             line = self._server_process.read_line(timeout)
 467
 468         error_lines = self._server_process.error.splitlines()
 469         # FIXME: This is a hack.  It is unclear why sometimes
 470         # we do not get any error lines from the server_process
 471         # probably we are not flushing stderr.
 472         if error_lines and error_lines[-1] == "#EOF":
 473             error_lines.pop()  # Remove the expected "#EOF"
 474         error = "\n".join(error_lines)
 475         # FIXME: This seems like the wrong section of code to be doing
 476         # this reset in.
 477         self._server_process.error = ""
 478         return test_output.TestOutput(output, image, actual_image_hash,
 479                                       self._server_process.crashed,
 480                                       time.time() - start_time,
 481                                       self._server_process.timed_out,
 482                                       error)
 483
 484     def stop(self):
 485         if self._server_process:
 486             self._server_process.stop()
 487             self._server_process = None