+2010-09-17 Dirk Pranke <dpranke@chromium.org>
+
+ Reviewed by Ojan Vafai.
+
+ new-run-webkit-tests: pull the list of tests from the Port, and
+ make it possible to run with no actual test files in the filesystem.
+
+ This set of changes allows us to delete the special
+ webkitpy/layout_tests/data directory and test the generic code without
+ touching the filesystem (except to write results). This speeds up
+ test-webkitpy substantially.
+
+ This also cleans up and removes several code paths in the generic
+ code and fixes some minor bugs, notably in the test_expectations
+ parsing, which was previously fragile if the tests weren't present.
+
+ We also change the way we diff image results to be able to do so
+ in memory, without relying on files. This slows down chromium
+ test_shell, which always writes images to files, but should speed
+ up DRT and ImageDiff-based implementations slightly.
+
+ Note that pulling the list of tests to run from the Port will allow
+ ports to specify what tests to run as an inclusive list; previously
+ you could only do this as an exclusive list by using a
+ test_expectations file or Skipped files; nothing actually uses this
+ feature yet and it's unclear if it should be used.
+
+ Note that there are no functional changes -- apart from now
+ always printing out the location of the checksum file when we are
+ tracing test runs -- and the total number of lines of non-test code
+ actually drops by two.
+
+ There is some more cleanup that can be done in the Port/Driver
+ interface and in some handling of filenames, but I'll save that
+ for another change.
+
+ https://bugs.webkit.org/show_bug.cgi?id=45801
+
+ * Scripts/webkitpy/layout_tests/data/*: Removed.
+ - no longer need special mock layout_tests_directory in the
+ repository.
+ * Scripts/webkitpy/layout_tests/layout_package/printing.py:
+ - add code to display missing text files, checksums when tracing
+ - update to not look at the filesystem directly.
+ * Scripts/webkitpy/layout_tests/layout_package/printing_unittest.py:
+ - add more unit tests
+ * Scripts/webkitpy/layout_tests/layout_package/test_expectations.py:
+ - remove tests_are_present flag
+ - update with changes in Port interface - no longer access
+ the filesystem directly, although we still use os.path for
+ filename manipulation.
+ * Scripts/webkitpy/layout_tests/layout_package/test_expectations_unittest.py:
+ - add more unit tests
+ * Scripts/webkitpy/layout_tests/layout_package/test_files.py: Removed.
+ - renamed to port/test_files.py
+ * Scripts/webkitpy/layout_tests/port/base.py:
+ - change diff_image() to expect actual image data, not filenames
+ - add expected_checksum(), expected_image(), expected_text() to
+ return the content of the files so that we don't need a filesystem
+ - add path_exists(), path_isdir() for filesystem-like access.
+ - add test_dirs() to keep clobber-test-results from needing to
+ actually look at a real layout tests dir
+ - add tests() to return the list of tests to run on the port
+ (calls port/test_files to do the work).
+ - add update_baseline() to be able to save new baselines
+ - add uri_to_test_name() from port/dryrun.py so we can easily check
+ filename_to_uri()
+ * Scripts/webkitpy/layout_tests/port/base_unittest.py:
+ - add more unit tests
+ * Scripts/webkitpy/layout_tests/port/chromium.py:
+ - change diff_image() to accept image content, not filenames.
+ This will be slower for test_shell/image_diff, but faster
+ for DRT/ImageDiff.
+ * Scripts/webkitpy/layout_tests/port/chromium_unittest.py:
+ - add more unit tests
+ * Scripts/webkitpy/layout_tests/port/dryrun.py:
+ - simplify greatly
+ * Scripts/webkitpy/layout_tests/port/port_testcase.py:
+ - add more unit tests
+ * Scripts/webkitpy/layout_tests/port/test.py:
+ - massive rewrite to use in-script list of tests and expectations
+ * Scripts/webkitpy/layout_tests/port/test_files.py:
+ - rename from layout_package/test_files.
+ * Scripts/webkitpy/layout_tests/port/test_files_unittest.py:
+ - add unit tests
+ * Scripts/webkitpy/layout_tests/port/webkit.py:
+ - update diff_image() to take image contents, not files. Should
+ make things slightly faster.
+ * Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests.py:
+ - update with changes to diff_image()
+ * Scripts/webkitpy/layout_tests/rebaseline_chromium_webkit_tests_unittest.py:
+ - update with changes to diff_image()
+ * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+ - remove tests_are_present from test_expectations
+ - pull the list of tests from port.tests() instead
+ of calling test_files.py directly.
+ * Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py:
+ - update unit tests
+ * Scripts/webkitpy/layout_tests/test_types/image_diff.py:
+ - update with changes to
+ * Scripts/webkitpy/layout_tests/test_types/test_type_base.py:
+ - update with change in Port interface
+ * Scripts/webkitpy/layout_tests/test_types/text_diff.py:
+ - update with change in Port interface
+ * Scripts/webkitpy/style/checkers/test_expectations.py:
+ - remove the tests_are_present flag
+
2010-09-21 Anders Carlsson <andersca@apple.com>
Disable logging.
+++ /dev/null
-checksum-checksum
+++ /dev/null
-checksum-png
+++ /dev/null
-checksum-txt
+++ /dev/null
-image_checksum
+++ /dev/null
-timeout-thread
+++ /dev/null
-image-checksum
+++ /dev/null
-image_failure
+++ /dev/null
-image_checksum-checksum
+++ /dev/null
-image_checksum-png
+++ /dev/null
-image_checksum-txt
+++ /dev/null
-image_checksum
+++ /dev/null
-missing_check-png
+++ /dev/null
-missing_check-txt
+++ /dev/null
-missing_image
+++ /dev/null
-missing_image
+++ /dev/null
-missing_text
+++ /dev/null
-text_failures-txt
+++ /dev/null
-text_failure
+++ /dev/null
-fail_checksum
+++ /dev/null
-fail_output
+++ /dev/null
-image-checksum
+++ /dev/null
-platform_image-generic-checksum
+++ /dev/null
-platform_image-generic-png
+++ /dev/null
-platform_image-generic-txt
+++ /dev/null
-platform_image
+++ /dev/null
-platform_image-checksum
+++ /dev/null
-platform_image-png
+++ /dev/null
-platform_image-txt
+++ /dev/null
-WONTFIX : failures/expected/checksum.html = IMAGE
-WONTFIX : failures/expected/crash.html = CRASH
-// This one actually passes because the checksums will match.
-WONTFIX : failures/expected/image.html = PASS
-WONTFIX : failures/expected/image_checksum.html = IMAGE
-WONTFIX : failures/expected/missing_check.html = MISSING PASS
-WONTFIX : failures/expected/missing_image.html = MISSING PASS
-WONTFIX : failures/expected/missing_text.html = MISSING PASS
-WONTFIX : failures/expected/text.html = TEXT
-WONTFIX : failures/expected/timeout.html = TIMEOUT
-WONTFIX SKIP : failures/expected/hang.html = TIMEOUT
-WONTFIX SKIP : failures/expected/keyboard.html = CRASH
-WONTFIX SKIP : failures/expected/exception.html = CRASH
+++ /dev/null
-This directory exists solely to make sure that when we gather the lists of
-tests, we skip over directories named 'resources'.
filename = result.filename
test_name = self._port.relative_test_filename(filename)
self._write('trace: %s' % test_name)
- self._write(' txt: %s' %
- self._port.relative_test_filename(
- self._port.expected_filename(filename, '.txt')))
+ txt_file = self._port.expected_filename(filename, '.txt')
+ if self._port.path_exists(txt_file):
+ self._write(' txt: %s' %
+ self._port.relative_test_filename(txt_file))
+ else:
+ self._write(' txt: <none>')
+ checksum_file = self._port.expected_filename(filename, '.checksum')
+ if self._port.path_exists(checksum_file):
+ self._write(' sum: %s' %
+ self._port.relative_test_filename(checksum_file))
+ else:
+ self._write(' sum: <none>')
png_file = self._port.expected_filename(filename, '.png')
- if os.path.exists(png_file):
+ if self._port.path_exists(png_file):
self._write(' png: %s' %
self._port.relative_test_filename(png_file))
else:
expectations = test_expectations.TestExpectations(
self._port, test_paths, expectations_str,
self._port.test_platform_name(), is_debug_mode=False,
- is_lint_mode=False, tests_are_present=False)
+ is_lint_mode=False)
rs = run_webkit_tests.ResultSummary(expectations, test_paths)
return test_paths, rs, expectations
self.assertFalse(err.empty())
printer, err, out = self.get_printer(['--print', 'trace-everything'])
+ result = self.get_result('passes/image.html')
+ printer.print_test_result(result, expected=True, exp_str='',
+ got_str='')
+ result = self.get_result('failures/expected/missing_text.html')
+ printer.print_test_result(result, expected=True, exp_str='',
+ got_str='')
+ result = self.get_result('failures/expected/missing_check.html')
+ printer.print_test_result(result, expected=True, exp_str='',
+ got_str='')
+ result = self.get_result('failures/expected/missing_image.html')
printer.print_test_result(result, expected=True, exp_str='',
got_str='')
self.assertFalse(err.empty())
TEST_LIST = "test_expectations.txt"
def __init__(self, port, tests, expectations, test_platform_name,
- is_debug_mode, is_lint_mode, tests_are_present=True,
- overrides=None):
+ is_debug_mode, is_lint_mode, overrides=None):
"""Loads and parses the test expectations given in the string.
Args:
port: handle to object containing platform-specific functionality
in the expectations
is_lint_mode: If True, just parse the expectations string
looking for errors.
- tests_are_present: whether the test files exist in the file
- system and can be probed for. This is useful for distinguishing
- test files from directories, and is needed by the LTTF
- dashboard, where the files aren't actually locally present.
overrides: test expectations that are allowed to override any
entries in |expectations|. This is used by callers
that need to manage two sets of expectations (e.g., upstream
"""
self._expected_failures = TestExpectationsFile(port, expectations,
tests, test_platform_name, is_debug_mode, is_lint_mode,
- tests_are_present=tests_are_present, overrides=overrides)
+ overrides=overrides)
# TODO(ojan): Allow for removing skipped tests when getting the list of
# tests to run, but not when getting metrics.
'flaky': FLAKY}
def __init__(self, port, expectations, full_test_list, test_platform_name,
- is_debug_mode, is_lint_mode, suppress_errors=False,
- tests_are_present=True, overrides=None):
+ is_debug_mode, is_lint_mode, suppress_errors=False, overrides=None):
"""
expectations: Contents of the expectations file
full_test_list: The list of all tests to be run pending processing of
is_debug_mode: Whether we testing a test_shell built debug mode.
is_lint_mode: Whether this is just linting test_expecatations.txt.
suppress_errors: Whether to suppress lint errors.
- tests_are_present: Whether the test files are present in the local
- filesystem. The LTTF Dashboard uses False here to avoid having to
- keep a local copy of the tree.
overrides: test expectations that are allowed to override any
entries in |expectations|. This is used by callers
that need to manage two sets of expectations (e.g., upstream
self._test_platform_name = test_platform_name
self._is_debug_mode = is_debug_mode
self._is_lint_mode = is_lint_mode
- self._tests_are_present = tests_are_present
self._overrides = overrides
self._suppress_errors = suppress_errors
self._errors = []
def remove_platform_from_expectations(self, tests, platform):
"""Returns a copy of the expectations with the tests matching the
- platform remove.
+ platform removed.
If a test is in the test list and has an option that matches the given
platform, remove the matching platform and save the updated test back
# WebKit's way of skipping tests is to add a -disabled suffix.
# So we should consider the path existing if the path or the
# -disabled version exists.
- if (self._tests_are_present and not os.path.exists(full_path)
- and not os.path.exists(full_path + '-disabled')):
+ if (not self._port.path_exists(full_path)
+ and not self._port.path_exists(full_path + '-disabled')):
# Log a non fatal error here since you hit this case any
# time you update test_expectations.txt without syncing
# the LayoutTests directory
path and make sure directories end with the OS path separator."""
path = os.path.join(self._port.layout_tests_dir(), test_list_path)
path = os.path.normpath(path)
- path = self._fix_dir(path)
+ if self._port.path_isdir(path):
+ path = os.path.join(path, '')
result = []
for test in self._full_test_list:
result.append(test)
return result
- def _fix_dir(self, path):
- """Check to see if the path points to a directory, and if so, append
- the directory separator if necessary."""
- if self._tests_are_present:
- if os.path.isdir(path):
- path = os.path.join(path, '')
- else:
- # If we can't check the filesystem to see if this is a directory,
- # we assume that files w/o an extension are directories.
- # TODO(dpranke): What happens w/ LayoutTests/css2.1 ?
- if os.path.splitext(path)[1] == '':
- path = os.path.join(path, '')
- return path
-
def _add_tests(self, tests, expectations, test_list_path, lineno,
modifiers, options, overrides_allowed):
for test in tests:
"""
def parse_exp(self, expectations, overrides=None, is_lint_mode=False,
- is_debug_mode=False, tests_are_present=True):
+ is_debug_mode=False):
self._exp = TestExpectations(self._port,
tests=self.get_basic_tests(),
expectations=expectations,
test_platform_name=self._port.test_platform_name(),
is_debug_mode=is_debug_mode,
is_lint_mode=is_lint_mode,
- tests_are_present=tests_are_present,
overrides=overrides)
def assert_exp(self, test, result):
self.assert_exp('failures/expected/text.html', TEXT)
self.assert_exp('failures/expected/crash.html', IMAGE)
- self.parse_exp(exp_str, tests_are_present=False)
- self.assert_exp('failures/expected/text.html', TEXT)
- self.assert_exp('failures/expected/crash.html', IMAGE)
-
def test_release_mode(self):
self.parse_exp('BUGX DEBUG : failures/expected/text.html = TEXT',
is_debug_mode=True)
import time
import apache_http_server
+import test_files
import http_server
import websocket_server
interface so that it can be overriden for testing purposes."""
return expected_text != actual_text
- def diff_image(self, expected_filename, actual_filename,
+ def diff_image(self, expected_contents, actual_contents,
diff_filename=None, tolerance=0):
- """Compare two image files and produce a delta image file.
+ """Compare two images and produce a delta image file.
- Return True if the two files are different, False if they are the same.
+ Return True if the two images are different, False if they are the same.
Also produce a delta image of the two images and write that into
|diff_filename| if it is not None.
return os.path.join(platform_dir, baseline_filename)
return os.path.join(self.layout_tests_dir(), baseline_filename)
+ def _expected_file_contents(self, test, extension, encoding):
+ path = self.expected_filename(test, extension)
+ if not os.path.exists(path):
+ return None
+ with codecs.open(path, 'r', encoding) as file:
+ return file.read()
+
+ def expected_checksum(self, test):
+ """Returns the checksum of the image we expect the test to produce, or None if it is a text-only test."""
+ return self._expected_file_contents(test, '.checksum', 'ascii')
+
+ def expected_image(self, test):
+ """Returns the image we expect the test to produce."""
+ return self._expected_file_contents(test, '.png', None)
+
+ def expected_text(self, test):
+ """Returns the text output we expect the test to produce."""
+ # NOTE: -expected.txt files are ALWAYS utf-8. However,
+ # we do not decode the output from DRT, so we should not
+ # decode the -expected.txt values either to allow comparisons.
+ text = self._expected_file_contents(test, '.txt', None)
+ if not text:
+ return ''
+ return text.strip("\r\n").replace("\r\n", "\n") + "\n"
+
def filename_to_uri(self, filename):
"""Convert a test file to a URI."""
LAYOUTTEST_HTTP_DIR = "http/tests/"
return "file:///" + self.get_absolute_path(filename)
return "file://" + self.get_absolute_path(filename)
+ def tests(self, paths):
+ """Return the list of tests found (relative to layout_tests_dir()."""
+ return test_files.find(self, paths)
+
+ def test_dirs(self):
+ """Returns the list of top-level test directories.
+
+ Used by --clobber-old-results."""
+ layout_tests_dir = self.layout_tests_dir()
+ return filter(lambda x: os.path.isdir(os.path.join(layout_tests_dir, x)),
+ os.listdir(layout_tests_dir))
+
+ def path_isdir(self, path):
+ """Returns whether the path refers to a directory of tests.
+
+ Used by test_expectations.py to apply rules to whole directories."""
+ return os.path.isdir(path)
+
+ def path_exists(self, path):
+ """Returns whether the path refers to an existing test or baseline."""
+ # Used by test_expectations.py to determine if an entry refers to a
+ # valid test and by printing.py to determine if baselines exist."""
+ return os.path.exists(path)
+
+ def update_baseline(self, path, data, encoding):
+ """Updates the baseline for a test.
+
+ Args:
+ path: the actual path to use for baseline, not the path to
+ the test. This function is used to update either generic or
+ platform-specific baselines, but we can't infer which here.
+ data: contents of the baseline.
+ encoding: file encoding to use for the baseline.
+ """
+ with codecs.open(path, "w", encoding=encoding) as file:
+ file.write(data)
+
+ def uri_to_test_name(self, uri):
+ """Return the base layout test name for a given URI.
+
+ This returns the test name for a given URI, e.g., if you passed in
+ "file:///src/LayoutTests/fast/html/keygen.html" it would return
+ "fast/html/keygen.html".
+
+ """
+ test = uri
+ if uri.startswith("file:///"):
+ if sys.platform == 'win32':
+ test = test.replace('file:///', '')
+ test = test.replace('/', '\\')
+ else:
+ test = test.replace('file://', '')
+ return self.relative_test_filename(test)
+
+ if uri.startswith("http://127.0.0.1:8880/"):
+ # websocket tests
+ return test.replace('http://127.0.0.1:8880/', '')
+
+ if uri.startswith("http://"):
+ # regular HTTP test
+ return test.replace('http://127.0.0.1:8000/', 'http/tests/')
+
+ if uri.startswith("https://"):
+ return test.replace('https://127.0.0.1:8443/', 'http/tests/')
+
+ raise NotImplementedError('unknown url type: %s' % uri)
+
def get_absolute_path(self, filename):
"""Return the absolute path in unix format for the given filename.
# This routine is a no-op. We just test it for coverage.
port.setup_test_run()
+ def test_test_dirs(self):
+ port = base.Port()
+ dirs = port.test_dirs()
+ self.assertTrue('canvas' in dirs)
+ self.assertTrue('css2.1' in dirs)
class VirtualTest(unittest.TestCase):
"""Tests that various methods expected to be virtual are."""
import http_server
from webkitpy.common.system.executive import Executive
-from webkitpy.layout_tests.layout_package import test_files
from webkitpy.layout_tests.layout_package import test_expectations
# Chromium DRT on OSX uses WebKitDriver.
return check_file_exists(image_diff_path, 'image diff exe',
override_step, logging)
- def diff_image(self, expected_filename, actual_filename,
+ def diff_image(self, expected_contents, actual_contents,
diff_filename=None, tolerance=0):
executable = self._path_to_image_diff()
+ expected_tmpfile = tempfile.NamedTemporaryFile()
+ expected_tmpfile.write(expected_contents)
+ actual_tmpfile = tempfile.NamedTemporaryFile()
+ actual_tmpfile.write(actual_contents)
if diff_filename:
- cmd = [executable, '--diff', expected_filename, actual_filename,
- diff_filename]
+ cmd = [executable, '--diff', expected_tmpfile.name,
+ actual_tmpfile.name, diff_filename]
else:
- cmd = [executable, expected_filename, actual_filename]
+ cmd = [executable, expected_tmpfile.name, actual_tmpfile.name]
result = True
try:
_compare_available = False
else:
raise e
+ finally:
+ expected_tmpfile.close()
+ actual_tmpfile.close()
return result
def driver_name(self):
test_platform_name = self.test_platform_name()
is_debug_mode = False
- all_test_files = test_files.gather_test_files(self, '*')
+ all_test_files = self.tests([])
if extra_test_files:
all_test_files.update(extra_test_files)
expectations = test_expectations.TestExpectations(
self, all_test_files, expectations_str, test_platform_name,
- is_debug_mode, is_lint_mode=True,
- tests_are_present=False, overrides=overrides_str)
+ is_debug_mode, is_lint_mode=True, overrides=overrides_str)
tests_dir = self.layout_tests_dir()
return [self.relative_test_filename(test)
for test in expectations.get_tests_with_result_type(test_expectations.SKIP)]
port.test_expectations = lambda: """BUG_TEST SKIP : fast/js/not-good.js = TEXT
DEFER LINUX WIN : fast/js/very-good.js = TIMEOUT PASS"""
port.test_expectations_overrides = lambda: ''
+ port.tests = lambda paths: set()
+ port.path_exists = lambda test: True
skipped_tests = port.skipped_layout_tests(extra_test_files=[fake_test, ])
self.assertTrue("fast/js/not-good.js" in skipped_tests)
from __future__ import with_statement
+import os
import sys
import base
import factory
-def _read_file(path, mode='r'):
- """Return the contents of a file as a string.
-
- Returns '' if anything goes wrong, instead of throwing an IOError.
-
- """
- contents = ''
- try:
- with open(path, mode) as f:
- contents = f.read()
- except IOError:
- pass
- return contents
-
-
-def _write_file(path, contents, mode='w'):
- """Write the string to the specified path.
-
- Writes should never fail, so we may raise IOError.
-
- """
- with open(path, mode) as f:
- f.write(contents)
-
-
class DryRunPort(object):
"""DryRun implementation of the Port interface."""
return None
def run_test(self, uri, timeoutms, image_hash):
- test_name = self._uri_to_test(uri)
-
- text_filename = self._port.expected_filename(test_name, '.txt')
- text_output = _read_file(text_filename)
+ test_name = self._port.uri_to_test_name(uri)
+ path = os.path.join(self._port.layout_tests_dir(), test_name)
+ text_output = self._port.expected_text(path)
if image_hash is not None:
- image_filename = self._port.expected_filename(test_name, '.png')
- image = _read_file(image_filename, 'rb')
- if self._image_path:
- _write_file(self._image_path, image)
- hash_filename = self._port.expected_filename(test_name,
- '.checksum')
- hash = _read_file(hash_filename)
+ image = self._port.expected_image(path)
+ if image and self._image_path:
+ with open(self._image_path, 'w') as f:
+ f.write(image)
+ hash = self._port.expected_checksum(path)
else:
hash = None
return (False, False, hash, text_output, None)
def stop(self):
pass
-
- def _uri_to_test(self, uri):
- """Return the base layout test name for a given URI.
-
- This returns the test name for a given URI, e.g., if you passed in
- "file:///src/LayoutTests/fast/html/keygen.html" it would return
- "fast/html/keygen.html".
-
- """
- if not self._layout_tests_dir:
- self._layout_tests_dir = self._port.layout_tests_dir()
- test = uri
-
- if uri.startswith("file:///"):
- if sys.platform == 'win32':
- test = test.replace('file:///', '')
- test = test.replace('/', '\\')
- else:
- test = test.replace('file://', '')
- return test
- elif uri.startswith("http://127.0.0.1:8880/"):
- # websocket tests
- test = test.replace('http://127.0.0.1:8880/',
- self._layout_tests_dir + '/')
- return test
- elif uri.startswith("http://"):
- # regular HTTP test
- test = test.replace('http://127.0.0.1:8000/',
- self._layout_tests_dir + '/http/tests/')
- return test
- elif uri.startswith("https://"):
- test = test.replace('https://127.0.0.1:8443/',
- self._layout_tests_dir + '/http/tests/')
- return test
- else:
- raise NotImplementedError('unknown url type: %s' % uri)
dir = port.layout_tests_dir()
file1 = os.path.join(dir, 'fast', 'css', 'button_center.png')
+ fh1 = file(file1)
+ contents1 = fh1.read()
file2 = os.path.join(dir, 'fast', 'css',
'remove-shorthand-expected.png')
+ fh2 = file(file2)
+ contents2 = fh2.read()
tmpfile = tempfile.mktemp()
- self.assertFalse(port.diff_image(file1, file1))
- self.assertTrue(port.diff_image(file1, file2))
+ self.assertFalse(port.diff_image(contents1, contents1))
+ self.assertTrue(port.diff_image(contents1, contents2))
- self.assertTrue(port.diff_image(file1, file2, tmpfile))
+ self.assertTrue(port.diff_image(contents1, contents2, tmpfile))
+ fh1.close()
+ fh2.close()
# FIXME: this may not be being written?
# self.assertTrue(os.path.exists(tmpfile))
# os.remove(tmpfile)
from __future__ import with_statement
import codecs
+import fnmatch
import os
+import sys
import time
import base
+# This sets basic expectations for a test. Each individual expectation
+# can be overridden by a keyword argument in TestList.add().
+class TestInstance:
+ def __init__(self, name):
+ self.name = name
+ self.base = name[(name.rfind("/") + 1):name.rfind(".html")]
+ self.crash = False
+ self.exception = False
+ self.hang = False
+ self.keyboard = False
+ self.error = ''
+ self.timeout = False
+ self.actual_text = self.base + '-txt\n'
+ self.actual_checksum = self.base + '-checksum\n'
+ self.actual_image = self.base + '-png\n'
+ self.expected_text = self.actual_text
+ self.expected_checksum = self.actual_checksum
+ self.expected_image = self.actual_image
+
+
+# This is an in-memory list of tests, what we want them to produce, and
+# what we want to claim are the expected results.
+class TestList:
+ def __init__(self, port):
+ self.port = port
+ self.tests = {}
+
+ def add(self, name, **kwargs):
+ test = TestInstance(name)
+ for key, value in kwargs.items():
+ test.__dict__[key] = value
+ self.tests[name] = test
+
+ def keys(self):
+ return self.tests.keys()
+
+ def __contains__(self, item):
+ return item in self.tests
+
+ def __getitem__(self, item):
+ return self.tests[item]
+
+
class TestPort(base.Port):
"""Test implementation of the Port interface."""
def __init__(self, port_name=None, options=None):
base.Port.__init__(self, port_name, options)
+ tests = TestList(self)
+ tests.add('passes/image.html')
+ tests.add('passes/text.html')
+ tests.add('failures/expected/checksum.html',
+ actual_checksum='checksum_fail-checksum')
+ tests.add('failures/expected/crash.html', crash=True)
+ tests.add('failures/expected/exception.html', exception=True)
+ tests.add('failures/expected/timeout.html', timeout=True)
+ tests.add('failures/expected/hang.html', hang=True)
+ tests.add('failures/expected/missing_text.html',
+ expected_text=None)
+ tests.add('failures/expected/image.html',
+ actual_image='image_fail-png',
+ expected_image='image-png')
+ tests.add('failures/expected/image_checksum.html',
+ actual_checksum='image_checksum_fail-checksum',
+ actual_image='image_checksum_fail-png')
+ tests.add('failures/expected/keyboard.html',
+ keyboard=True)
+ tests.add('failures/expected/missing_check.html',
+ expected_checksum=None)
+ tests.add('failures/expected/missing_image.html',
+ expected_image=None)
+ tests.add('failures/expected/missing_text.html',
+ expected_text=None)
+ tests.add('failures/expected/text.html',
+ actual_text='text_fail-png')
+ tests.add('failures/unexpected/text-image-checksum.html',
+ actual_text='text-image-checksum_fail-txt',
+ actual_checksum='text-image-checksum_fail-checksum')
+ tests.add('http/tests/passes/text.html')
+ tests.add('http/tests/ssl/text.html')
+ tests.add('passes/error.html', error='stuff going to stderr')
+ tests.add('passes/image.html')
+ tests.add('passes/platform_image.html')
+ tests.add('passes/text.html')
+ tests.add('websocket/tests/passes/text.html')
+ self._tests = tests
def baseline_path(self):
return os.path.join(self.layout_tests_dir(), 'platform',
def check_build(self, needs_http):
return True
- def diff_image(self, expected_filename, actual_filename,
+ def diff_image(self, expected_contents, actual_contents,
diff_filename=None, tolerance=0):
- with codecs.open(actual_filename, "r", "utf-8") as actual_fh:
- actual_contents = actual_fh.read()
- with codecs.open(expected_filename, "r", "utf-8") as expected_fh:
- expected_contents = expected_fh.read()
diffed = actual_contents != expected_contents
if diffed and diff_filename:
with codecs.open(diff_filename, "w", "utf-8") as diff_fh:
(expected_contents, actual_contents))
return diffed
+ def expected_checksum(self, test):
+ test = self.relative_test_filename(test)
+ return self._tests[test].expected_checksum
+
+ def expected_image(self, test):
+ test = self.relative_test_filename(test)
+ return self._tests[test].expected_image
+
+ def expected_text(self, test):
+ test = self.relative_test_filename(test)
+ text = self._tests[test].expected_text
+ if not text:
+ text = ''
+ return text
+
+ def tests(self, paths):
+ # Test the idea of port-specific overrides for test lists. Also
+ # keep in memory to speed up the test harness.
+ if not paths:
+ paths = ['*']
+
+ matched_tests = []
+ for p in paths:
+ if self.path_isdir(p):
+ matched_tests.extend(fnmatch.filter(self._tests.keys(), p + '*'))
+ else:
+ matched_tests.extend(fnmatch.filter(self._tests.keys(), p))
+ layout_tests_dir = self.layout_tests_dir()
+ return set([os.path.join(layout_tests_dir, p) for p in matched_tests])
+
+ def path_exists(self, path):
+ # used by test_expectations.py and printing.py
+ rpath = self.relative_test_filename(path)
+ if rpath in self._tests:
+ return True
+ if self.path_isdir(rpath):
+ return True
+ if rpath.endswith('-expected.txt'):
+ test = rpath.replace('-expected.txt', '.html')
+ return (test in self._tests and
+ self._tests[test].expected_text)
+ if rpath.endswith('-expected.checksum'):
+ test = rpath.replace('-expected.checksum', '.html')
+ return (test in self._tests and
+ self._tests[test].expected_checksum)
+ if rpath.endswith('-expected.png'):
+ test = rpath.replace('-expected.png', '.html')
+ return (test in self._tests and
+ self._tests[test].expected_image)
+ return False
+
def layout_tests_dir(self):
return self.path_from_webkit_base('WebKitTools', 'Scripts',
'webkitpy', 'layout_tests', 'data')
+ def path_isdir(self, path):
+ # Used by test_expectations.py
+ #
+ # We assume that a path is a directory if we have any tests that
+ # whose prefix matches the path plus a directory modifier.
+ if path[-1] != '/':
+ path += '/'
+ return any([t.startswith(path) for t in self._tests.keys()])
+
+ def test_dirs(self):
+ return ['passes', 'failures']
+
def name(self):
return self._name
def options(self):
return self._options
- def skipped_layout_tests(self):
- return []
-
- def path_to_test_expectations_file(self):
- return self.path_from_webkit_base('WebKitTools', 'Scripts',
- 'webkitpy', 'layout_tests', 'data', 'platform', 'test',
- 'test_expectations.txt')
-
def _path_to_wdiff(self):
return None
Basically this string should contain the equivalent of a
test_expectations file. See test_expectations.py for more details."""
- expectations_path = self.path_to_test_expectations_file()
- with codecs.open(expectations_path, "r", "utf-8") as file:
- return file.read()
+ return """
+WONTFIX : failures/expected/checksum.html = IMAGE
+WONTFIX : failures/expected/crash.html = CRASH
+// This one actually passes because the checksums will match.
+WONTFIX : failures/expected/image.html = PASS
+WONTFIX : failures/expected/image_checksum.html = IMAGE
+WONTFIX : failures/expected/missing_check.html = MISSING PASS
+WONTFIX : failures/expected/missing_image.html = MISSING PASS
+WONTFIX : failures/expected/missing_text.html = MISSING PASS
+WONTFIX : failures/expected/text.html = TEXT
+WONTFIX : failures/expected/timeout.html = TIMEOUT
+WONTFIX SKIP : failures/expected/hang.html = TIMEOUT
+WONTFIX SKIP : failures/expected/keyboard.html = CRASH
+WONTFIX SKIP : failures/expected/exception.html = CRASH
+"""
def test_base_platform_names(self):
return ('mac', 'win')
return True
def run_test(self, uri, timeoutms, image_hash):
- basename = uri[(uri.rfind("/") + 1):uri.rfind(".html")]
-
- if 'error' in basename:
- error = basename + "_error\n"
- else:
- error = ''
- checksum = None
- # There are four currently supported types of tests: text, image,
- # image hash (checksum), and stderr output. The fake output
- # is the basename of the file + "-" plus the type of test output
- # (or a blank string for stderr).
- #
- # If 'image' or 'check' appears in the basename, we assume this is
- # simulating a pixel test.
- #
- # If 'failures' appears in the URI, then we assume this test should
- # fail. Which type of failures are determined by which strings appear
- # in the basename of the test. For failures that produce outputs,
- # we change the fake output to basename + "_failed-".
- #
- # The fact that each test produces (more or less) unique output data
- # will allow us to see if any results get crossed by the rest of the
- # program.
- if 'failures' in uri:
- if 'keyboard' in basename:
- raise KeyboardInterrupt
- if 'exception' in basename:
- raise ValueError('exception from ' + basename)
-
- crash = 'crash' in basename
- timeout = 'timeout' in basename or 'hang' in basename
- timeout = 'timeout' in basename
- if 'text' in basename:
- output = basename + '_failed-txt\n'
- else:
- output = basename + '-txt\n'
- if self._port.options().pixel_tests:
- if ('image' in basename or 'check' in basename):
- checksum = basename + "-checksum\n"
-
- if 'image' in basename:
- with open(self._image_path, "w") as f:
- f.write(basename + "_failed-png\n")
- elif 'check' in basename:
- with open(self._image_path, "w") as f:
- f.write(basename + "-png\n")
- if 'checksum' in basename:
- checksum = basename + "_failed-checksum\n"
-
- if 'hang' in basename:
- time.sleep((float(timeoutms) * 4) / 1000.0)
- else:
- crash = False
- timeout = False
- output = basename + '-txt\n'
- if self._options.pixel_tests and (
- 'image' in basename or 'check' in basename):
- checksum = basename + '-checksum\n'
- with open(self._image_path, "w") as f:
- f.write(basename + "-png")
-
- return (crash, timeout, checksum, output, error)
+ test_name = self._port.uri_to_test_name(uri)
+ test = self._port._tests[test_name]
+ if test.keyboard:
+ raise KeyboardInterrupt
+ if test.exception:
+ raise ValueError('exception from ' + test_name)
+ if test.hang:
+ time.sleep((float(timeoutms) * 4) / 1000.0)
+
+ if self._port.options().pixel_tests and test.actual_image:
+ with open(self._image_path, 'w') as file:
+ file.write(test.actual_image)
+
+ return (test.crash, test.timeout, test.actual_checksum,
+ test.actual_text, test.error)
def start(self):
pass
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-"""This module is used to find all of the layout test files used by Chromium
-(across all platforms). It exposes one public function - GatherTestFiles() -
+"""This module is used to find all of the layout test files used by
+run-webkit-tests. It exposes one public function - find() -
which takes an optional list of paths. If a list is passed in, the returned
list of test files is constrained to those found under the paths passed in,
-i.e. calling GatherTestFiles(["LayoutTests/fast"]) will only return files
+i.e. calling find(["LayoutTests/fast"]) will only return files
under that directory."""
import glob
_skipped_directories = set(['.svn', '_svn', 'resources', 'script-tests'])
-def gather_test_files(port, paths):
- """Generate a set of test files and return them.
+def find(port, paths):
+ """Finds the set of tests under port.layout_tests_dir().
Args:
- paths: a list of command line paths relative to the webkit/tests
- directory. glob patterns are ok.
+ paths: a list of command line paths relative to the layout_tests_dir()
+ to limit the search to. glob patterns are ok.
"""
gather_start_time = time.time()
paths_to_walk = set()
--- /dev/null
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import unittest
+
+import base
+import test_files
+
+
+class TestFilesTest(unittest.TestCase):
+ def test_find_no_paths_specified(self):
+ port = base.Port()
+ layout_tests_dir = port.layout_tests_dir()
+ port.layout_tests_dir = lambda: os.path.join(layout_tests_dir,
+ 'fast', 'html')
+ tests = test_files.find(port, [])
+ self.assertNotEqual(tests, 0)
+
+ def test_find_one_test(self):
+ port = base.Port()
+ # This is just a test picked at random but known to exist.
+ tests = test_files.find(port, ['fast/html/keygen.html'])
+ self.assertEqual(len(tests), 1)
+
+ def test_find_glob(self):
+ port = base.Port()
+ tests = test_files.find(port, ['fast/html/key*'])
+ self.assertEqual(len(tests), 1)
+
+ def test_find_with_skipped_directories(self):
+ port = base.Port()
+ tests = port.tests('userscripts')
+ self.assertTrue('userscripts/resources/frame1.html' not in tests)
+
+ def test_find_with_skipped_directories_2(self):
+ port = base.Port()
+ tests = test_files.find(port, ['userscripts/resources'])
+ self.assertEqual(tests, set([]))
+
+
+if __name__ == '__main__':
+ unittest.main()
return False
return True
- def diff_image(self, expected_filename, actual_filename,
+ def diff_image(self, expected_contents, actual_contents,
diff_filename=None, tolerance=0.1):
"""Return True if the two files are different. Also write a delta
image of the two images into |diff_filename| if it is not None."""
# parameter, or make it go away and always use exact matches.
# Handle the case where the test didn't actually generate an image.
- actual_length = os.stat(actual_filename).st_size
- if actual_length == 0:
- if diff_filename:
- shutil.copyfile(actual_filename, expected_filename)
+ if not actual_contents:
return True
- sp = self._diff_image_request(expected_filename, actual_filename, tolerance)
- return self._diff_image_reply(sp, expected_filename, diff_filename)
+ sp = self._diff_image_request(expected_contents, actual_contents,
+ tolerance)
+ return self._diff_image_reply(sp, diff_filename)
- def _diff_image_request(self, expected_filename, actual_filename, tolerance):
+ def _diff_image_request(self, expected_contents, actual_contents, tolerance):
command = [self._path_to_image_diff(), '--tolerance', str(tolerance)]
sp = server_process.ServerProcess(self, 'ImageDiff', command)
- actual_length = os.stat(actual_filename).st_size
- with open(actual_filename) as file:
- actual_file = file.read()
- expected_length = os.stat(expected_filename).st_size
- with open(expected_filename) as file:
- expected_file = file.read()
sp.write('Content-Length: %d\n%sContent-Length: %d\n%s' %
- (actual_length, actual_file, expected_length, expected_file))
+ (len(actual_contents), actual_contents,
+ len(expected_contents), expected_contents))
return sp
- def _diff_image_reply(self, sp, expected_filename, diff_filename):
+ def _diff_image_reply(self, sp, diff_filename):
timeout = 2.0
deadline = time.time() + timeout
output = sp.read_line(timeout)
with open(diff_filename, 'w') as file:
file.write(output)
elif sp.timed_out:
- _log.error("ImageDiff timed out on %s" % expected_filename)
+ _log.error("ImageDiff timed out")
elif sp.crashed:
_log.error("ImageDiff crashed")
sp.stop()
pass
def show_results_html_file(self, results_filename):
- uri = self.filename_to_uri(results_filename)
# FIXME: We should open results in the version of WebKit we built.
- webbrowser.open(uri, new=1)
+ webbrowser.open(results_filename, new=1)
def create_driver(self, image_path, options):
return WebKitDriver(self, image_path, options,
fallback_fullpath = os.path.normpath(
os.path.join(fallback_dir, fallback_file))
if fallback_fullpath.lower() != baseline_path.lower():
+ with codecs.open(file1, "r", "utf8") as file_handle1:
+ output1 = file_handle1.read()
+ with codecs.open(file2, "r", "utf8") as file_handle2:
+ output2 = file_handle2.read()
if not self._diff_baselines(new_baseline,
fallback_fullpath):
_log.info(' Found same baseline at %s',
return False
- def _diff_baselines(self, file1, file2):
+ def _diff_baselines(self, output1, output2, is_image):
"""Check whether two baselines are different.
Args:
- file1, file2: full paths of the baselines to compare.
+ output1, output2: contents of the baselines to compare.
Returns:
True if two files are different or have different extensions.
False otherwise.
"""
- ext1 = os.path.splitext(file1)[1].upper()
- ext2 = os.path.splitext(file2)[1].upper()
- if ext1 != ext2:
- _log.warn('Files to compare have different ext. '
- 'File1: %s; File2: %s', file1, file2)
- return True
-
- if ext1 == '.PNG':
- return self._port.diff_image(file1, file2)
+ if is_image:
+ return self._port.diff_image(output1, output2)
else:
- with codecs.open(file1, "r", "utf8") as file_handle1:
- output1 = file_handle1.read()
- with codecs.open(file2, "r", "utf8") as file_handle2:
- output2 = file_handle2.read()
return self._port.compare_text(output1, output2)
def _delete_baseline(self, filename):
def test_diff_baselines_txt(self):
rebaseliner = self.make_rebaseliner()
- path = os.path.join(rebaseliner._port.layout_tests_dir(),
- "passes", "text-expected.txt")
- self.assertFalse(rebaseliner._diff_baselines(path, path))
+ output = rebaseliner._port.expected_text(
+ os.path.join(rebaseliner._port.layout_tests_dir(),
+ 'passes/text.html'))
+ self.assertFalse(rebaseliner._diff_baselines(output, output,
+ is_image=False))
def test_diff_baselines_png(self):
+ return
rebaseliner = self.make_rebaseliner()
- path = os.path.join(rebaseliner._port.layout_tests_dir(),
- "passes", "image-expected.png")
- self.assertFalse(rebaseliner._diff_baselines(path, path))
+ image = rebaseliner._port.expected_image(
+ os.path.join(rebaseliner._port.layout_tests_dir(),
+ 'passes/image.html'))
+ self.assertFalse(rebaseliner._diff_baselines(image, image,
+ is_image=True))
if __name__ == '__main__':
unittest.main()
from layout_package import printing
from layout_package import test_expectations
from layout_package import test_failures
-from layout_package import test_files
from layout_package import test_results_uploader
from test_types import image_diff
from test_types import text_diff
timeout: Timeout for running the test in TestShell.
"""
self.filename = filename
+ self._port = port
self.uri = port.filename_to_uri(filename)
self.timeout = timeout
- # FIXME: Confusing that the file is .checksum and we call it "hash"
- self._expected_hash_path = port.expected_filename(filename, '.checksum')
- self._have_read_expected_hash = False
- self._image_hash = None
-
- def _read_image_hash(self):
- if not os.path.exists(self._expected_hash_path):
- return None
-
- with codecs.open(self._expected_hash_path, "r", "ascii") as hash_file:
- return hash_file.read()
+ self._image_checksum = -1
def image_hash(self):
# Read the image_hash lazily to reduce startup time.
# This class is accessed across threads, but only one thread should
# ever be dealing with any given TestInfo so no locking is needed.
- if not self._have_read_expected_hash:
- self._have_read_expected_hash = True
- self._image_hash = self._read_image_hash()
- return self._image_hash
+ #
+ # Note that we use -1 to indicate that we haven't read the value,
+ # because expected_checksum() returns a string or None.
+ if self._image_checksum == -1:
+ self._image_checksum = self._port.expected_checksum(self.filename)
+ return self._image_checksum
class ResultSummary(object):
paths += last_unexpected_results
if self._options.test_list:
paths += read_test_files(self._options.test_list)
- self._test_files = test_files.gather_test_files(self._port, paths)
+ self._test_files = self._port.tests(paths)
def lint(self):
# Creating the expecations for each platform/configuration pair does
self._expectations = test_expectations.TestExpectations(
self._port, test_files, expectations_str, test_platform_name,
is_debug_mode, self._options.lint_test_files,
- tests_are_present=True, overrides=overrides_str)
+ overrides=overrides_str)
return self._expectations
except SyntaxError, err:
if self._options.lint_test_files:
self._printer.print_update("Clobbering old results in %s" %
self._options.results_directory)
layout_tests_dir = self._port.layout_tests_dir()
- possible_dirs = os.listdir(layout_tests_dir)
+ possible_dirs = self._port.test_dirs()
for dirname in possible_dirs:
if os.path.isdir(os.path.join(layout_tests_dir, dirname)):
shutil.rmtree(os.path.join(self._options.results_directory,
new_args.extend(args)
if not tests_included:
new_args.extend(['passes',
- 'http/tests'
+ 'http/tests',
'websocket/tests',
'failures/expected/*'])
options, parsed_args = run_webkit_tests.parse_args(new_args)
def test_test(self):
res, out, err = logging_run(['--platform', 'dryrun-test',
'--pixel-tests'])
- self.assertEqual(res, 2)
+ self.assertEqual(res, 0)
self.assertFalse(out.empty())
self.assertFalse(err.empty())
expected_filename = self.output_filename(filename,
self.FILENAME_SUFFIX_EXPECTED + '.png')
- result = port.diff_image(expected_filename, actual_filename,
+ expected_image = port.expected_image(filename)
+ with codecs.open(actual_filename, 'r', None) as file:
+ actual_image = file.read()
+
+ result = port.diff_image(expected_image, actual_image,
diff_filename)
return result
return failures
# Compare hashes.
- expected_hash_file = self._port.expected_filename(filename,
- '.checksum')
- expected_png_file = self._port.expected_filename(filename, '.png')
-
- # FIXME: We repeat this pattern often, we should share code.
- expected_hash = ''
- if os.path.exists(expected_hash_file):
- with codecs.open(expected_hash_file, "r", "ascii") as file:
- expected_hash = file.read()
+ expected_hash = self._port.expected_checksum(filename)
+ expected_png = self._port.expected_image(filename)
- if not os.path.isfile(expected_png_file):
+ if not expected_png:
# Report a missing expected PNG file.
- self.write_output_files(port, filename, '.checksum',
+ self.write_output_files(filename, '.checksum',
test_args.hash, expected_hash,
encoding="ascii",
print_text_diffs=False)
# Hash matched (no diff needed, okay to return).
return failures
- self.write_output_files(port, filename, '.checksum',
+ self.write_output_files(filename, '.checksum',
test_args.hash, expected_hash,
encoding="ascii",
print_text_diffs=False)
+
+ # FIXME: combine next two lines
self._copy_output_png(filename, test_args.png_path, '-actual.png')
- self._copy_output_png(filename, expected_png_file, '-expected.png')
+ self.write_output_files(filename, '.png', output=None,
+ expected=expected_png,
+ encoding=None, print_text_diffs=False)
# Even though we only use the result in one codepath below but we
# still need to call CreateImageDiff for other codepaths.
images_are_different = self._create_image_diff(port, filename, configuration)
- if expected_hash == '':
+ if not expected_hash:
failures.append(test_failures.FailureMissingImageHash())
elif test_args.hash != expected_hash:
if images_are_different:
output_path = self._port.expected_filename(filename, modifier)
_log.debug('resetting baseline result "%s"' % output_path)
- self._write_into_file_at_path(output_path, data, encoding)
+ self._port.update_baseline(output_path, data, encoding)
def output_filename(self, filename, modifier):
"""Returns a filename inside the output dir that contains modifier.
with codecs.open(file_path, "w", encoding=encoding) as file:
file.write(contents)
- def write_output_files(self, port, filename, file_type,
+ def write_output_files(self, filename, file_type,
output, expected, encoding,
print_text_diffs=False):
"""Writes the test output, the expected output and optionally the diff
# Note: We pass encoding=None for all diff writes, as we treat diff
# output as binary. Diff output may contain multiple files in
# conflicting encodings.
- diff = port.diff_text(expected, output, expected_filename, actual_filename)
+ diff = self._port.diff_text(expected, output, expected_filename, actual_filename)
diff_filename = self.output_filename(filename, self.FILENAME_SUFFIX_DIFF + file_type)
self._write_into_file_at_path(diff_filename, diff, encoding=None)
# Shell out to wdiff to get colored inline diffs.
- wdiff = port.wdiff_text(expected_filename, actual_filename)
+ wdiff = self._port.wdiff_text(expected_filename, actual_filename)
wdiff_filename = self.output_filename(filename, self.FILENAME_SUFFIX_WDIFF)
self._write_into_file_at_path(wdiff_filename, wdiff, encoding=None)
# Use WebKit's PrettyPatch.rb to get an HTML diff.
- pretty_patch = port.pretty_patch_text(diff_filename)
+ pretty_patch = self._port.pretty_patch_text(diff_filename)
pretty_patch_filename = self.output_filename(filename, self.FILENAME_SUFFIX_PRETTY_PATCH)
self._write_into_file_at_path(pretty_patch_filename, pretty_patch, encoding=None)
"""Given the filename of the test, read the expected output from a file
and normalize the text. Returns a string with the expected text, or ''
if the expected output file was not found."""
- # Read the port-specific expected text.
- expected_filename = self._port.expected_filename(filename, '.txt')
- return self._get_normalized_text(expected_filename)
-
- def _get_normalized_text(self, filename):
- # FIXME: We repeat this pattern often, we should share code.
- if not os.path.exists(filename):
- return ''
-
- # NOTE: -expected.txt files are ALWAYS utf-8. However,
- # we do not decode the output from DRT, so we should not
- # decode the -expected.txt values either to allow comparisons.
- with codecs.open(filename, "r", encoding=None) as file:
- text = file.read()
- # We could assert that the text is valid utf-8.
-
- # Normalize line endings
- return text.strip("\r\n").replace("\r\n", "\n") + "\n"
+ return self._port.expected_text(filename)
def compare_output(self, port, filename, output, test_args, configuration):
"""Implementation of CompareOutput that checks the output text against
# Write output files for new tests, too.
if port.compare_text(output, expected):
# Text doesn't match, write output files.
- self.write_output_files(port, filename, ".txt", output,
+ self.write_output_files(filename, ".txt", output,
expected, encoding=None,
print_text_diffs=True)
expectations = test_expectations.TestExpectationsFile(
port=self._port_obj, expectations=expectations_str, full_test_list=tests,
test_platform_name=self._port_to_check, is_debug_mode=False,
- is_lint_mode=True, suppress_errors=False, tests_are_present=True,
- overrides=overrides)
+ is_lint_mode=True, suppress_errors=False, overrides=overrides)
except SyntaxError, error:
errors = str(error).splitlines()
git://git.webkit.org / WebKit.git / commitdiff
