2010-09-01 Dirk Pranke <dpranke@chromium.org>
authordpranke@chromium.org <dpranke@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 8 Sep 2010 00:46:34 +0000 (00:46 +0000)
committerdpranke@chromium.org <dpranke@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 8 Sep 2010 00:46:34 +0000 (00:46 +0000)
        Reviewed by Ojan Vafai.

        new-run-webkit-tests: still more unit tests

        Clean up and remove unnecessary code. Biggest notable change is
        moving the chromium-specific imagediff code from port/base to
        port/chromium.

        Add more unit tests for run_webkit_tests.py, port/base.py,
        port/factory.py, port/dryrun.py, and
        layout_package/dump_render_tree_thread.py

        This covers almost all of the generic and test code paths except for
        a few error paths involving invalid or port-specific command line
        arguments, and the code path for uploading results files to the
        buildbots.

        https://bugs.webkit.org/show_bug.cgi?id=45090

        * Scripts/webkitpy/layout_tests/data/failures/expected/hang.html: Added.
        * Scripts/webkitpy/layout_tests/data/http/tests/passes/text-expected.txt: Copied from WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text-expected.txt.
        * Scripts/webkitpy/layout_tests/data/http/tests/passes/text.html: Copied from WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text.html.
        * Scripts/webkitpy/layout_tests/data/http/tests/ssl/text-expected.txt: Copied from WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text-expected.txt.
        * Scripts/webkitpy/layout_tests/data/http/tests/ssl/text.html: Copied from WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text.html.
        * Scripts/webkitpy/layout_tests/data/platform/test/test_expectations.txt:
        * Scripts/webkitpy/layout_tests/data/websocket/tests/passes/text-expected.txt: Copied from WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text-expected.txt.
        * Scripts/webkitpy/layout_tests/data/websocket/tests/passes/text.html: Copied from WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text.html.
        * Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py:
        * Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py: Added.
        * Scripts/webkitpy/layout_tests/port/base.py:
        * Scripts/webkitpy/layout_tests/port/base_unittest.py:
        * Scripts/webkitpy/layout_tests/port/chromium.py:
        * Scripts/webkitpy/layout_tests/port/dryrun.py:
        * Scripts/webkitpy/layout_tests/port/factory_unittest.py:
        * Scripts/webkitpy/layout_tests/port/mac_unittest.py:
        * Scripts/webkitpy/layout_tests/port/port_testcase.py: Added.
        * Scripts/webkitpy/layout_tests/port/server_process.py:
        * Scripts/webkitpy/layout_tests/port/test.py:
        * Scripts/webkitpy/layout_tests/port/webkit.py:
        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
        * Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py:

git-svn-id: http://svn.webkit.org/repository/webkit/trunk@66937 268f45cc-cd09-0410-ab3c-d52691b4dbfc

23 files changed:
WebKitTools/ChangeLog
WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/hang.html [new file with mode: 0644]
WebKitTools/Scripts/webkitpy/layout_tests/data/http/tests/passes/text-expected.txt [new file with mode: 0644]
WebKitTools/Scripts/webkitpy/layout_tests/data/http/tests/passes/text.html [new file with mode: 0644]
WebKitTools/Scripts/webkitpy/layout_tests/data/http/tests/ssl/text-expected.txt [new file with mode: 0644]
WebKitTools/Scripts/webkitpy/layout_tests/data/http/tests/ssl/text.html [new file with mode: 0644]
WebKitTools/Scripts/webkitpy/layout_tests/data/platform/test/test_expectations.txt
WebKitTools/Scripts/webkitpy/layout_tests/data/websocket/tests/passes/text-expected.txt [new file with mode: 0644]
WebKitTools/Scripts/webkitpy/layout_tests/data/websocket/tests/passes/text.html [new file with mode: 0644]
WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py
WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py [new file with mode: 0644]
WebKitTools/Scripts/webkitpy/layout_tests/port/base.py
WebKitTools/Scripts/webkitpy/layout_tests/port/base_unittest.py
WebKitTools/Scripts/webkitpy/layout_tests/port/chromium.py
WebKitTools/Scripts/webkitpy/layout_tests/port/dryrun.py
WebKitTools/Scripts/webkitpy/layout_tests/port/factory_unittest.py
WebKitTools/Scripts/webkitpy/layout_tests/port/mac_unittest.py
WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py [new file with mode: 0644]
WebKitTools/Scripts/webkitpy/layout_tests/port/server_process.py
WebKitTools/Scripts/webkitpy/layout_tests/port/test.py
WebKitTools/Scripts/webkitpy/layout_tests/port/webkit.py
WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py

index 6f3cacb88b3cf20f548db1274d446880e13a5d4c..ae49dfc790dd6f5db11840da43dd856fba5b21ec 100644 (file)
@@ -1,3 +1,47 @@
+2010-09-01  Dirk Pranke  <dpranke@chromium.org>
+
+        Reviewed by Ojan Vafai.
+
+        new-run-webkit-tests: still more unit tests
+
+        Clean up and remove unnecessary code. Biggest notable change is
+        moving the chromium-specific imagediff code from port/base to
+        port/chromium.
+
+        Add more unit tests for run_webkit_tests.py, port/base.py,
+        port/factory.py, port/dryrun.py, and
+        layout_package/dump_render_tree_thread.py
+
+        This covers almost all of the generic and test code paths except for
+        a few error paths involving invalid or port-specific command line
+        arguments, and the code path for uploading results files to the
+        buildbots.
+        
+        https://bugs.webkit.org/show_bug.cgi?id=45090
+
+        * Scripts/webkitpy/layout_tests/data/failures/expected/hang.html: Added.
+        * Scripts/webkitpy/layout_tests/data/http/tests/passes/text-expected.txt: Copied from WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text-expected.txt.
+        * Scripts/webkitpy/layout_tests/data/http/tests/passes/text.html: Copied from WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text.html.
+        * Scripts/webkitpy/layout_tests/data/http/tests/ssl/text-expected.txt: Copied from WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text-expected.txt.
+        * Scripts/webkitpy/layout_tests/data/http/tests/ssl/text.html: Copied from WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text.html.
+        * Scripts/webkitpy/layout_tests/data/platform/test/test_expectations.txt:
+        * Scripts/webkitpy/layout_tests/data/websocket/tests/passes/text-expected.txt: Copied from WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text-expected.txt.
+        * Scripts/webkitpy/layout_tests/data/websocket/tests/passes/text.html: Copied from WebKitTools/Scripts/webkitpy/layout_tests/data/passes/text.html.
+        * Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py:
+        * Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py: Added.
+        * Scripts/webkitpy/layout_tests/port/base.py:
+        * Scripts/webkitpy/layout_tests/port/base_unittest.py:
+        * Scripts/webkitpy/layout_tests/port/chromium.py:
+        * Scripts/webkitpy/layout_tests/port/dryrun.py:
+        * Scripts/webkitpy/layout_tests/port/factory_unittest.py:
+        * Scripts/webkitpy/layout_tests/port/mac_unittest.py:
+        * Scripts/webkitpy/layout_tests/port/port_testcase.py: Added.
+        * Scripts/webkitpy/layout_tests/port/server_process.py:
+        * Scripts/webkitpy/layout_tests/port/test.py:
+        * Scripts/webkitpy/layout_tests/port/webkit.py:
+        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+        * Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py:
+
 2010-09-07  Joseph Pecoraro  <joepeck@webkit.org>
 
         Reviewed by Darin Adler.
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/hang.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/failures/expected/hang.html
new file mode 100644 (file)
index 0000000..4e0de08
--- /dev/null
@@ -0,0 +1 @@
+timeout-thread
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/http/tests/passes/text-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/http/tests/passes/text-expected.txt
new file mode 100644 (file)
index 0000000..2b38a06
--- /dev/null
@@ -0,0 +1 @@
+text-txt
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/http/tests/passes/text.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/http/tests/passes/text.html
new file mode 100644 (file)
index 0000000..8e27be7
--- /dev/null
@@ -0,0 +1 @@
+text
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/http/tests/ssl/text-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/http/tests/ssl/text-expected.txt
new file mode 100644 (file)
index 0000000..2b38a06
--- /dev/null
@@ -0,0 +1 @@
+text-txt
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/http/tests/ssl/text.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/http/tests/ssl/text.html
new file mode 100644 (file)
index 0000000..8e27be7
--- /dev/null
@@ -0,0 +1 @@
+text
index 16556e3f8c1940711441bd46e2c889a3b14c289b..0619fdeaca832338e4b4e519781eb0ee669bb4c4 100644 (file)
@@ -8,5 +8,6 @@ WONTFIX : failures/expected/missing_image.html = MISSING PASS
 WONTFIX : failures/expected/missing_text.html = MISSING PASS
 WONTFIX : failures/expected/text.html = TEXT
 WONTFIX : failures/expected/timeout.html = TIMEOUT
+WONTFIX SKIP : failures/expected/hang.html = TIMEOUT
 WONTFIX SKIP : failures/expected/keyboard.html = CRASH
 WONTFIX SKIP : failures/expected/exception.html = CRASH
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/websocket/tests/passes/text-expected.txt b/WebKitTools/Scripts/webkitpy/layout_tests/data/websocket/tests/passes/text-expected.txt
new file mode 100644 (file)
index 0000000..2b38a06
--- /dev/null
@@ -0,0 +1 @@
+text-txt
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/data/websocket/tests/passes/text.html b/WebKitTools/Scripts/webkitpy/layout_tests/data/websocket/tests/passes/text.html
new file mode 100644 (file)
index 0000000..8e27be7
--- /dev/null
@@ -0,0 +1 @@
+text
index ec33086e17fc8de7267698eb3600eb43f263e49a..9b963ca9f7dd0c5333ca2b8dadff94bccb745247 100644 (file)
@@ -47,6 +47,7 @@ import sys
 import thread
 import threading
 import time
+import traceback
 
 import test_failures
 
@@ -54,6 +55,23 @@ _log = logging.getLogger("webkitpy.layout_tests.layout_package."
                          "dump_render_tree_thread")
 
 
+def find_thread_stack(id):
+    """Returns a stack object that can be used to dump a stack trace for
+    the given thread id (or None if the id is not found)."""
+    for thread_id, stack in sys._current_frames().items():
+        if thread_id == id:
+            return stack
+    return None
+
+
+def log_stack(stack):
+    """Log a stack trace to log.error()."""
+    for filename, lineno, name, line in traceback.extract_stack(stack):
+        _log.error('File: "%s", line %d, in %s' % (filename, lineno, name))
+        if line:
+            _log.error('  %s' % line.strip())
+
+
 def _process_output(port, test_info, test_types, test_args, configuration,
                     output_dir, crash, timeout, test_run_time, actual_checksum,
                     output, error):
@@ -167,6 +185,7 @@ class SingleTestThread(threading.Thread):
         self._test_args = test_args
         self._configuration = configuration
         self._output_dir = output_dir
+        self._driver = None
 
     def run(self):
         self._covered_run()
@@ -175,18 +194,19 @@ class SingleTestThread(threading.Thread):
         # FIXME: this is a separate routine to work around a bug
         # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
         test_info = self._test_info
-        driver = self._port.create_driver(self._image_path, self._shell_args)
-        driver.start()
+        self._driver = self._port.create_driver(self._image_path,
+                                                self._shell_args)
+        self._driver.start()
         start = time.time()
         crash, timeout, actual_checksum, output, error = \
-            driver.run_test(test_info.uri.strip(), test_info.timeout,
-                            test_info.image_hash())
+            self._driver.run_test(test_info.uri.strip(), test_info.timeout,
+                                  test_info.image_hash())
         end = time.time()
         self._test_result = _process_output(self._port,
             test_info, self._test_types, self._test_args,
             self._configuration, self._output_dir, crash, timeout, end - start,
             actual_checksum, output, error)
-        driver.stop()
+        self._driver.stop()
 
     def get_test_result(self):
         return self._test_result
@@ -312,9 +332,7 @@ class TestShellThread(WatchableThread):
             # Save the exception for our caller to see.
             self._exception_info = sys.exc_info()
             self._stop_time = time.time()
-            # Re-raise it and die.
-            _log.error('%s dying, exception raised: %s' % (self.getName(),
-                       self._exception_info))
+            _log.error('%s dying, exception raised' % self.getName())
 
         self._stop_time = time.time()
 
@@ -426,7 +444,7 @@ class TestShellThread(WatchableThread):
         worker.start()
 
         thread_timeout = _milliseconds_to_seconds(
-            _pad_timeout(test_info.timeout))
+            _pad_timeout(int(test_info.timeout)))
         thread._next_timeout = time.time() + thread_timeout
         worker.join(thread_timeout)
         if worker.isAlive():
@@ -439,11 +457,13 @@ class TestShellThread(WatchableThread):
             # that tradeoff in order to avoid losing the rest of this
             # thread's results.
             _log.error('Test thread hung: killing all DumpRenderTrees')
-            worker._driver.stop()
+            if worker._driver:
+                worker._driver.stop()
 
         try:
             result = worker.get_test_result()
         except AttributeError, e:
+            # This gets raised if the worker thread has already exited.
             failures = []
             _log.error('Cannot get results of test: %s' %
                        test_info.filename)
@@ -476,7 +496,7 @@ class TestShellThread(WatchableThread):
         start = time.time()
 
         thread_timeout = _milliseconds_to_seconds(
-             _pad_timeout(test_info.timeout))
+             _pad_timeout(int(test_info.timeout)))
         self._next_timeout = start + thread_timeout
 
         crash, timeout, actual_checksum, output, error = \
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py
new file mode 100644 (file)
index 0000000..63f86d9
--- /dev/null
@@ -0,0 +1,49 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+""""Tests code paths not covered by the regular unit tests."""
+
+import sys
+import unittest
+
+import dump_render_tree_thread
+
+
+class Test(unittest.TestCase):
+    def test_find_thread_stack_found(self):
+        id, stack = sys._current_frames().items()[0]
+        found_stack = dump_render_tree_thread.find_thread_stack(id)
+        self.assertNotEqual(found_stack, None)
+
+    def test_find_thread_stack_not_found(self):
+        found_stack = dump_render_tree_thread.find_thread_stack(0)
+        self.assertEqual(found_stack, None)
+
+
+if __name__ == '__main__':
+    unittest.main()
index d6dd2b2efdbdc07851e317a06cf40d642698f33f..9125f9ed23f90529f25b86230618deafced72965 100644 (file)
@@ -120,7 +120,7 @@ class Port(object):
 
     def check_image_diff(self, override_step=None, logging=True):
         """This routine is used to check whether image_diff binary exists."""
-        raise NotImplemented('Port.check_image_diff')
+        raise NotImplementedError('Port.check_image_diff')
 
     def compare_text(self, expected_text, actual_text):
         """Return whether or not the two strings are *not* equal. This
@@ -141,26 +141,9 @@ class Port(object):
         |tolerance| should be a percentage value (0.0 - 100.0).
         If it is omitted, the port default tolerance value is used.
 
-        While this is a generic routine, we include it in the Port
-        interface so that it can be overriden for testing purposes."""
-        executable = self._path_to_image_diff()
+        """
+        raise NotImplementedError('Port.diff_image')
 
-        if diff_filename:
-            cmd = [executable, '--diff', expected_filename, actual_filename,
-                   diff_filename]
-        else:
-            cmd = [executable, expected_filename, actual_filename]
-
-        result = True
-        try:
-            if self._executive.run_command(cmd, return_exit_code=True) == 0:
-                return False
-        except OSError, e:
-            if e.errno == errno.ENOENT or e.errno == errno.EACCES:
-                _compare_available = False
-            else:
-                raise e
-        return result
 
     def diff_text(self, expected_text, actual_text,
                   expected_filename, actual_filename):
@@ -353,84 +336,16 @@ class Port(object):
         if the port does not use expectations files."""
         raise NotImplementedError('Port.path_to_test_expectations_file')
 
-    def remove_directory(self, *path):
-        """Recursively removes a directory, even if it's marked read-only.
-
-        Remove the directory located at *path, if it exists.
-
-        shutil.rmtree() doesn't work on Windows if any of the files
-        or directories are read-only, which svn repositories and
-        some .svn files are.  We need to be able to force the files
-        to be writable (i.e., deletable) as we traverse the tree.
-
-        Even with all this, Windows still sometimes fails to delete a file,
-        citing a permission error (maybe something to do with antivirus
-        scans or disk indexing).  The best suggestion any of the user
-        forums had was to wait a bit and try again, so we do that too.
-        It's hand-waving, but sometimes it works. :/
-        """
-        file_path = os.path.join(*path)
-        if not os.path.exists(file_path):
-            return
-
-        win32 = False
-        if sys.platform == 'win32':
-            win32 = True
-            # Some people don't have the APIs installed. In that case we'll do
-            # without.
-            try:
-                win32api = __import__('win32api')
-                win32con = __import__('win32con')
-            except ImportError:
-                win32 = False
-
-            def remove_with_retry(rmfunc, path):
-                os.chmod(path, os.stat.S_IWRITE)
-                if win32:
-                    win32api.SetFileAttributes(path,
-                                              win32con.FILE_ATTRIBUTE_NORMAL)
-                try:
-                    return rmfunc(path)
-                except EnvironmentError, e:
-                    if e.errno != errno.EACCES:
-                        raise
-                    print 'Failed to delete %s: trying again' % repr(path)
-                    time.sleep(0.1)
-                    return rmfunc(path)
-        else:
-
-            def remove_with_retry(rmfunc, path):
-                if os.path.islink(path):
-                    return os.remove(path)
-                else:
-                    return rmfunc(path)
-
-        for root, dirs, files in os.walk(file_path, topdown=False):
-            # For POSIX:  making the directory writable guarantees
-            # removability. Windows will ignore the non-read-only
-            # bits in the chmod value.
-            os.chmod(root, 0770)
-            for name in files:
-                remove_with_retry(os.remove, os.path.join(root, name))
-            for name in dirs:
-                remove_with_retry(os.rmdir, os.path.join(root, name))
-
-        remove_with_retry(os.rmdir, file_path)
-
-    def test_platform_name(self):
-        return self._name
-
     def relative_test_filename(self, filename):
         """Relative unix-style path for a filename under the LayoutTests
         directory. Filenames outside the LayoutTests directory should raise
         an error."""
-        # FIXME This should assert() here but cannot due to printing_unittest.Testprinter
-        # assert(filename.startswith(self.layout_tests_dir()))
+        assert(filename.startswith(self.layout_tests_dir()))
         return filename[len(self.layout_tests_dir()) + 1:]
 
     def results_directory(self):
         """Absolute path to the place to store the test results."""
-        raise NotImplemented('Port.results_directory')
+        raise NotImplementedError('Port.results_directory')
 
     def setup_test_run(self):
         """Perform port-specific work at the beginning of a test run."""
@@ -620,7 +535,6 @@ class Port(object):
                 _wdiff_available = False
                 return ""
             raise
-        assert(False)  # Should never be reached.
 
     _pretty_patch_error_html = "Failed to run PrettyPatch, see error console."
 
@@ -824,10 +738,5 @@ class Driver:
         if it has exited."""
         raise NotImplementedError('Driver.poll')
 
-    def returncode(self):
-        """Returns the system-specific returncode if the Driver has stopped or
-        exited."""
-        raise NotImplementedError('Driver.returncode')
-
     def stop(self):
         raise NotImplementedError('Driver.stop')
index 1ae099ade19db79c2d23f588f148b30466f3bf46..1cc426f78187bf9d43eb47ec403e132a89f9e75a 100644 (file)
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import base
-import unittest
+import os
+import StringIO
+import sys
 import tempfile
+import unittest
 
 from webkitpy.common.system.executive import Executive, ScriptError
 from webkitpy.thirdparty.mock import Mock
 
 
-class PortTest(unittest.TestCase):
+# FIXME: This makes StringIO objects work with "with". Remove
+# when we upgrade to 2.6.
+class NewStringIO(StringIO.StringIO):
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        pass
+
+
+class MockExecutive():
+    def __init__(self, exception):
+        self._exception = exception
+
+    def run_command(self, *args, **kwargs):
+        raise self._exception
+
 
+class UnitTestPort(base.Port):
+    """Subclass of base.Port used for unit testing."""
+    def __init__(self, configuration_contents=None, executive_exception=None):
+        base.Port.__init__(self)
+        self._configuration_contents = configuration_contents
+        if executive_exception:
+            self._executive = MockExecutive(executive_exception)
+
+    def _open_configuration_file(self):
+        if self._configuration_contents:
+            return NewStringIO(self._configuration_contents)
+        return base.Port._open_configuration_file(self)
+
+
+class PortTest(unittest.TestCase):
     def test_format_wdiff_output_as_html(self):
         output = "OUTPUT %s %s %s" % (base.Port._WDIFF_DEL, base.Port._WDIFF_ADD, base.Port._WDIFF_END)
         html = base.Port()._format_wdiff_output_as_html(output)
@@ -63,6 +97,26 @@ class PortTest(unittest.TestCase):
         new_file.flush()
         return new_file
 
+    def test_pretty_patch_os_error(self):
+        port = UnitTestPort(executive_exception=OSError)
+        self.assertEqual(port.pretty_patch_text("patch.txt"),
+                         port._pretty_patch_error_html)
+
+        # This tests repeated calls to make sure we cache the result.
+        self.assertEqual(port.pretty_patch_text("patch.txt"),
+                         port._pretty_patch_error_html)
+
+    def test_pretty_patch_script_error(self):
+        # FIXME: This is some ugly white-box test hacking ...
+        base._pretty_patch_available = True
+        port = UnitTestPort(executive_exception=ScriptError)
+        self.assertEqual(port.pretty_patch_text("patch.txt"),
+                         port._pretty_patch_error_html)
+
+        # This tests repeated calls to make sure we cache the result.
+        self.assertEqual(port.pretty_patch_text("patch.txt"),
+                         port._pretty_patch_error_html)
+
     def test_run_wdiff(self):
         executive = Executive()
         # This may fail on some systems.  We could ask the port
@@ -109,6 +163,10 @@ class PortTest(unittest.TestCase):
         self.assertFalse(base._wdiff_available)
         base._wdiff_available = True
 
+    def test_default_configuration_notfound(self):
+        port = UnitTestPort()
+        self.assertEqual(port.default_configuration(), "Release")
+
     def test_layout_tests_skipping(self):
         port = base.Port()
         port.skipped_layout_tests = lambda: ['foo/bar.html', 'media']
@@ -116,6 +174,68 @@ class PortTest(unittest.TestCase):
         self.assertTrue(port.skips_layout_test('media/video-zoom.html'))
         self.assertFalse(port.skips_layout_test('foo/foo.html'))
 
+    def test_default_configuration_found(self):
+        port = UnitTestPort(configuration_contents="Debug")
+        self.assertEqual(port.default_configuration(), "Debug")
+
+    def test_default_configuration_unknown(self):
+        port = UnitTestPort(configuration_contents="weird_value")
+        self.assertEqual(port.default_configuration(), "weird_value")
+
+    def test_setup_test_run(self):
+        port = base.Port()
+        # This routine is a no-op. We just test it for coverage.
+        port.setup_test_run()
+
+
+class VirtualTest(unittest.TestCase):
+    """Tests that various methods expected to be virtual are."""
+    def assertVirtual(self, method, *args, **kwargs):
+        self.assertRaises(NotImplementedError, method, *args, **kwargs)
+
+    def test_virtual_methods(self):
+        port = base.Port()
+        self.assertVirtual(port.baseline_path)
+        self.assertVirtual(port.baseline_search_path)
+        self.assertVirtual(port.check_build, None)
+        self.assertVirtual(port.check_image_diff)
+        self.assertVirtual(port.create_driver, None, None)
+        self.assertVirtual(port.diff_image, None, None)
+        self.assertVirtual(port.path_to_test_expectations_file)
+        self.assertVirtual(port.test_platform_name)
+        self.assertVirtual(port.results_directory)
+        self.assertVirtual(port.show_html_results_file, None)
+        self.assertVirtual(port.test_expectations)
+        self.assertVirtual(port.test_base_platform_names)
+        self.assertVirtual(port.test_platform_name)
+        self.assertVirtual(port.test_platforms)
+        self.assertVirtual(port.test_platform_name_to_name, None)
+        self.assertVirtual(port.version)
+        self.assertVirtual(port._path_to_apache)
+        self.assertVirtual(port._path_to_apache_config_file)
+        self.assertVirtual(port._path_to_driver)
+        self.assertVirtual(port._path_to_helper)
+        self.assertVirtual(port._path_to_image_diff)
+        self.assertVirtual(port._path_to_lighttpd)
+        self.assertVirtual(port._path_to_lighttpd_modules)
+        self.assertVirtual(port._path_to_lighttpd_php)
+        self.assertVirtual(port._path_to_wdiff)
+        self.assertVirtual(port._shut_down_http_server, None)
+
+    def test_virtual_driver_method(self):
+        self.assertRaises(NotImplementedError, base.Driver, base.Port, "", None)
+        self.assertVirtual(base.Driver, base.Port, "", None)
+
+    def test_virtual_driver_methods(self):
+        class VirtualDriver(base.Driver):
+            def __init__(self):
+                pass
+
+        driver = VirtualDriver()
+        self.assertVirtual(driver.run_test, None, None, None)
+        self.assertVirtual(driver.poll)
+        self.assertVirtual(driver.stop)
+
 
 class DriverTest(unittest.TestCase):
 
@@ -131,3 +251,7 @@ class DriverTest(unittest.TestCase):
         command_with_spaces = "valgrind --smc-check=\"check with spaces!\" --foo"
         expected_parse = ["valgrind", "--smc-check=check with spaces!", "--foo"]
         self._assert_wrapper(command_with_spaces, expected_parse)
+
+
+if __name__ == '__main__':
+    unittest.main()
index d5e6ea77d2fe31930c8dce88ab33add3040fb863..8110568f147bf49c2b57183611484666039aa95b 100644 (file)
@@ -126,6 +126,26 @@ class ChromiumPort(base.Port):
         return check_file_exists(image_diff_path, 'image diff exe',
                                  override_step, logging)
 
+    def diff_image(self, expected_filename, actual_filename,
+                   diff_filename=None, tolerance=0):
+        executable = self._path_to_image_diff()
+        if diff_filename:
+            cmd = [executable, '--diff', expected_filename, actual_filename,
+                   diff_filename]
+        else:
+            cmd = [executable, expected_filename, actual_filename]
+
+        result = True
+        try:
+            if self._executive.run_command(cmd, return_exit_code=True) == 0:
+                return False
+        except OSError, e:
+            if e.errno == errno.ENOENT or e.errno == errno.EACCES:
+                _compare_available = False
+            else:
+                raise e
+        return result
+
     def driver_name(self):
         return "test_shell"
 
@@ -350,9 +370,6 @@ class ChromiumDriver(base.Driver):
         # http://bugs.python.org/issue1731717
         return self._proc.poll()
 
-    def returncode(self):
-        return self._proc.returncode
-
     def _write_command_and_read_line(self, input=None):
         """Returns a tuple: (line, did_crash)"""
         try:
index e01bd2fe36eb2ddeceeff96697d28cab85e1d298..1af01ad90833d21270225c11c5f4bd81f6033c7e 100644 (file)
@@ -70,14 +70,11 @@ def _read_file(path, mode='r'):
 def _write_file(path, contents, mode='w'):
     """Write the string to the specified path.
 
-    Returns nothing if the write fails, instead of raising an IOError.
+    Writes should never fail, so we may raise IOError.
 
     """
-    try:
-        with open(path, mode) as f:
+    with open(path, mode) as f:
             f.write(contents)
-    except IOError:
-        pass
 
 
 class DryRunPort(object):
@@ -134,9 +131,6 @@ class DryrunDriver(base.Driver):
     def poll(self):
         return None
 
-    def returncode(self):
-        return 0
-
     def run_test(self, uri, timeoutms, image_hash):
         test_name = self._uri_to_test(uri)
 
index ff9ebcee20f08e939ab7f5459b62b4fa789ae0f5..c0a4c5e5d402f85df09350b8210a54ce93080104 100644 (file)
@@ -34,6 +34,7 @@ import chromium_mac
 import chromium_win
 import dryrun
 import factory
+import google_chrome
 import gtk
 import mac
 import qt
@@ -114,6 +115,18 @@ class FactoryTest(unittest.TestCase):
         self.assert_platform_port("cygwin", None, win.WinPort)
         self.assert_platform_port("cygwin", self.webkit_options, win.WinPort)
 
+    def test_google_chrome(self):
+        # The actual Chrome class names aren't available so we test that the
+        # objects we get are at least subclasses of the Chromium versions.
+        self.assert_port("google-chrome-linux32",
+                         chromium_linux.ChromiumLinuxPort)
+        self.assert_port("google-chrome-linux64",
+                         chromium_linux.ChromiumLinuxPort)
+        self.assert_port("google-chrome-win",
+                         chromium_win.ChromiumWinPort)
+        self.assert_port("google-chrome-mac",
+                         chromium_mac.ChromiumMacPort)
+
     def test_gtk(self):
         self.assert_port("gtk", gtk.GtkPort)
 
@@ -154,3 +167,20 @@ class FactoryTest(unittest.TestCase):
                          ports["chromium-linux"])
         self.assert_port("chromium-win", chromium_win.ChromiumWinPort,
                          ports["chromium-win"])
+
+    def test_unknown_specified(self):
+        # Test what happens when you specify an unknown port.
+        orig_platform = sys.platform
+        self.assertRaises(NotImplementedError, factory.get,
+                          port_name='unknown')
+
+    def test_unknown_default(self):
+        # Test what happens when you're running on an unknown platform.
+        orig_platform = sys.platform
+        sys.platform = 'unknown'
+        self.assertRaises(NotImplementedError, factory.get)
+        sys.platform = orig_platform
+
+
+if __name__ == '__main__':
+    unittest.main()
index ae7d40c45af0b331d64bf636a58fc19ea55818ca..b5797039a4b53ddf74d7d7d69d1e64665b66996e 100644 (file)
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import StringIO
 import unittest
+
 import mac
-import StringIO
+import port_testcase
+
 
-class MacTest(unittest.TestCase):
+class MacTest(port_testcase.PortTestCase):
+    def make_port(self, options=port_testcase.MockOptions()):
+        port_obj = mac.MacPort(options=options)
+        port_obj._options.results_directory = port_obj.results_directory()
+        port_obj._options.configuration = 'Release'
+        return port_obj
 
     def test_skipped_file_paths(self):
-        port = mac.MacPort()
+        port = self.make_port()
         skipped_paths = port._skipped_file_paths()
         # FIXME: _skipped_file_paths should return WebKit-relative paths.
         # So to make it unit testable, we strip the WebKit directory from the path.
@@ -57,7 +65,7 @@ svg/batik/text/smallFonts.svg
     ]
 
     def test_skipped_file_paths(self):
-        port = mac.MacPort()
+        port = self.make_port()
         skipped_file = StringIO.StringIO(self.example_skipped_file)
         self.assertEqual(port._tests_from_skipped_file(skipped_file), self.example_skipped_tests)
 
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py b/WebKitTools/Scripts/webkitpy/layout_tests/port/port_testcase.py
new file mode 100644 (file)
index 0000000..7a97916
--- /dev/null
@@ -0,0 +1,76 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#    * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#    * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Unit testing base class for Port implementations."""
+
+import os
+import tempfile
+import unittest
+
+
+class MockOptions(object):
+    def __init__(self,
+                 results_directory='layout-test-results',
+                 use_apache=True,
+                 configuration='Release'):
+        self.results_directory = results_directory
+        self.use_apache = use_apache
+        self.configuration = configuration
+
+
+class PortTestCase(unittest.TestCase):
+    """Tests the WebKit port implementation."""
+    def make_port(self, options=MockOptions()):
+        """Override in subclass."""
+        raise NotImplementedError()
+
+    def test_http_server(self):
+        port = self.make_port()
+        port.start_http_server()
+        port.stop_http_server()
+
+    def test_image_diff(self):
+        port = self.make_port()
+        dir = port.layout_tests_dir()
+        file1 = os.path.join(dir, 'fast', 'css', 'button_center.png')
+        file2 = os.path.join(dir, 'fast', 'css',
+                             'remove-shorthand-expected.png')
+        tmpfile = tempfile.mktemp()
+
+        self.assertFalse(port.diff_image(file1, file1))
+        self.assertTrue(port.diff_image(file1, file2))
+
+        self.assertTrue(port.diff_image(file1, file2, tmpfile))
+        # FIXME: this may not be being written?
+        # self.assertTrue(os.path.exists(tmpfile))
+        # os.remove(tmpfile)
+
+    def test_websocket_server(self):
+        port = self.make_port()
+        port.start_websocket_server()
+        port.stop_websocket_server()
index 62ca69312ab07ae3ab59ff3e5c94b6752538debb..bccdf181d25e97ab023bb87f73ac753622899ca7 100644 (file)
@@ -109,14 +109,6 @@ class ServerProcess:
             return self._proc.poll()
         return None
 
-    def returncode(self):
-        """Returns the exit code from the subprcoess; returns None if the
-        process hasn't exited (this is a wrapper around subprocess.returncode).
-        """
-        if self._proc:
-            return self._proc.returncode
-        return None
-
     def write(self, input):
         """Write a request to the subprocess. The subprocess is (re-)start()'ed
         if is not already running."""
index a64823c1739ba543a04ec2e9ea68c516ac568cd4..a3a16c34a5a2266746af5e080ddc6452019ccf47 100644 (file)
@@ -148,9 +148,6 @@ class TestDriver(base.Driver):
     def poll(self):
         return True
 
-    def returncode(self):
-        return 0
-
     def run_test(self, uri, timeoutms, image_hash):
         basename = uri[(uri.rfind("/") + 1):uri.rfind(".html")]
 
@@ -182,6 +179,7 @@ class TestDriver(base.Driver):
                 raise ValueError('exception from ' + basename)
 
             crash = 'crash' in basename
+            timeout = 'timeout' in basename or 'hang' in basename
             timeout = 'timeout' in basename
             if 'text' in basename:
                 output = basename + '_failed-txt\n'
@@ -199,6 +197,9 @@ class TestDriver(base.Driver):
                         f.write(basename + "-png\n")
                 if 'checksum' in basename:
                     checksum = basename + "_failed-checksum\n"
+
+            if 'hang' in basename:
+                time.sleep((float(timeoutms) * 4) / 1000.0)
         else:
             crash = False
             timeout = False
index 401466491cce801e382dc5b716498bb23251f47d..b085ceb7e68fc6bdfc25bea2aafae8ab8bc92db8 100644 (file)
@@ -435,9 +435,6 @@ class WebKitDriver(base.Driver):
         self._server_process.start()
         return
 
-    def returncode(self):
-        return self._server_process.returncode()
-
     # FIXME: This function is huge.
     def run_test(self, uri, timeoutms, image_hash):
         if uri.startswith("file:///"):
index ff27927dbbc22baff0daacafdcb98162e9052c5f..2e2da6d587377180c5bfbd1748db8594efaaa225 100755 (executable)
@@ -53,7 +53,6 @@ import logging
 import math
 import optparse
 import os
-import pdb
 import platform
 import Queue
 import random
@@ -371,7 +370,7 @@ class TestRunner:
                 assert(test_size > 0)
             except:
                 _log.critical("invalid chunk '%s'" % chunk_value)
-                sys.exit(1)
+                return None
 
             # Get the number of tests
             num_tests = len(test_files)
@@ -1349,11 +1348,18 @@ class TestRunner:
 def read_test_files(files):
     tests = []
     for file in files:
-        # FIXME: This could be cleaner using a list comprehension.
-        for line in codecs.open(file, "r", "utf-8"):
-            line = test_expectations.strip_comments(line)
-            if line:
-                tests.append(line)
+        try:
+            with codecs.open(file, 'r', 'utf-8') as file_contents:
+                # FIXME: This could be cleaner using a list comprehension.
+                for line in file_contents:
+                    line = test_expectations.strip_comments(line)
+                    if line:
+                        tests.append(line)
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                _log.critical('')
+                _log.critical('--test-list file "%s" not found' % file)
+            raise
     return tests
 
 
@@ -1397,7 +1403,12 @@ def run(port, options, args, regular_output=sys.stderr,
         test_runner._print_config()
 
         printer.print_update("Collecting tests ...")
-        test_runner.collect_tests(args, last_unexpected_results)
+        try:
+            test_runner.collect_tests(args, last_unexpected_results)
+        except IOError, e:
+            if e.errno == errno.ENOENT:
+                return -1
+            raise
 
         printer.print_update("Parsing expectations ...")
         if options.lint_test_files:
@@ -1694,31 +1705,14 @@ def parse_args(args=None):
     return options, args
 
 
-def _find_thread_stack(id):
-    """Returns a stack object that can be used to dump a stack trace for
-    the given thread id (or None if the id is not found)."""
-    for thread_id, stack in sys._current_frames().items():
-        if thread_id == id:
-            return stack
-    return None
-
-
-def _log_stack(stack):
-    """Log a stack trace to log.error()."""
-    for filename, lineno, name, line in traceback.extract_stack(stack):
-        _log.error('File: "%s", line %d, in %s' % (filename, lineno, name))
-        if line:
-            _log.error('  %s' % line.strip())
-
-
 def _log_wedged_thread(thread):
     """Log information about the given thread state."""
     id = thread.id()
-    stack = _find_thread_stack(id)
+    stack = dump_render_tree_thread.find_thread_stack(id)
     assert(stack is not None)
     _log.error("")
     _log.error("thread %s (%d) is wedged" % (thread.getName(), id))
-    _log_stack(stack)
+    dump_render_tree_thread.log_stack(stack)
     _log.error("")
 
 
index 4cbfdfcdf8b9fd5a338ef14895b87b581d5c9898..aa9696272d9a878e6a9b96a442a9d5a3bfa99944 100644 (file)
@@ -32,9 +32,9 @@
 import codecs
 import logging
 import os
-import pdb
 import Queue
 import sys
+import tempfile
 import thread
 import time
 import threading
@@ -59,7 +59,10 @@ def passing_run(args=[], port_obj=None, record_results=False,
     new_args.extend(args)
     if not tests_included:
         # We use the glob to test that globbing works.
-        new_args.extend(['passes', 'failures/expected/*'])
+        new_args.extend(['passes',
+                         'http/tests',
+                         'websocket/tests',
+                         'failures/expected/*'])
     options, parsed_args = run_webkit_tests.parse_args(new_args)
     if port_obj is None:
         port_obj = port.get(options.platform, options)
@@ -71,10 +74,12 @@ def logging_run(args=[], tests_included=False):
     new_args = ['--no-record-results']
     if not '--platform' in args:
         new_args.extend(['--platform', 'test'])
-    if args:
-        new_args.extend(args)
+    new_args.extend(args)
     if not tests_included:
-        new_args.extend(['passes', 'failures/expected/*'])
+        new_args.extend(['passes',
+                         'http/tests'
+                         'websocket/tests',
+                         'failures/expected/*'])
     options, parsed_args = run_webkit_tests.parse_args(new_args)
     port_obj = port.get(options.platform, options)
     buildbot_output = array_stream.ArrayStream()
@@ -119,6 +124,14 @@ class MainTest(unittest.TestCase):
         self.assertTrue(out.empty())
         self.assertFalse(err.empty())
 
+    def test_hung_thread(self):
+        res, out, err = logging_run(['--run-singly', '--time-out-ms=50',
+                                     'failures/expected/hang.html'],
+                                    tests_included=True)
+        self.assertEqual(res, 0)
+        self.assertFalse(out.empty())
+        self.assertFalse(err.empty())
+
     def test_keyboard_interrupt(self):
         # Note that this also tests running a test marked as SKIP if
         # you specify it explicitly.
@@ -175,6 +188,19 @@ class MainTest(unittest.TestCase):
         # FIXME: verify # of tests run
         self.assertTrue(passing_run(['passes/text.html'], tests_included=True))
 
+    def test_test_list(self):
+        filename = tempfile.mktemp()
+        tmpfile = file(filename, mode='w+')
+        tmpfile.write('passes/text.html')
+        tmpfile.close()
+        self.assertTrue(passing_run(['--test-list=%s' % filename],
+                                    tests_included=True))
+        os.remove(filename)
+        res, out, err = logging_run(['--test-list=%s' % filename],
+                                    tests_included=True)
+        self.assertEqual(res, -1)
+        self.assertFalse(err.empty())
+
     def test_unexpected_failures(self):
         # Run tests including the unexpected failures.
         res, out, err = logging_run(tests_included=True)
@@ -279,6 +305,13 @@ class DryrunTest(unittest.TestCase):
         self.assertTrue(passing_run(['--platform', 'dryrun-mac',
                                      'fast/html']))
 
+    def test_test(self):
+        res, out, err = logging_run(['--platform', 'dryrun-test',
+                                     '--pixel-tests'])
+        self.assertEqual(res, 2)
+        self.assertFalse(out.empty())
+        self.assertFalse(err.empty())
+
 
 class TestThread(dump_render_tree_thread.WatchableThread):
     def __init__(self, started_queue, stopping_queue):
@@ -388,13 +421,6 @@ class StandaloneFunctionsTest(unittest.TestCase):
         self.assertFalse(child_thread.isAlive())
         oc.restore_output()
 
-    def test_find_thread_stack(self):
-        id, stack = sys._current_frames().items()[0]
-        found_stack = run_webkit_tests._find_thread_stack(id)
-        self.assertNotEqual(found_stack, None)
-
-        found_stack = run_webkit_tests._find_thread_stack(0)
-        self.assertEqual(found_stack, None)
 
 if __name__ == '__main__':
     unittest.main()