2010-11-18 Dirk Pranke <dpranke@chromium.org>
authordpranke@chromium.org <dpranke@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 18 Nov 2010 23:23:31 +0000 (23:23 +0000)
committerdpranke@chromium.org <dpranke@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 18 Nov 2010 23:23:31 +0000 (23:23 +0000)
        Reviewed by Tony Chang.

        new-run-webkit-tests: create first part of 'message_broker' class for multiprocessing fixes

        Create the first version of the 'message_broker' package. This
        class will encapsulate all of the threading/multiprocessing and
        message-sending details for the communication between the
        'manager' object and the 'worker' objects. For the moment, it
        just holds some routines and tests for logging thread stacks.

        There should be no functional changes in this patch, just moving stuff
        around.

        https://bugs.webkit.org/show_bug.cgi?id=49707

        * Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py:
        * Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py: Removed.
        * Scripts/webkitpy/layout_tests/layout_package/message_broker.py: Added.
        * Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py: Added.
        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
        * Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py:

git-svn-id: http://svn.webkit.org/repository/webkit/trunk@72339 268f45cc-cd09-0410-ab3c-d52691b4dbfc

WebKitTools/ChangeLog
WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py
WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py [deleted file]
WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py [new file with mode: 0644]
WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py [new file with mode: 0644]
WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py

index 28e95b226731fc38beb604a4b53640512495e56d..0b48ebb2efcdfbd5ddba8e90e38b0bda6c3118f5 100644 (file)
@@ -1,3 +1,27 @@
+2010-11-18  Dirk Pranke  <dpranke@chromium.org>
+
+        Reviewed by Tony Chang.
+
+        new-run-webkit-tests: create first part of 'message_broker' class for multiprocessing fixes
+
+        Create the first version of the 'message_broker' package. This
+        class will encapsulate all of the threading/multiprocessing and
+        message-sending details for the communication between the
+        'manager' object and the 'worker' objects. For the moment, it
+        just holds some routines and tests for logging thread stacks.
+
+        There should be no functional changes in this patch, just moving stuff
+        around.
+
+        https://bugs.webkit.org/show_bug.cgi?id=49707
+
+        * Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread.py:
+        * Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py: Removed.
+        * Scripts/webkitpy/layout_tests/layout_package/message_broker.py: Added.
+        * Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py: Added.
+        * Scripts/webkitpy/layout_tests/run_webkit_tests.py:
+        * Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py:
+
 2010-11-18  Steve Falkenburg  <sfalken@apple.com>
 
         Reviewed by Adam Roben.
index 88f493d0f98adb3411539df9209042e44b55393c..de97ad4e8a717d43cb47929597a5e06fd65e5045 100644 (file)
@@ -48,7 +48,6 @@ import sys
 import thread
 import threading
 import time
-import traceback
 
 import test_failures
 import test_output
@@ -58,23 +57,6 @@ _log = logging.getLogger("webkitpy.layout_tests.layout_package."
                          "dump_render_tree_thread")
 
 
-def find_thread_stack(id):
-    """Returns a stack object that can be used to dump a stack trace for
-    the given thread id (or None if the id is not found)."""
-    for thread_id, stack in sys._current_frames().items():
-        if thread_id == id:
-            return stack
-    return None
-
-
-def log_stack(stack):
-    """Log a stack trace to log.error()."""
-    for filename, lineno, name, line in traceback.extract_stack(stack):
-        _log.error('File: "%s", line %d, in %s' % (filename, lineno, name))
-        if line:
-            _log.error('  %s' % line.strip())
-
-
 def _expected_test_output(port, filename):
     """Returns an expected TestOutput object."""
     return test_output.TestOutput(port.expected_text(filename),
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/dump_render_tree_thread_unittest.py
deleted file mode 100644 (file)
index 63f86d9..0000000
+++ /dev/null
@@ -1,49 +0,0 @@
-# Copyright (C) 2010 Google Inc. All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#    * Redistributions of source code must retain the above copyright
-# notice, this list of conditions and the following disclaimer.
-#    * Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the following disclaimer
-# in the documentation and/or other materials provided with the
-# distribution.
-#    * Neither the name of Google Inc. nor the names of its
-# contributors may be used to endorse or promote products derived from
-# this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-""""Tests code paths not covered by the regular unit tests."""
-
-import sys
-import unittest
-
-import dump_render_tree_thread
-
-
-class Test(unittest.TestCase):
-    def test_find_thread_stack_found(self):
-        id, stack = sys._current_frames().items()[0]
-        found_stack = dump_render_tree_thread.find_thread_stack(id)
-        self.assertNotEqual(found_stack, None)
-
-    def test_find_thread_stack_not_found(self):
-        found_stack = dump_render_tree_thread.find_thread_stack(0)
-        self.assertEqual(found_stack, None)
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker.py
new file mode 100644 (file)
index 0000000..13951c0
--- /dev/null
@@ -0,0 +1,68 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Module for handling messages, threads, processes, and concurrency for run-webkit-tests.
+
+The model we use is that of a message broker - it provides a messaging
+abstraction and message loops, and handles launching threads and/or processes
+depending on the requested configuration.
+"""
+
+import logging
+import sys
+import traceback
+
+
+_log = logging.getLogger(__name__)
+
+
+def log_wedged_thread(id):
+    """Log information about the given thread state."""
+    stack = _find_thread_stack(id)
+    assert(stack is not None)
+    _log.error("")
+    _log.error("Thread %d is wedged" % id)
+    _log_stack(stack)
+    _log.error("")
+
+
+def _find_thread_stack(id):
+    """Returns a stack object that can be used to dump a stack trace for
+    the given thread id (or None if the id is not found)."""
+    for thread_id, stack in sys._current_frames().items():
+        if thread_id == id:
+            return stack
+    return None
+
+
+def _log_stack(stack):
+    """Log a stack trace to log.error()."""
+    for filename, lineno, name, line in traceback.extract_stack(stack):
+        _log.error('File: "%s", line %d, in %s' % (filename, lineno, name))
+        if line:
+            _log.error('  %s' % line.strip())
diff --git a/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py b/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/message_broker_unittest.py
new file mode 100644 (file)
index 0000000..d708745
--- /dev/null
@@ -0,0 +1,164 @@
+# Copyright (C) 2010 Google Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import logging
+import Queue
+import sys
+import thread
+import time
+import unittest
+
+from webkitpy.common import array_stream
+from webkitpy.common.system import outputcapture
+
+from webkitpy.layout_tests import run_webkit_tests
+
+import dump_render_tree_thread
+import message_broker
+
+
+class TestThread(dump_render_tree_thread.WatchableThread):
+    def __init__(self, started_queue, stopping_queue):
+        dump_render_tree_thread.WatchableThread.__init__(self)
+        self._started_queue = started_queue
+        self._stopping_queue = stopping_queue
+        self._timeout = False
+        self._timeout_queue = Queue.Queue()
+
+    def run(self):
+        self._covered_run()
+
+    def _covered_run(self):
+        # FIXME: this is a separate routine to work around a bug
+        # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
+        self._thread_id = thread.get_ident()
+        try:
+            self._started_queue.put('')
+            msg = self._stopping_queue.get()
+            if msg == 'KeyboardInterrupt':
+                raise KeyboardInterrupt
+            elif msg == 'Exception':
+                raise ValueError()
+            elif msg == 'Timeout':
+                self._timeout = True
+                self._timeout_queue.get()
+        except:
+            self._exception_info = sys.exc_info()
+
+    def next_timeout(self):
+        if self._timeout:
+            self._timeout_queue.put('done')
+            return time.time() - 10
+        return time.time()
+
+
+class TestHandler(logging.Handler):
+    def __init__(self, astream):
+        logging.Handler.__init__(self)
+        self._stream = astream
+
+    def emit(self, record):
+        self._stream.write(self.format(record))
+
+
+class WaitForThreadsToFinishTest(unittest.TestCase):
+    class MockTestRunner(run_webkit_tests.TestRunner):
+        def __init__(self):
+            pass
+
+        def __del__(self):
+            pass
+
+        def update_summary(self, result_summary):
+            pass
+
+    def run_one_thread(self, msg):
+        runner = self.MockTestRunner()
+        starting_queue = Queue.Queue()
+        stopping_queue = Queue.Queue()
+        child_thread = TestThread(starting_queue, stopping_queue)
+        child_thread.start()
+        started_msg = starting_queue.get()
+        stopping_queue.put(msg)
+        threads = [child_thread]
+        return runner._wait_for_threads_to_finish(threads, None)
+
+    def test_basic(self):
+        interrupted = self.run_one_thread('')
+        self.assertFalse(interrupted)
+
+    def test_interrupt(self):
+        interrupted = self.run_one_thread('KeyboardInterrupt')
+        self.assertTrue(interrupted)
+
+    def test_timeout(self):
+        oc = outputcapture.OutputCapture()
+        oc.capture_output()
+        interrupted = self.run_one_thread('Timeout')
+        self.assertFalse(interrupted)
+        oc.restore_output()
+
+    def test_exception(self):
+        self.assertRaises(ValueError, self.run_one_thread, 'Exception')
+
+
+class Test(unittest.TestCase):
+    def test_find_thread_stack_found(self):
+        id, stack = sys._current_frames().items()[0]
+        found_stack = message_broker._find_thread_stack(id)
+        self.assertNotEqual(found_stack, None)
+
+    def test_find_thread_stack_not_found(self):
+        found_stack = message_broker._find_thread_stack(0)
+        self.assertEqual(found_stack, None)
+
+    def test_log_wedged_thread(self):
+        oc = outputcapture.OutputCapture()
+        oc.capture_output()
+        logger = message_broker._log
+        astream = array_stream.ArrayStream()
+        handler = TestHandler(astream)
+        logger.addHandler(handler)
+
+        starting_queue = Queue.Queue()
+        stopping_queue = Queue.Queue()
+        child_thread = TestThread(starting_queue, stopping_queue)
+        child_thread.start()
+        msg = starting_queue.get()
+
+        message_broker.log_wedged_thread(child_thread.id())
+        stopping_queue.put('')
+        child_thread.join(timeout=1.0)
+
+        self.assertFalse(astream.empty())
+        self.assertFalse(child_thread.isAlive())
+        oc.restore_output()
+
+
+if __name__ == '__main__':
+    unittest.main()
index 119de8cfefe998e1305a8c1c7968a7163cbb5542..b3804f470bc63fb3d5b7b60eaf7693a3d9989a71 100755 (executable)
@@ -66,6 +66,7 @@ import traceback
 
 from layout_package import dump_render_tree_thread
 from layout_package import json_layout_results_generator
+from layout_package import message_broker
 from layout_package import printing
 from layout_package import test_expectations
 from layout_package import test_failures
@@ -664,7 +665,7 @@ class TestRunner:
                         some_thread_is_alive = True
                         next_timeout = thread.next_timeout()
                         if (next_timeout and t > next_timeout):
-                            _log_wedged_thread(thread)
+                            message_broker.log_wedged_thread(thread.id())
                             thread.clear_next_timeout()
 
                 self.update_summary(result_summary)
@@ -1667,16 +1668,6 @@ def parse_args(args=None):
     return options, args
 
 
-def _log_wedged_thread(thread):
-    """Log information about the given thread state."""
-    id = thread.id()
-    stack = dump_render_tree_thread.find_thread_stack(id)
-    assert(stack is not None)
-    _log.error("")
-    _log.error("thread %s (%d) is wedged" % (thread.getName(), id))
-    dump_render_tree_thread.log_stack(stack)
-    _log.error("")
-
 
 def main():
     options, args = parse_args()
index 54e1dc05df01c70607ce336d5d90004261445570..0bd1b3a35ca9129c2b7ac882067d376296a88bb0 100644 (file)
@@ -520,114 +520,5 @@ class DryrunTest(unittest.TestCase):
                                            '--pixel-tests']))
 
 
-class TestThread(dump_render_tree_thread.WatchableThread):
-    def __init__(self, started_queue, stopping_queue):
-        dump_render_tree_thread.WatchableThread.__init__(self)
-        self._started_queue = started_queue
-        self._stopping_queue = stopping_queue
-        self._timeout = False
-        self._timeout_queue = Queue.Queue()
-
-    def run(self):
-        self._covered_run()
-
-    def _covered_run(self):
-        # FIXME: this is a separate routine to work around a bug
-        # in coverage: see http://bitbucket.org/ned/coveragepy/issue/85.
-        self._thread_id = thread.get_ident()
-        try:
-            self._started_queue.put('')
-            msg = self._stopping_queue.get()
-            if msg == 'KeyboardInterrupt':
-                raise KeyboardInterrupt
-            elif msg == 'Exception':
-                raise ValueError()
-            elif msg == 'Timeout':
-                self._timeout = True
-                self._timeout_queue.get()
-        except:
-            self._exception_info = sys.exc_info()
-
-    def next_timeout(self):
-        if self._timeout:
-            self._timeout_queue.put('done')
-            return time.time() - 10
-        return time.time()
-
-
-class TestHandler(logging.Handler):
-    def __init__(self, astream):
-        logging.Handler.__init__(self)
-        self._stream = astream
-
-    def emit(self, record):
-        self._stream.write(self.format(record))
-
-
-class WaitForThreadsToFinishTest(unittest.TestCase):
-    class MockTestRunner(run_webkit_tests.TestRunner):
-        def __init__(self):
-            pass
-
-        def __del__(self):
-            pass
-
-        def update_summary(self, result_summary):
-            pass
-
-    def run_one_thread(self, msg):
-        runner = self.MockTestRunner()
-        starting_queue = Queue.Queue()
-        stopping_queue = Queue.Queue()
-        child_thread = TestThread(starting_queue, stopping_queue)
-        child_thread.start()
-        started_msg = starting_queue.get()
-        stopping_queue.put(msg)
-        threads = [child_thread]
-        return runner._wait_for_threads_to_finish(threads, None)
-
-    def test_basic(self):
-        interrupted = self.run_one_thread('')
-        self.assertFalse(interrupted)
-
-    def test_interrupt(self):
-        interrupted = self.run_one_thread('KeyboardInterrupt')
-        self.assertTrue(interrupted)
-
-    def test_timeout(self):
-        oc = outputcapture.OutputCapture()
-        oc.capture_output()
-        interrupted = self.run_one_thread('Timeout')
-        self.assertFalse(interrupted)
-        oc.restore_output()
-
-    def test_exception(self):
-        self.assertRaises(ValueError, self.run_one_thread, 'Exception')
-
-
-class StandaloneFunctionsTest(unittest.TestCase):
-    def test_log_wedged_thread(self):
-        oc = outputcapture.OutputCapture()
-        oc.capture_output()
-        logger = run_webkit_tests._log
-        astream = array_stream.ArrayStream()
-        handler = TestHandler(astream)
-        logger.addHandler(handler)
-
-        starting_queue = Queue.Queue()
-        stopping_queue = Queue.Queue()
-        child_thread = TestThread(starting_queue, stopping_queue)
-        child_thread.start()
-        msg = starting_queue.get()
-
-        run_webkit_tests._log_wedged_thread(child_thread)
-        stopping_queue.put('')
-        child_thread.join(timeout=1.0)
-
-        self.assertFalse(astream.empty())
-        self.assertFalse(child_thread.isAlive())
-        oc.restore_output()
-
-
 if __name__ == '__main__':
     unittest.main()