test-webkitpy: prepare for better test run output
authordpranke@chromium.org <dpranke@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 27 Mar 2012 19:15:53 +0000 (19:15 +0000)
committerdpranke@chromium.org <dpranke@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 27 Mar 2012 19:15:53 +0000 (19:15 +0000)
https://bugs.webkit.org/show_bug.cgi?id=82290

Reviewed by Adam Barth.

This code basically re-implements the output of the TextTestRunner default
runner code from unittest, although the implementation is quite
different, in preparation for changing the test output to be
metered and possibly running in parallel.

The output is almost identical to before, except that instead of
logging "test_regular (webkitpy.main.RunnerTest) passed" we log
"webkitpy.main.RunnerTest.test_regular passed". It has always
annoyed me that they invert the names to be harder to read and
so that you can't copy & paste back to the input for
test-webkitpy.

This patch is provided to add a skeleton for unit tests and for
comparison to an upcoming patch that will actually add new
functionality.

* Scripts/webkitpy/test/main.py:
(Tester.__init__):
(Tester._configure_logging):
(Tester._run_tests):
* Scripts/webkitpy/test/runner.py: Added.
(TestRunner):
(TestRunner.__init__):
(TestRunner.test_name):
(TestRunner.all_test_names):
(TestRunner.run):
(TestRunner.write_result):
(TestRunner.write_summary):
* Scripts/webkitpy/test/runner_unittest.py: Added.
(FakeModuleSuite):
(FakeModuleSuite.__init__):
(FakeModuleSuite.__str__):
(FakeModuleSuite.run):
(FakeTopSuite):
(FakeTopSuite.__init__):
(FakeLoader):
(FakeLoader.__init__):
(FakeLoader.top_suite):
(FakeLoader.loadTestsFromName):
(RunnerTest):
(RunnerTest.test_regular):
(RunnerTest.test_verbose):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@112298 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/webkitpy/test/main.py
Tools/Scripts/webkitpy/test/runner.py [new file with mode: 0644]
Tools/Scripts/webkitpy/test/runner_unittest.py [new file with mode: 0644]

index 7c0b1a2..70a59ca 100644 (file)
@@ -1,3 +1,53 @@
+2012-03-27  Dirk Pranke  <dpranke@chromium.org>
+
+        test-webkitpy: prepare for better test run output
+        https://bugs.webkit.org/show_bug.cgi?id=82290
+
+        Reviewed by Adam Barth.
+
+        This code basically re-implements the output of the TextTestRunner default
+        runner code from unittest, although the implementation is quite
+        different, in preparation for changing the test output to be
+        metered and possibly running in parallel.
+
+        The output is almost identical to before, except that instead of
+        logging "test_regular (webkitpy.main.RunnerTest) passed" we log
+        "webkitpy.main.RunnerTest.test_regular passed". It has always
+        annoyed me that they invert the names to be harder to read and
+        so that you can't copy & paste back to the input for
+        test-webkitpy.
+
+        This patch is provided to add a skeleton for unit tests and for
+        comparison to an upcoming patch that will actually add new
+        functionality.
+
+        * Scripts/webkitpy/test/main.py:
+        (Tester.__init__):
+        (Tester._configure_logging):
+        (Tester._run_tests):
+        * Scripts/webkitpy/test/runner.py: Added.
+        (TestRunner):
+        (TestRunner.__init__):
+        (TestRunner.test_name):
+        (TestRunner.all_test_names):
+        (TestRunner.run):
+        (TestRunner.write_result):
+        (TestRunner.write_summary):
+        * Scripts/webkitpy/test/runner_unittest.py: Added.
+        (FakeModuleSuite):
+        (FakeModuleSuite.__init__):
+        (FakeModuleSuite.__str__):
+        (FakeModuleSuite.run):
+        (FakeTopSuite):
+        (FakeTopSuite.__init__):
+        (FakeLoader):
+        (FakeLoader.__init__):
+        (FakeLoader.top_suite):
+        (FakeLoader.loadTestsFromName):
+        (RunnerTest):
+        (RunnerTest.test_regular):
+        (RunnerTest.test_verbose):
+
 2012-03-27  Gustavo Noronha Silva  <gns@gnome.org>
 
         [GTK] Build gnutls without p11-kit support
index b2ebabe..a035794 100644 (file)
@@ -32,6 +32,7 @@ import unittest
 
 from webkitpy.common.system.filesystem import FileSystem
 from webkitpy.test.test_finder import TestFinder
+from webkitpy.test.runner import TestRunner
 
 _log = logging.getLogger(__name__)
 
@@ -40,6 +41,7 @@ class Tester(object):
     def __init__(self, filesystem=None):
         self._verbosity = 1
         self.finder = TestFinder(filesystem or FileSystem())
+        self.stream = sys.stderr
 
     def add_tree(self, top_directory, starting_subdirectory=None):
         self.finder.add_tree(top_directory, starting_subdirectory)
@@ -92,7 +94,7 @@ class Tester(object):
         except for messages from the autoinstall module.  Also set the
         logging level as described below.
         """
-        handler = logging.StreamHandler(sys.stderr)
+        handler = logging.StreamHandler(self.stream)
         # We constrain the level on the handler rather than on the root
         # logger itself.  This is probably better because the handler is
         # configured and known only to this module, whereas the root logger
@@ -142,7 +144,7 @@ class Tester(object):
         names = self.finder.find_names(args, self._options.skip_integrationtests, self._options.all)
         return self._run_tests(names)
 
-    def _run_tests(self, args):
+    def _run_tests(self, names):
         if self._options.coverage:
             try:
                 import webkitpy.thirdparty.autoinstalled.coverage as coverage
@@ -159,16 +161,18 @@ class Tester(object):
 
         loader = unittest.defaultTestLoader
         suites = []
-        for name in args:
+        for name in names:
             if self.finder.is_module(name):
-                # import modules explicitly before loading their tests because
-                # loadTestsFromName() produces lousy error messages for bad modules.
+                # if we failed to load a name and it looks like a module,
+                # try importing it directly, because loadTestsFromName()
+                # produces lousy error messages for bad modules.
                 try:
                     __import__(name)
                 except ImportError, e:
                     _log.fatal('Failed to import %s:' % name)
                     self._log_exception()
                     return False
+
             suites.append(loader.loadTestsFromName(name, None))
 
         test_suite = unittest.TestSuite(suites)
@@ -176,7 +180,7 @@ class Tester(object):
             from webkitpy.thirdparty.autoinstalled.xmlrunner import XMLTestRunner
             test_runner = XMLTestRunner(output='test-webkitpy-xml-reports')
         else:
-            test_runner = unittest.TextTestRunner(verbosity=self._verbosity)
+            test_runner = TestRunner(self.stream, self._options, loader)
 
         _log.debug("Running the tests.")
         result = test_runner.run(test_suite)
diff --git a/Tools/Scripts/webkitpy/test/runner.py b/Tools/Scripts/webkitpy/test/runner.py
new file mode 100644 (file)
index 0000000..b2b0dfc
--- /dev/null
@@ -0,0 +1,129 @@
+# Copyright (C) 2012 Google, Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1.  Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+# 2.  Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""code to actually run a list of python tests."""
+
+import logging
+import re
+import time
+import unittest
+
+
+_log = logging.getLogger(__name__)
+
+
+class TestRunner(object):
+    def __init__(self, stream, options, loader):
+        self.options = options
+        self.stream = stream
+        self.loader = loader
+        self.test_description = re.compile("(\w+) \(([\w.]+)\)")
+
+    def test_name(self, test):
+        m = self.test_description.match(str(test))
+        return "%s.%s" % (m.group(2), m.group(1))
+
+    def all_test_names(self, suite):
+        names = []
+        if hasattr(suite, '_tests'):
+            for t in suite._tests:
+                names.extend(self.all_test_names(t))
+        else:
+            names.append(self.test_name(suite))
+        return names
+
+    def run(self, suite):
+        run_start_time = time.time()
+        all_test_names = self.all_test_names(suite)
+        result = unittest.TestResult()
+        stop = run_start_time
+        for test_name in all_test_names:
+            if self.options.verbose:
+                self.stream.write(test_name)
+            num_failures = len(result.failures)
+            num_errors = len(result.errors)
+
+            start = time.time()
+            # FIXME: it's kinda lame that we re-load the test suites for each
+            # test, and this may slow things down, but this makes implementing
+            # the logging easy and will also allow us to parallelize nicely.
+            self.loader.loadTestsFromName(test_name, None).run(result)
+            stop = time.time()
+
+            err = None
+            failure = None
+            if len(result.failures) > num_failures:
+                failure = result.failures[num_failures][1]
+            elif len(result.errors) > num_errors:
+                err = result.errors[num_errors][1]
+            self.write_result(result, test_name, stop - start, failure, err)
+
+        self.write_summary(result, stop - run_start_time)
+
+        return result
+
+    def write_result(self, result, test_name, test_time, failure=None, err=None):
+        if self.options.verbose:
+            if failure:
+                msg = ' failed'
+            elif err:
+                msg = ' erred'
+            else:
+                msg = ' passed'
+            self.stream.write(msg + '\n')
+        else:
+            if failure:
+                msg = 'F'
+            elif err:
+                msg = 'E'
+            else:
+                msg = '.'
+            self.stream.write(msg)
+
+    def write_summary(self, result, run_time):
+        self.stream.write('\n')
+
+        for (test, err) in result.errors:
+            self.stream.write("=" * 80 + '\n')
+            self.stream.write("ERROR: " + self.test_name(test) + '\n')
+            self.stream.write("-" * 80 + '\n')
+            for line in err.splitlines():
+                self.stream.write(line + '\n')
+            self.stream.write('\n')
+
+        for (test, failure) in result.failures:
+            self.stream.write("=" * 80 + '\n')
+            self.stream.write("FAILURE: " + self.test_name(test) + '\n')
+            self.stream.write("-" * 80 + '\n')
+            for line in failure.splitlines():
+                self.stream.write(line + '\n')
+            self.stream.write('\n')
+
+        self.stream.write('-' * 80 + '\n')
+        self.stream.write('Ran %d test%s in %.3fs\n' %
+            (result.testsRun, result.testsRun != 1 and "s" or "", run_time))
+
+        if result.wasSuccessful():
+            self.stream.write('\nOK\n')
+        else:
+            self.stream.write('FAILED (failures=%d, errors=%d)\n' %
+                (len(result.failures), len(result.errors)))
diff --git a/Tools/Scripts/webkitpy/test/runner_unittest.py b/Tools/Scripts/webkitpy/test/runner_unittest.py
new file mode 100644 (file)
index 0000000..70c0b37
--- /dev/null
@@ -0,0 +1,99 @@
+# Copyright (C) 2012 Google, Inc.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1.  Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+# 2.  Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import re
+import StringIO
+import unittest
+
+from webkitpy.tool.mocktool import MockOptions
+from webkitpy.test.runner import TestRunner
+
+
+class FakeModuleSuite(object):
+    def __init__(self, name, result, msg):
+        self.name = name
+        self.result = result
+        self.msg = msg
+
+    def __str__(self):
+        return self.name
+
+    def run(self, result):
+        result.testsRun += 1
+        if self.result == 'F':
+            result.failures.append((self.name, self.msg))
+        elif self.result == 'E':
+            result.errors.append((self.name, self.msg))
+
+
+class FakeTopSuite(object):
+    def __init__(self, tests):
+        self._tests = tests
+
+
+class FakeLoader(object):
+    def __init__(self, *test_triples):
+        self.triples = test_triples
+        self._tests = []
+        self._results = {}
+        for test_name, result, msg in self.triples:
+            self._tests.append(test_name)
+            m = re.match("(\w+) \(([\w.]+)\)", test_name)
+            self._results['%s.%s' % (m.group(2), m.group(1))] = tuple([test_name, result, msg])
+
+    def top_suite(self):
+        return FakeTopSuite(self._tests)
+
+    def loadTestsFromName(self, name, dummy):
+        return FakeModuleSuite(*self._results[name])
+
+
+class RunnerTest(unittest.TestCase):
+    def test_regular(self):
+        options = MockOptions(verbose=False)
+        stream = StringIO.StringIO()
+        loader = FakeLoader(('test1 (Foo)', '.', ''),
+                            ('test2 (Foo)', 'F', 'test2\nfailed'),
+                            ('test3 (Foo)', 'E', 'test3\nerred'))
+        result = TestRunner(stream, options, loader).run(loader.top_suite())
+        self.assertFalse(result.wasSuccessful())
+        self.assertEquals(result.testsRun, 3)
+        self.assertEquals(len(result.failures), 1)
+        self.assertEquals(len(result.errors), 1)
+        # FIXME: check the output from the test
+
+    def test_verbose(self):
+        options = MockOptions(verbose=True)
+        stream = StringIO.StringIO()
+        loader = FakeLoader(('test1 (Foo)', '.', ''),
+                            ('test2 (Foo)', 'F', 'test2\nfailed'),
+                            ('test3 (Foo)', 'E', 'test3\nerred'))
+        result = TestRunner(stream, options, loader).run(loader.top_suite())
+        self.assertFalse(result.wasSuccessful())
+        self.assertEquals(result.testsRun, 3)
+        self.assertEquals(len(result.failures), 1)
+        self.assertEquals(len(result.errors), 1)
+        # FIXME: check the output from the test
+
+
+if __name__ == '__main__':
+    unittest.main()