test-webkitpy: clean up runner in preparation for running tests serially as necessary
authordpranke@chromium.org <dpranke@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 2 Aug 2012 21:35:34 +0000 (21:35 +0000)
committerdpranke@chromium.org <dpranke@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 2 Aug 2012 21:35:34 +0000 (21:35 +0000)
https://bugs.webkit.org/show_bug.cgi?id=92922

Reviewed by Ojan Vafai.

In order to run some tests by themselves (serially, rather than
in parallel with other tests), we will need to be able to run
multiple test suites; this causes us to move loading the logic
for finding test method names out of the runner and into main.

I'm taking advantage of this to simplify some other stuff from
the runner as well; it is now very simple and doesn't expose its
dependency on unittest.TestResult at all (nor will the Printer
use TestResult).

Subsequent patches will move the custom loader from
port_testcase so that we can properly choose whether to run
integration tests and/or serial tests, and then update the
appropriate tests to run only serially.

* Scripts/webkitpy/test/main.py:
(Tester._run_tests):
(Tester):
(Tester._check_imports):
(Tester._test_names):
(Tester._all_test_names):
* Scripts/webkitpy/test/printer.py:
(Printer.__init__):
(Printer.write_update):
(Printer):
(Printer.print_finished_test):
(Printer.print_result):
* Scripts/webkitpy/test/runner.py:
(unit_test_name):
(Runner.__init__):
(Runner.run):
(Runner.handle):
(_Worker.handle):
* Scripts/webkitpy/test/runner_unittest.py:
(FakeLoader.loadTestsFromName):
(RunnerTest.test_run):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@124501 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/webkitpy/test/main.py
Tools/Scripts/webkitpy/test/printer.py
Tools/Scripts/webkitpy/test/runner.py
Tools/Scripts/webkitpy/test/runner_unittest.py

index 7ce6820..e191f68 100644 (file)
@@ -1,3 +1,47 @@
+2012-08-02  Dirk Pranke  <dpranke@chromium.org>
+
+        test-webkitpy: clean up runner in preparation for running tests serially as necessary
+        https://bugs.webkit.org/show_bug.cgi?id=92922
+
+        Reviewed by Ojan Vafai.
+
+        In order to run some tests by themselves (serially, rather than
+        in parallel with other tests), we will need to be able to run
+        multiple test suites; this causes us to move loading the logic
+        for finding test method names out of the runner and into main.
+
+        I'm taking advantage of this to simplify some other stuff from
+        the runner as well; it is now very simple and doesn't expose its
+        dependency on unittest.TestResult at all (nor will the Printer
+        use TestResult).
+
+        Subsequent patches will move the custom loader from
+        port_testcase so that we can properly choose whether to run
+        integration tests and/or serial tests, and then update the
+        appropriate tests to run only serially.
+
+        * Scripts/webkitpy/test/main.py:
+        (Tester._run_tests):
+        (Tester):
+        (Tester._check_imports):
+        (Tester._test_names):
+        (Tester._all_test_names):
+        * Scripts/webkitpy/test/printer.py:
+        (Printer.__init__):
+        (Printer.write_update):
+        (Printer):
+        (Printer.print_finished_test):
+        (Printer.print_result):
+        * Scripts/webkitpy/test/runner.py:
+        (unit_test_name):
+        (Runner.__init__):
+        (Runner.run):
+        (Runner.handle):
+        (_Worker.handle):
+        * Scripts/webkitpy/test/runner_unittest.py:
+        (FakeLoader.loadTestsFromName):
+        (RunnerTest.test_run):
+
 2012-08-02  Adam Barth  <abarth@webkit.org>
 
         Turn on tests for the mac-ews, for realz this time.
index 3932ad7..bfdf564 100644 (file)
@@ -29,13 +29,14 @@ import optparse
 import os
 import StringIO
 import sys
+import time
 import traceback
 import unittest
 
 from webkitpy.common.system.filesystem import FileSystem
 from webkitpy.test.finder import Finder
 from webkitpy.test.printer import Printer
-from webkitpy.test.runner import Runner
+from webkitpy.test.runner import Runner, unit_test_name
 
 _log = logging.getLogger(__name__)
 
@@ -128,10 +129,30 @@ class Tester(object):
         # Make sure PYTHONPATH is set up properly.
         sys.path = self.finder.additional_paths(sys.path) + sys.path
 
-        _log.debug("Loading the tests...")
+        self.printer.write_update("Checking imports ...")
+        if not self._check_imports(names):
+            return False
 
+        self.printer.write_update("Finding the individual test methods ...")
         loader = unittest.defaultTestLoader
-        suites = []
+        test_names = self._test_names(loader, names)
+
+        self.printer.write_update("Running the tests ...")
+        self.printer.num_tests = len(test_names)
+        start = time.time()
+        test_runner = Runner(self.printer, loader)
+        test_runner.run(test_names, self._options.child_processes)
+
+        self.printer.print_result(time.time() - start)
+
+        if self._options.coverage:
+            cov.stop()
+            cov.save()
+            cov.report(show_missing=False)
+
+        return not self.printer.num_errors and not self.printer.num_failures
+
+    def _check_imports(self, names):
         for name in names:
             if self.finder.is_module(name):
                 # if we failed to load a name and it looks like a module,
@@ -143,19 +164,23 @@ class Tester(object):
                     _log.fatal('Failed to import %s:' % name)
                     self._log_exception()
                     return False
+        return True
 
-            suites.append(loader.loadTestsFromName(name, None))
-
-        test_suite = unittest.TestSuite(suites)
-        test_runner = Runner(self.printer, self._options, loader)
-
-        _log.debug("Running the tests.")
-        result = test_runner.run(test_suite)
-        if self._options.coverage:
-            cov.stop()
-            cov.save()
-            cov.report(show_missing=False)
-        return result.wasSuccessful()
+    def _test_names(self, loader, names):
+        test_names = []
+        for name in names:
+            test_names.extend(self._all_test_names(loader.loadTestsFromName(name, None)))
+
+        return test_names
+
+    def _all_test_names(self, suite):
+        names = []
+        if hasattr(suite, '_tests'):
+            for t in suite._tests:
+                names.extend(self._all_test_names(t))
+        else:
+            names.append(unit_test_name(suite))
+        return names
 
     def _log_exception(self):
         s = StringIO.StringIO()
index 042fba1..0ec3035 100644 (file)
@@ -37,6 +37,8 @@ class Printer(object):
         self.options = options
         self.num_tests = 0
         self.num_completed = 0
+        self.num_errors = 0
+        self.num_failures = 0
         self.running_tests = []
         self.completed_tests = []
         if options:
@@ -102,6 +104,9 @@ class Printer(object):
         if self.options.pass_through:
             outputcapture.OutputCapture.stream_wrapper = _CaptureAndPassThroughStream
 
+    def write_update(self, msg):
+        self.meter.write_update(msg)
+
     def print_started_test(self, source, test_name):
         self.running_tests.append(test_name)
         if len(self.running_tests) > 1:
@@ -116,14 +121,16 @@ class Printer(object):
 
         write(self._test_line(self.running_tests[0], suffix))
 
-    def print_finished_test(self, result, test_name, test_time, failure, err):
+    def print_finished_test(self, source, test_name, test_time, failures, errors):
         write = self.meter.writeln
-        if failure:
-            lines = failure[0][1].splitlines() + ['']
+        if failures:
+            lines = failures[0].splitlines() + ['']
             suffix = ' failed:'
-        elif err:
-            lines = err[0][1].splitlines() + ['']
+            self.num_failures += 1
+        elif errors:
+            lines = errors[0].splitlines() + ['']
             suffix = ' erred:'
+            self.num_errors += 1
         else:
             suffix = ' passed'
             lines = []
@@ -154,13 +161,13 @@ class Printer(object):
     def _test_line(self, test_name, suffix):
         return '[%d/%d] %s%s' % (self.num_completed, self.num_tests, test_name, suffix)
 
-    def print_result(self, result, run_time):
+    def print_result(self, run_time):
         write = self.meter.writeln
-        write('Ran %d test%s in %.3fs' % (result.testsRun, result.testsRun != 1 and "s" or "", run_time))
-        if result.wasSuccessful():
-            write('\nOK\n')
+        write('Ran %d test%s in %.3fs' % (self.num_completed, self.num_completed != 1 and "s" or "", run_time))
+        if self.num_failures or self.num_errors:
+            write('FAILED (failures=%d, errors=%d)\n' % (self.num_failures, self.num_errors))
         else:
-            write('FAILED (failures=%d, errors=%d)\n' % (len(result.failures), len(result.errors)))
+            write('\nOK\n')
 
 
 class _CaptureAndPassThroughStream(object):
index fd8af6f..d3f5764 100644 (file)
 
 """code to actually run a list of python tests."""
 
-import logging
 import re
 import time
 import unittest
 
 from webkitpy.common import message_pool
 
-_log = logging.getLogger(__name__)
-
-
 _test_description = re.compile("(\w+) \(([\w.]+)\)")
 
 
-def _test_name(test):
+def unit_test_name(test):
     m = _test_description.match(str(test))
     return "%s.%s" % (m.group(2), m.group(1))
 
 
 class Runner(object):
-    def __init__(self, printer, options, loader):
-        self.options = options
+    def __init__(self, printer, loader):
         self.printer = printer
         self.loader = loader
-        self.result = unittest.TestResult()
+        self.tests_run = 0
+        self.errors = []
+        self.failures = []
         self.worker_factory = lambda caller: _Worker(caller, self.loader)
 
-    def all_test_names(self, suite):
-        names = []
-        if hasattr(suite, '_tests'):
-            for t in suite._tests:
-                names.extend(self.all_test_names(t))
-        else:
-            names.append(_test_name(suite))
-        return names
-
-    def run(self, suite):
-        run_start_time = time.time()
-        all_test_names = self.all_test_names(suite)
-        self.printer.num_tests = len(all_test_names)
-
-        with message_pool.get(self, self.worker_factory, int(self.options.child_processes)) as pool:
-            pool.run(('test', test_name) for test_name in all_test_names)
-
-        self.printer.print_result(self.result, time.time() - run_start_time)
-        return self.result
+    def run(self, test_names, num_workers):
+        if not test_names:
+            return
+        num_workers = min(num_workers, len(test_names))
+        with message_pool.get(self, self.worker_factory, num_workers) as pool:
+            pool.run(('test', test_name) for test_name in test_names)
 
-    def handle(self, message_name, source, test_name, delay=None, result=None):
+    def handle(self, message_name, source, test_name, delay=None, failures=None, errors=None):
         if message_name == 'started_test':
             self.printer.print_started_test(source, test_name)
             return
 
-        self.result.testsRun += 1
-        self.result.errors.extend(result.errors)
-        self.result.failures.extend(result.failures)
-        self.printer.print_finished_test(source, test_name, delay, result.failures, result.errors)
+        self.tests_run += 1
+        if failures:
+            self.failures.append((test_name, failures))
+        if errors:
+            self.errors.append((test_name, errors))
+        self.printer.print_finished_test(source, test_name, delay, failures, errors)
 
 
 class _Worker(object):
@@ -89,13 +75,8 @@ class _Worker(object):
         result = unittest.TestResult()
         start = time.time()
         self._caller.post('started_test', test_name)
-        self._loader.loadTestsFromName(test_name, None).run(result)
 
-        # The tests in the TestResult contain file objects and other unpicklable things; we only
-        # care about the test name, so we rewrite the result to replace the test with the test name.
-        # FIXME: We need an automated test for this, but I don't know how to write an automated
-        # test that will fail in this case that doesn't get picked up by test-webkitpy normally :(.
-        result.failures = [(_test_name(failure[0]), failure[1]) for failure in result.failures]
-        result.errors = [(_test_name(error[0]), error[1]) for error in result.errors]
-
-        self._caller.post('finished_test', test_name, time.time() - start, result)
+        # We will need to rework this if a test_name results in multiple tests.
+        self._loader.loadTestsFromName(test_name, None).run(result)
+        self._caller.post('finished_test', test_name, time.time() - start,
+            [failure[1] for failure in result.failures], [error[1] for error in result.errors])
index 4cd3a49..8fe1b06 100644 (file)
@@ -65,7 +65,7 @@ class FakeLoader(object):
     def top_suite(self):
         return FakeTopSuite(self._tests)
 
-    def loadTestsFromName(self, name, dummy):
+    def loadTestsFromName(self, name, _):
         return FakeModuleSuite(*self._results[name])
 
 
@@ -84,28 +84,17 @@ class RunnerTest(unittest.TestCase):
         for handler in self.log_handlers:
             handler.level = self.log_levels.pop(0)
 
-    def assert_run(self, verbose=0, timing=False, child_processes=1, quiet=False):
+    def test_run(self, verbose=0, timing=False, child_processes=1, quiet=False):
         options = MockOptions(verbose=verbose, timing=timing, child_processes=child_processes, quiet=quiet, pass_through=False)
         stream = StringIO.StringIO()
         loader = FakeLoader(('test1 (Foo)', '.', ''),
                             ('test2 (Foo)', 'F', 'test2\nfailed'),
                             ('test3 (Foo)', 'E', 'test3\nerred'))
-        runner = Runner(Printer(stream, options), options, loader)
-        result = runner.run(loader.top_suite())
-        self.assertFalse(result.wasSuccessful())
-        self.assertEquals(result.testsRun, 3)
-        self.assertEquals(len(result.failures), 1)
-        self.assertEquals(len(result.errors), 1)
-        # FIXME: check the output from the test
-
-    def test_regular(self):
-        self.assert_run()
-
-    def test_verbose(self):
-        self.assert_run(verbose=1)
-
-    def test_timing(self):
-        self.assert_run(timing=True)
+        runner = Runner(Printer(stream, options), loader)
+        runner.run(['Foo.test1', 'Foo.test2', 'Foo.test3'], 1)
+        self.assertEquals(runner.tests_run, 3)
+        self.assertEquals(len(runner.failures), 1)
+        self.assertEquals(len(runner.errors), 1)
 
 
 if __name__ == '__main__':