new-run-webkit-tests: fix bugs in 'mock' drt implementation
authordpranke@chromium.org <dpranke@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 6 Apr 2012 22:30:37 +0000 (22:30 +0000)
committerdpranke@chromium.org <dpranke@chromium.org@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Fri, 6 Apr 2012 22:30:37 +0000 (22:30 +0000)
https://bugs.webkit.org/show_bug.cgi?id=83341

Reviewed by Eric Seidel.

The current implementation crashed in some circumstances for
reference tests and files that were missing expected results.
This patch fixes those cases and adds more unit tests; it is a
precursor to adding unit tests each port that actually uses this
code, in order to get better coverage of the port/*
implementations.

* Scripts/webkitpy/layout_tests/port/mock_drt.py:
(parse_options):
(MockDRT.input_from_line):
(MockDRT.output_for_test):
(MockDRT.run_one_test):
(MockChromiumDRT.input_from_line):
(MockChromiumDRT.output_for_test):
(MockChromiumDRT.run_one_test):
* Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py:
(MockDRTTest.make_input_output):
(MockDRTTest.expected_output):
(MockDRTTest):
(MockDRTTest.assertTest):
(MockDRTTest.test_missing_image):
(MockDRTTest.test_missing_text):
(MockDRTTest.test_reftest_match):
(MockDRTTest.test_reftest_mismatch):
(MockChromiumDRTTest.expected_output):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@113507 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/webkitpy/layout_tests/port/mock_drt.py
Tools/Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py

index 7675a19..6152920 100644 (file)
@@ -1,3 +1,36 @@
+2012-04-06  Dirk Pranke  <dpranke@chromium.org>
+
+        new-run-webkit-tests: fix bugs in 'mock' drt implementation
+        https://bugs.webkit.org/show_bug.cgi?id=83341
+
+        Reviewed by Eric Seidel.
+
+        The current implementation crashed in some circumstances for
+        reference tests and files that were missing expected results.
+        This patch fixes those cases and adds more unit tests; it is a
+        precursor to adding unit tests each port that actually uses this
+        code, in order to get better coverage of the port/*
+        implementations.
+
+        * Scripts/webkitpy/layout_tests/port/mock_drt.py:
+        (parse_options):
+        (MockDRT.input_from_line):
+        (MockDRT.output_for_test):
+        (MockDRT.run_one_test):
+        (MockChromiumDRT.input_from_line):
+        (MockChromiumDRT.output_for_test):
+        (MockChromiumDRT.run_one_test):
+        * Scripts/webkitpy/layout_tests/port/mock_drt_unittest.py:
+        (MockDRTTest.make_input_output):
+        (MockDRTTest.expected_output):
+        (MockDRTTest):
+        (MockDRTTest.assertTest):
+        (MockDRTTest.test_missing_image):
+        (MockDRTTest.test_missing_text):
+        (MockDRTTest.test_reftest_match):
+        (MockDRTTest.test_reftest_mismatch):
+        (MockChromiumDRTTest.expected_output):
+
 2012-04-06  James Robinson  <jamesr@chromium.org>
 
         Enable webkit_unit_tests for commit queue and EWS while tracking failures
index 3d41ebd..8c062c0 100644 (file)
@@ -161,7 +161,7 @@ def parse_options(argv):
     else:
         pixel_tests = '--pixel-tests' in argv
     options = MockOptions(chromium=chromium, platform=platform, pixel_tests=pixel_tests, pixel_path=pixel_path)
-    return (options, [])
+    return (options, argv)
 
 
 class MockDRT(object):
@@ -199,9 +199,9 @@ class MockDRT(object):
         else:
             test_name = self._port.relative_test_filename(uri)
 
+        dirname, basename = self._port.split_test(test_name)
         is_reftest = (self._port.reference_files(test_name) or
-                      test_name.endswith('-expected.html') or
-                      test_name.endswith('-mismatch.html'))
+                      self._port.is_reference_html_file(self._port._filesystem, dirname, basename))
         return DriverInput(test_name, 0, checksum, is_reftest)
 
     def output_for_test(self, test_input):
@@ -213,10 +213,11 @@ class MockDRT(object):
         if test_input.is_reftest:
             # Make up some output for reftests.
             actual_text = 'reference text\n'
-            actual_checksum = 'None'
+            actual_checksum = 'mock-checksum'
             actual_image = 'blank'
-            if test_name.endswith('-mismatch.html'):
-                actual_checksum = 'True'
+            if test_input.test_name.endswith('-mismatch.html'):
+                actual_text = 'not reference text\n'
+                actual_checksum = 'not-mock-checksum'
                 actual_image = 'not blank'
         elif self._options.pixel_tests and test_input.image_hash:
             actual_checksum = port.expected_checksum(test_input.test_name)
@@ -235,11 +236,12 @@ class MockDRT(object):
             self._stdout.write('Content-Type: text/plain\n')
             # FIXME: Note that we don't ensure there is a trailing newline!
             # This mirrors actual (Mac) DRT behavior but is a bug.
-            self._stdout.write(output.text)
+            if output.text:
+                self._stdout.write(output.text)
 
         self._stdout.write('#EOF\n')
 
-        if self._options.pixel_tests and (test_input.image_hash or is_reftest):
+        if self._options.pixel_tests and (test_input.image_hash or test_input.is_reftest):
             self._stdout.write('\n')
             self._stdout.write('ActualHash: %s\n' % output.image_hash)
             self._stdout.write('ExpectedHash: %s\n' % test_input.image_hash)
@@ -263,26 +265,34 @@ class MockChromiumDRT(MockDRT):
             checksum = None
 
         test_name = self._driver.uri_to_test(uri)
+        dirname, basename = self._port.split_test(test_name)
         is_reftest = (self._port.reference_files(test_name) or
-                      test_name.endswith('-expected.html') or
-                      test_name.endswith('-mismatch.html'))
+                      self._port.is_reference_html_file(self._port._filesystem, dirname, basename))
 
         return DriverInput(test_name, timeout, checksum, is_reftest)
 
+    def output_for_test(self, test_input):
+        # FIXME: This is a hack to make virtual tests work. Need something more general.
+        original_test_name = test_input.test_name
+        if '--enable-accelerated-2d-canvas' in self._args and 'canvas' in test_input.test_name:
+            test_input.test_name = 'platform/chromium/virtual/gpu/' + test_input.test_name
+        output = super(MockChromiumDRT, self).output_for_test(test_input)
+        test_input.test_name = original_test_name
+        return output
+
     def run_one_test(self, test_input):
         output = self.output_for_test(test_input)
 
         self._stdout.write("#URL:%s\n" % self._driver.test_to_uri(test_input.test_name))
         if self._options.pixel_tests and (test_input.image_hash or test_input.is_reftest):
             self._stdout.write("#MD5:%s\n" % output.image_hash)
-            self._host.filesystem.maybe_make_directory(self._host.filesystem.dirname(self._options.pixel_path))
-            self._host.filesystem.write_binary_file(self._options.pixel_path,
-                                                    output.image)
-        self._stdout.write(output.text)
-
-        # FIXME: (See above FIXME as well). Chromium DRT appears to always
-        # ensure the text output has a trailing newline. Mac DRT does not.
-        if not output.text.endswith('\n'):
+            if output.image:
+                self._host.filesystem.maybe_make_directory(self._host.filesystem.dirname(self._options.pixel_path))
+                self._host.filesystem.write_binary_file(self._options.pixel_path, output.image)
+        if output.text:
+            self._stdout.write(output.text)
+
+        if output.text and not output.text.endswith('\n'):
             self._stdout.write('\n')
         self._stdout.write('#EOF\n')
         self._stdout.flush()
index bcb9119..6050718 100755 (executable)
@@ -111,13 +111,13 @@ class MockDRTTest(unittest.TestCase):
         return mock_drt.MockDRT(options, args, host, stdin, stdout, stderr)
 
     def make_input_output(self, port, test_name, pixel_tests,
-                          expected_checksum, drt_output, drt_input=None):
+                          expected_checksum, drt_output, drt_input=None, expected_text=None):
         if pixel_tests:
             if not expected_checksum:
                 expected_checksum = port.expected_checksum(test_name)
         if not drt_input:
             drt_input = self.input_line(port, test_name, expected_checksum)
-        text_output = port.expected_text(test_name)
+        text_output = expected_text or port.expected_text(test_name) or ''
 
         if not drt_output:
             drt_output = self.expected_output(port, test_name, pixel_tests,
@@ -125,27 +125,24 @@ class MockDRTTest(unittest.TestCase):
         return (drt_input, drt_output)
 
     def expected_output(self, port, test_name, pixel_tests, text_output, expected_checksum):
+        output = ['Content-Type: text/plain\n']
+        if text_output:
+            output.append(text_output)
+        output.append('#EOF\n')
         if pixel_tests and expected_checksum:
-            return ['Content-Type: text/plain\n',
-                    text_output,
-                    '#EOF\n',
-                    '\n',
-                    'ActualHash: %s\n' % expected_checksum,
-                    'ExpectedHash: %s\n' % expected_checksum,
-                    '#EOF\n']
-        else:
-            return ['Content-Type: text/plain\n',
-                    text_output,
-                    '#EOF\n',
-                    '#EOF\n']
-
-    def assertTest(self, test_name, pixel_tests, expected_checksum=None, drt_output=None, host=None):
+            output.extend(['\n',
+                           'ActualHash: %s\n' % expected_checksum,
+                           'ExpectedHash: %s\n' % expected_checksum])
+        output.append('#EOF\n')
+        return output
+
+    def assertTest(self, test_name, pixel_tests, expected_checksum=None, drt_output=None, host=None, expected_text=None):
         port_name = 'test'
         host = host or MockSystemHost()
         test.add_unit_tests_to_mock_filesystem(host.filesystem)
         port = PortFactory(host).get(port_name)
         drt_input, drt_output = self.make_input_output(port, test_name,
-            pixel_tests, expected_checksum, drt_output)
+            pixel_tests, expected_checksum, drt_output, drt_input=None, expected_text=expected_text)
 
         args = ['--platform', port_name] + self.extra_args(pixel_tests)
         stdin = newstringio.StringIO(drt_input)
@@ -197,6 +194,21 @@ class MockDRTTest(unittest.TestCase):
     def test_checksum_in_png(self):
         self.assertTest('passes/checksum_in_image.html', True)
 
+    def test_missing_image(self):
+        self.assertTest('failures/expected/missing_image.html', True)
+
+    def test_missing_text(self):
+        self.assertTest('failures/expected/missing_text.html', True)
+
+    def test_reftest_match(self):
+        self.assertTest('passes/reftest.html', False, expected_checksum='mock-checksum', expected_text='reference text\n')
+        self.assertTest('passes/reftest.html', True, expected_checksum='mock-checksum', expected_text='reference text\n')
+
+    def test_reftest_mismatch(self):
+        self.assertTest('passes/mismatch.html', False, expected_checksum='mock-checksum', expected_text='reference text\n')
+        self.assertTest('passes/mismatch.html', True, expected_checksum='mock-checksum', expected_text='reference text\n')
+
+
 
 class MockChromiumDRTTest(MockDRTTest):
     def extra_args(self, pixel_tests):
@@ -222,17 +234,15 @@ class MockChromiumDRTTest(MockDRTTest):
 
     def expected_output(self, port, test_name, pixel_tests, text_output, expected_checksum):
         url = port.create_driver(0).test_to_uri(test_name)
-        if pixel_tests and expected_checksum:
-            return ['#URL:%s\n' % url,
-                    '#MD5:%s\n' % expected_checksum,
-                    text_output,
-                    '\n',
-                    '#EOF\n']
-        else:
-            return ['#URL:%s\n' % url,
-                    text_output,
-                    '\n',
-                    '#EOF\n']
+        output = ['#URL:%s\n' % url]
+        if expected_checksum:
+            output.append('#MD5:%s\n' % expected_checksum)
+        if text_output:
+            output.append(text_output)
+            if not text_output.endswith('\n'):
+                output.append('\n')
+        output.append('#EOF\n')
+        return output
 
     def test_pixeltest__fails(self):
         host = MockSystemHost()