REGRESSION (r126189): No more mismatch-did-not-occur failures in reftests
authorzandobersek@gmail.com <zandobersek@gmail.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 26 Sep 2012 07:18:16 +0000 (07:18 +0000)
committerzandobersek@gmail.com <zandobersek@gmail.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 26 Sep 2012 07:18:16 +0000 (07:18 +0000)
https://bugs.webkit.org/show_bug.cgi?id=97112

Reviewed by Dirk Pranke.

Don't use the reftest's image hash when gathering output of its reference.
This indirectly forces the driver to return the reference's image output
as well. This specifically solves the problem when the reftest and its
mismatch reference are equal but the unexpected match failure is not
reported due to missing image output of the reference file.

* Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
(SingleTestRunner._run_reftest):
* Scripts/webkitpy/layout_tests/port/test.py:
(TestPort.diff_image):
(TestDriver.run_test):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@129600 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Tools/ChangeLog
Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py
Tools/Scripts/webkitpy/layout_tests/port/test.py

index 453dfe2..fe52caf 100644 (file)
@@ -1,3 +1,22 @@
+2012-09-26  Zan Dobersek  <zandobersek@gmail.com>
+
+        REGRESSION (r126189): No more mismatch-did-not-occur failures in reftests
+        https://bugs.webkit.org/show_bug.cgi?id=97112
+
+        Reviewed by Dirk Pranke.
+
+        Don't use the reftest's image hash when gathering output of its reference.
+        This indirectly forces the driver to return the reference's image output
+        as well. This specifically solves the problem when the reftest and its
+        mismatch reference are equal but the unexpected match failure is not
+        reported due to missing image output of the reference file.
+
+        * Scripts/webkitpy/layout_tests/controllers/single_test_runner.py:
+        (SingleTestRunner._run_reftest):
+        * Scripts/webkitpy/layout_tests/port/test.py:
+        (TestPort.diff_image):
+        (TestDriver.run_test):
+
 2012-09-25  Ryosuke Niwa  <rniwa@webkit.org>
 
         Update master.cfg to invoke run-perf-tests with --no-show-results and --reset-results
index 0cb7aef..28e9d63 100644 (file)
@@ -295,7 +295,7 @@ class SingleTestRunner(object):
         putAllMismatchBeforeMatch = sorted
         for expectation, reference_filename in putAllMismatchBeforeMatch(self._reference_files):
             reference_test_name = self._port.relative_test_filename(reference_filename)
-            reference_output = self._driver.run_test(DriverInput(reference_test_name, self._timeout, test_output.image_hash, should_run_pixel_test=True), self._stop_when_done)
+            reference_output = self._driver.run_test(DriverInput(reference_test_name, self._timeout, None, should_run_pixel_test=True), self._stop_when_done)
             test_result = self._compare_output_with_reference(reference_output, test_output, reference_filename, expectation == '!=')
 
             if (expectation == '!=' and test_result.failures) or (expectation == '==' and not test_result.failures):
index cbc0dbe..d008d99 100644 (file)
@@ -405,6 +405,10 @@ class TestPort(Port):
 
     def diff_image(self, expected_contents, actual_contents, tolerance=None):
         diffed = actual_contents != expected_contents
+        if not actual_contents and not expected_contents:
+            return (None, 0, None)
+        if not actual_contents or not expected_contents:
+            return (True, 0, None)
         if 'ref' in expected_contents:
             assert tolerance == 0
         if diffed:
@@ -568,7 +572,11 @@ class TestDriver(Driver):
         if stop_when_done:
             self.stop()
 
-        return DriverOutput(actual_text, test.actual_image, test.actual_checksum, audio,
+        if test.actual_checksum == test_input.image_hash:
+            image = None
+        else:
+            image = test.actual_image
+        return DriverOutput(actual_text, image, test.actual_checksum, audio,
             crash=test.crash or test.web_process_crash, crashed_process_name=crashed_process_name,
             crashed_pid=crashed_pid, crash_log=crash_log,
             test_time=time.time() - start_time, timeout=test.timeout, error=test.error)