Add tolerance to WebAudio tests
authorap@apple.com <ap@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 16 Mar 2015 21:52:40 +0000 (21:52 +0000)
committerap@apple.com <ap@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Mon, 16 Mar 2015 21:52:40 +0000 (21:52 +0000)
https://bugs.webkit.org/show_bug.cgi?id=142676

Reviewed by Tim Horton.

Tools:

* Scripts/webkitpy/common/wavediff.py: Added. Based on Jer Noble's work.

* Scripts/webkitpy/layout_tests/controllers/test_result_writer.py:
(TestResultWriter.create_audio_diff_and_write_result):
* Scripts/webkitpy/layout_tests/models/test_failures.py:
(FailureAudio.write_failure):
* Scripts/webkitpy/port/base.py:
(Port.do_audio_results_differ):
Diff audio failures.

* Scripts/webkitpy/port/test.py: Added a test for the tolerance, fixed existing
tests to use real parseable WAV data, and got rid of base64, which there didn't
seem to have been any reason for.

LayoutTests:

* fast/harness/results.html: Display a diff link for audio tests, as we now have the diff.

* platform/mac/TestExpectations: Unmark tests that should now pass everywhere.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@181579 268f45cc-cd09-0410-ab3c-d52691b4dbfc

LayoutTests/ChangeLog
LayoutTests/fast/harness/results.html
LayoutTests/platform/mac/TestExpectations
Tools/ChangeLog
Tools/Scripts/webkitpy/common/wavediff.py [new file with mode: 0644]
Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py
Tools/Scripts/webkitpy/layout_tests/models/test_failures.py
Tools/Scripts/webkitpy/port/base.py
Tools/Scripts/webkitpy/port/test.py

index 4329535..a8ea2a2 100644 (file)
@@ -1,3 +1,14 @@
+2015-03-16  Alexey Proskuryakov  <ap@apple.com>
+
+        Add tolerance to WebAudio tests
+        https://bugs.webkit.org/show_bug.cgi?id=142676
+
+        Reviewed by Tim Horton.
+
+        * fast/harness/results.html: Display a diff link for audio tests, as we now have the diff.
+
+        * platform/mac/TestExpectations: Unmark tests that should now pass everywhere.
+
 2015-03-16  Chris Dumez  <cdumez@apple.com>
 
         Make DatabaseContext suspendable if there is no pending database activity
index 472b23c..9db2963 100644 (file)
@@ -659,6 +659,7 @@ function tableRow(testObject)
     if (actual.indexOf('AUDIO') != -1) {
         row += resultLink(testPrefix, '-expected.wav', 'expected audio');
         row += resultLink(testPrefix, '-actual.wav', 'actual audio');
+        row += resultLink(testPrefix, '-diff.txt', 'textual diff');
     }
 
     if (actual.indexOf('MISSING') != -1) {
index 11fd9b0..6a9f012 100644 (file)
@@ -617,7 +617,6 @@ webaudio/mediastreamaudiodestinationnode.html [ Failure ]
 webaudio/mediastreamaudiosourcenode.html [ Failure ]
 webaudio/codec-tests/vorbis/ [ WontFix ]
 webkit.org/b/119467 webaudio/audiobuffersource-loop-points.html [ Skip ]
-[ Mavericks ] webaudio/codec-tests/wav/24bit-22khz-resample.html [ Failure ]
 
 # Text Autosizing is not enabled.
 webkit.org/b/84186 fast/text-autosizing
@@ -759,14 +758,10 @@ webkit.org/b/123369 svg/css/root-shadow-offscreen.svg [ Pass ImageOnlyFailure ]
 # webkit.org/b/100846, webkit.org/b/136715
 inspector-protocol/debugger
 
-webkit.org/b/123490 [ Mavericks ] webaudio/oscillator-sawtooth.html [ Pass Failure ]
-
 webkit.org/b/124311 compositing/regions/transform-transparent-positioned-video-inside-region.html [ ImageOnlyFailure ]
 
 webkit.org/b/124321 [ Mavericks ] animations/resume-after-page-cache.html [ Pass Failure ]
 
-webkit.org/b/121646 webaudio/delaynode-max-default-delay.html [ Pass Failure ]
-
 # These fast/forms/select tests open a pop-up menu (visible on screen even when using run-webkit-tests), and get stuck in its nested event loop.
 webkit.org/b/87748 fast/forms/select/optgroup-clicking.html [ Skip ]
 webkit.org/b/73304 fast/forms/select/menulist-popup-crash.html [ Skip ]
@@ -1090,10 +1085,6 @@ webkit.org/b/136994 http/tests/media/hls/video-cookie.html [ Failure ]
 webkit.org/b/137157 [ Release ] inspector/page/main-frame-resource.html [ Pass Failure ]
 [ Debug ] inspector/page/main-frame-resource.html [ Pass Failure Slow ]
 
-# FIXME: Needs bugzilla (<rdar://problem/15393179>)
-[ Yosemite+ ] webaudio/codec-tests/wav/24bit-22khz-resample.html [ Failure ]
-[ Yosemite+ ] webaudio/oscillator-sawtooth.html [ Pass Failure ]
-
 webkit.org/b/137505 media/track/track-forced-subtitles-in-band.html [ Failure Pass ]
 
 # FIXME: Needs bugzilla (<rdar://problem/15971968>)
index 025f0ea..758e3f0 100644 (file)
@@ -1,5 +1,26 @@
 2015-03-16  Alexey Proskuryakov  <ap@apple.com>
 
+        Add tolerance to WebAudio tests
+        https://bugs.webkit.org/show_bug.cgi?id=142676
+
+        Reviewed by Tim Horton.
+
+        * Scripts/webkitpy/common/wavediff.py: Added. Based on Jer Noble's work.
+
+        * Scripts/webkitpy/layout_tests/controllers/test_result_writer.py:
+        (TestResultWriter.create_audio_diff_and_write_result):
+        * Scripts/webkitpy/layout_tests/models/test_failures.py:
+        (FailureAudio.write_failure):
+        * Scripts/webkitpy/port/base.py:
+        (Port.do_audio_results_differ):
+        Diff audio failures.
+
+        * Scripts/webkitpy/port/test.py: Added a test for the tolerance, fixed existing
+        tests to use real parseable WAV data, and got rid of base64, which there didn't
+        seem to have been any reason for.
+
+2015-03-16  Alexey Proskuryakov  <ap@apple.com>
+
         [Mac] fast/forms/text-control-intrinsic-widths.html fails when MS Office is installed
         https://bugs.webkit.org/show_bug.cgi?id=142720
 
diff --git a/Tools/Scripts/webkitpy/common/wavediff.py b/Tools/Scripts/webkitpy/common/wavediff.py
new file mode 100644 (file)
index 0000000..19c4e35
--- /dev/null
@@ -0,0 +1,112 @@
+# Copyright (C) 2015 Apple Inc. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1.  Redistributions of source code must retain the above copyright
+#     notice, this list of conditions and the following disclaimer.
+# 2.  Redistributions in binary form must reproduce the above copyright
+#     notice, this list of conditions and the following disclaimer in the
+#     documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import StringIO
+import struct
+import sys
+import tempfile
+import wave
+
+
+class WaveDiff(object):
+    _paramNames = ('Number of channels', 'Sample width', 'Sample rate', 'Number of frames', 'Compression type', 'Compression name')
+
+    # Audio effect processing is intrinsically imprecise, so we need to always allow tolerance.
+    _tolerance = 1
+
+    def __init__(self, in1, in2):
+        if isinstance(in1, file):
+            waveFile1 = wave.open(in1, 'rb')
+        else:
+            waveFile1 = wave.open(StringIO.StringIO(in1), 'rb')
+        if isinstance(in2, file):
+            waveFile1 = wave.open(in2, 'rb')
+        else:
+            waveFile2 = wave.open(StringIO.StringIO(in2), 'rb')
+
+        params1 = waveFile1.getparams()
+        params2 = waveFile2.getparams()
+
+        self._diff = ''
+
+        self._filesAreIdentical = not sum(map(self._diffParam, params1, params2, self._paramNames))
+        self._filesAreIdenticalWithinTolerance = self._filesAreIdentical
+
+        if not self._filesAreIdentical:
+            return
+
+        # Metadata is identical, compare the content now.
+
+        channelCount1 = waveFile1.getnchannels()
+        frameCount1 = waveFile1.getnframes()
+        sampleWidth1 = waveFile1.getsampwidth()
+        channelCount2 = waveFile2.getnchannels()
+        frameCount2 = waveFile2.getnframes()
+        sampleWidth2 = waveFile2.getsampwidth()
+
+        allData1 = self._readSamples(waveFile1, sampleWidth1, frameCount1 * channelCount1)
+        allData2 = self._readSamples(waveFile2, sampleWidth2, frameCount2 * channelCount2)
+        results = map(self._diffSample, allData1, allData2, xrange(max(frameCount1 * channelCount1, frameCount2 * channelCount2)))
+
+        cumulativeSampleDiff = sum(results)
+        differingSampleCount = len(filter(bool, results))
+        self._filesAreIdentical = not differingSampleCount
+        self._filesAreIdenticalWithinTolerance = not len(filter(lambda x: x > self._tolerance, results))
+
+        if differingSampleCount:
+            self._diff += '\n'
+            self._diff += 'Total differing samples: %d\n' % differingSampleCount
+            self._diff += 'Percentage of differing samples: %0.3f%%\n' % (100 * float(differingSampleCount) / max(frameCount1, frameCount2))
+            self._diff += 'Cumulative sample difference: %d\n' % cumulativeSampleDiff
+            self._diff += 'Average sample difference: %f\n' % (float(cumulativeSampleDiff) / differingSampleCount)
+
+    def _diffParam(self, param1, param2, paramName):
+        if param1 == param2:
+            return False
+        self._diff += paramName + '\n'
+        self._diff += '< %s\n' % str(param1)
+        self._diff += '---\n'
+        self._diff += '> %s\n' % str(param2)
+        return True
+
+    @staticmethod
+    def _readSamples(file, sampleWidth, nSamples):
+        allFrames = file.readframes(nSamples)
+        unpackFormat = 'b' if sampleWidth == 1 else 'h'
+        return struct.unpack('<%d%s' % (nSamples, unpackFormat), allFrames)
+
+    def _diffSample(self, data1, data2, i):
+        if (data1 != data2):
+            self._diff += 'Sample #%d\n' % i
+            self._diff += '< %d\n' % data1
+            self._diff += '---\n'
+            self._diff += '> %d\n' % data2
+        return abs(data1 - data2)
+
+    def filesAreIdentical(self):
+        return self._filesAreIdentical
+
+    def filesAreIdenticalWithinTolerance(self):
+        return self._filesAreIdenticalWithinTolerance
+
+    def diffText(self):
+        return self._diff
index 91a50ed..d33ccc6 100644 (file)
@@ -29,6 +29,7 @@
 
 import logging
 
+from webkitpy.common.wavediff import WaveDiff
 from webkitpy.layout_tests.models import test_failures
 
 
@@ -161,6 +162,10 @@ class TestResultWriter(object):
     def write_audio_files(self, actual_audio, expected_audio):
         self.write_output_files('.wav', actual_audio, expected_audio)
 
+    def create_audio_diff_and_write_result(self, actual_audio, expected_audio):
+        diff_filename = self.output_filename(self.FILENAME_SUFFIX_DIFF + '.txt')
+        self._write_text_file(diff_filename, WaveDiff(expected_audio, actual_audio).diffText())
+
     def write_image_files(self, actual_image, expected_image):
         self.write_output_files('.png', actual_image, expected_image)
 
index 5c92de2..20f1b49 100644 (file)
@@ -122,6 +122,7 @@ class FailureText(TestFailure):
 class FailureAudio(TestFailure):
     def write_failure(self, writer, driver_output, expected_driver_output, port):
         writer.write_audio_files(driver_output.audio, expected_driver_output.audio)
+        writer.create_audio_diff_and_write_result(driver_output.audio, expected_driver_output.audio)
 
 
 class FailureTimeout(TestFailure):
index 70cf579..412709a 100644 (file)
@@ -50,6 +50,7 @@ from webkitpy.common.prettypatch import PrettyPatch
 from webkitpy.common.system import path
 from webkitpy.common.system.executive import ScriptError
 from webkitpy.common.system.systemhost import SystemHost
+from webkitpy.common.wavediff import WaveDiff
 from webkitpy.common.webkit_finder import WebKitFinder
 from webkitpy.layout_tests.models.test_configuration import TestConfiguration
 from webkitpy.port import config as port_config
@@ -282,7 +283,9 @@ class Port(object):
         return expected_text != actual_text
 
     def do_audio_results_differ(self, expected_audio, actual_audio):
-        return expected_audio != actual_audio
+        if expected_audio == actual_audio:
+            return False
+        return not WaveDiff(expected_audio, actual_audio).filesAreIdenticalWithinTolerance()
 
     def diff_image(self, expected_contents, actual_contents, tolerance=None):
         """Compare two images and return a tuple of an image diff, a percentage difference (0-100), and an error string.
index ab11637..e721fe6 100644 (file)
@@ -26,7 +26,6 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-import base64
 import sys
 import time
 
@@ -99,7 +98,7 @@ class TestList(object):
 #
 # These numbers may need to be updated whenever we add or delete tests.
 #
-TOTAL_TESTS = 71
+TOTAL_TESTS = 72
 TOTAL_SKIPS = 9
 TOTAL_RETRIES = 14
 
@@ -107,6 +106,9 @@ UNEXPECTED_PASSES = 7
 UNEXPECTED_FAILURES = 17
 
 def unit_test_list():
+    silent_audio = "RIFF2\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x22\x56\x00\x00\x44\xAC\x00\x00\x02\x00\x10\x00data\x0E\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+    silent_audio_with_single_bit_difference = "RIFF2\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x22\x56\x00\x00\x44\xAC\x00\x00\x02\x00\x10\x00data\x0E\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
+    audio2 = "RIFF2\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x22\x56\x00\x00\x44\xAC\x00\x00\x02\x00\x10\x00data\x0E\x00\x00\x00\x40\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
     tests = TestList()
     tests.add('failures/expected/crash.html', crash=True)
     tests.add('failures/expected/exception.html', exception=True)
@@ -120,7 +122,7 @@ def unit_test_list():
               actual_checksum='image_checksum_fail-checksum',
               actual_image='image_checksum_fail-png')
     tests.add('failures/expected/audio.html',
-              actual_audio=base64.b64encode('audio_fail-wav'), expected_audio='audio-wav',
+              actual_audio=silent_audio, expected_audio=audio2,
               actual_text=None, expected_text=None,
               actual_image=None, expected_image=None,
               actual_checksum=None)
@@ -181,7 +183,12 @@ layer at (0,0) size 800x34
     tests.add('passes/error.html', error='stuff going to stderr')
     tests.add('passes/image.html')
     tests.add('passes/audio.html',
-              actual_audio=base64.b64encode('audio-wav'), expected_audio='audio-wav',
+              actual_audio=silent_audio, expected_audio=silent_audio,
+              actual_text=None, expected_text=None,
+              actual_image=None, expected_image=None,
+              actual_checksum=None)
+    tests.add('passes/audio-tolerance.html',
+              actual_audio=silent_audio_with_single_bit_difference, expected_audio=silent_audio,
               actual_text=None, expected_text=None,
               actual_image=None, expected_image=None,
               actual_checksum=None)
@@ -558,7 +565,7 @@ class TestDriver(Driver):
             actual_text = actual_text + ' ' + ' '.join(test_args)
 
         if test.actual_audio:
-            audio = base64.b64decode(test.actual_audio)
+            audio = test.actual_audio
         crashed_process_name = None
         crashed_pid = None
         if test.crash: