+2011-12-22 Chris Rogers <crogers@google.com>
+
+ Implement MediaElementAudioSourceNode::setFormat() so numberOfChannels and sampleRate are accounted for
+ https://bugs.webkit.org/show_bug.cgi?id=75057
+
+ Reviewed by Eric Carlson.
+
+ * GNUmakefile.list.am:
+ * WebCore.gypi:
+ * WebCore.xcodeproj/project.pbxproj:
+ Add MultiChannelResampler source files to makefiles.
+ * platform/audio/MultiChannelResampler.cpp: Added.
+ (WebCore::MultiChannelResampler::MultiChannelResampler):
+ (WebCore::MultiChannelResampler::process):
+ * platform/audio/MultiChannelResampler.h: Added.
+ Add MultiChannelResampler implementation which uses one SincResampler per channel.
+ * webaudio/MediaElementAudioSourceNode.cpp:
+ (WebCore::MediaElementAudioSourceNode::create):
+ (WebCore::MediaElementAudioSourceNode::MediaElementAudioSourceNode):
+ (WebCore::MediaElementAudioSourceNode::setFormat):
+ (WebCore::MediaElementAudioSourceNode::process):
+ Implement MediaElementAudioSourceNode::setFormat() so that we can
+ properly setup a sample-rate converter and set the number of channels
+ of the MediaElementAudioSourceNode output.
+ * webaudio/MediaElementAudioSourceNode.h:
+
2011-12-22 Chris Fleizach <cfleizach@apple.com>
AX: WebKit should ignore ARIA role=presentation on focusable elements
Source/WebCore/platform/audio/HRTFKernel.h \
Source/WebCore/platform/audio/HRTFPanner.cpp \
Source/WebCore/platform/audio/HRTFPanner.h \
+ Source/WebCore/platform/audio/MultiChannelResampler.cpp \
+ Source/WebCore/platform/audio/MultiChannelResampler.h \
Source/WebCore/platform/audio/Panner.cpp \
Source/WebCore/platform/audio/Panner.h \
Source/WebCore/platform/audio/ReverbAccumulationBuffer.cpp \
'platform/audio/HRTFKernel.h',
'platform/audio/HRTFPanner.cpp',
'platform/audio/HRTFPanner.h',
+ 'platform/audio/MultiChannelResampler.cpp',
+ 'platform/audio/MultiChannelResampler.h',
'platform/audio/Panner.cpp',
'platform/audio/Panner.h',
'platform/audio/Reverb.cpp',
FDA3E95A134A49EF008D4B5A /* OfflineAudioCompletionEvent.h in Headers */ = {isa = PBXBuildFile; fileRef = FDA3E956134A49EF008D4B5A /* OfflineAudioCompletionEvent.h */; };
FDA3E95B134A49EF008D4B5A /* OfflineAudioDestinationNode.cpp in Sources */ = {isa = PBXBuildFile; fileRef = FDA3E957134A49EF008D4B5A /* OfflineAudioDestinationNode.cpp */; };
FDA3E95C134A49EF008D4B5A /* OfflineAudioDestinationNode.h in Headers */ = {isa = PBXBuildFile; fileRef = FDA3E958134A49EF008D4B5A /* OfflineAudioDestinationNode.h */; };
+ FDB1700514A2BAB200A2B5D9 /* MultiChannelResampler.cpp in Sources */ = {isa = PBXBuildFile; fileRef = FDB1700314A2BAB200A2B5D9 /* MultiChannelResampler.cpp */; };
+ FDB1700614A2BAB200A2B5D9 /* MultiChannelResampler.h in Headers */ = {isa = PBXBuildFile; fileRef = FDB1700414A2BAB200A2B5D9 /* MultiChannelResampler.h */; };
FDC54F041399B0DA008D9117 /* BiquadFilterNode.cpp in Sources */ = {isa = PBXBuildFile; fileRef = FDC54F011399B0DA008D9117 /* BiquadFilterNode.cpp */; };
FDC54F051399B0DA008D9117 /* BiquadFilterNode.h in Headers */ = {isa = PBXBuildFile; fileRef = FDC54F021399B0DA008D9117 /* BiquadFilterNode.h */; };
FDEAAAF312B02EE400DCF33B /* JSAudioBufferSourceNodeCustom.cpp in Sources */ = {isa = PBXBuildFile; fileRef = FDEAAAEF12B02EE400DCF33B /* JSAudioBufferSourceNodeCustom.cpp */; };
FDA3E957134A49EF008D4B5A /* OfflineAudioDestinationNode.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = OfflineAudioDestinationNode.cpp; sourceTree = "<group>"; };
FDA3E958134A49EF008D4B5A /* OfflineAudioDestinationNode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OfflineAudioDestinationNode.h; sourceTree = "<group>"; };
FDA3E95D134A49FF008D4B5A /* OfflineAudioCompletionEvent.idl */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = OfflineAudioCompletionEvent.idl; sourceTree = "<group>"; };
+ FDB1700314A2BAB200A2B5D9 /* MultiChannelResampler.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = MultiChannelResampler.cpp; sourceTree = "<group>"; };
+ FDB1700414A2BAB200A2B5D9 /* MultiChannelResampler.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MultiChannelResampler.h; sourceTree = "<group>"; };
FDC54F011399B0DA008D9117 /* BiquadFilterNode.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = BiquadFilterNode.cpp; sourceTree = "<group>"; };
FDC54F021399B0DA008D9117 /* BiquadFilterNode.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = BiquadFilterNode.h; sourceTree = "<group>"; };
FDC54F031399B0DA008D9117 /* BiquadFilterNode.idl */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = BiquadFilterNode.idl; sourceTree = "<group>"; };
FD31606912B026F700C1A359 /* HRTFKernel.h */,
FD31606A12B026F700C1A359 /* HRTFPanner.cpp */,
FD31606B12B026F700C1A359 /* HRTFPanner.h */,
+ FDB1700314A2BAB200A2B5D9 /* MultiChannelResampler.cpp */,
+ FDB1700414A2BAB200A2B5D9 /* MultiChannelResampler.h */,
FD31606C12B026F700C1A359 /* Panner.cpp */,
FD31606D12B026F700C1A359 /* Panner.h */,
FD31606E12B026F700C1A359 /* Reverb.cpp */,
1AA7160B149BC4DB0016EC19 /* TileCache.h in Headers */,
C37CDEBD149EF2030042090D /* ColorChooserClient.h in Headers */,
1AC6926A14A1253200BD85F1 /* ScrollableAreaClient.h in Headers */,
+ FDB1700614A2BAB200A2B5D9 /* MultiChannelResampler.h in Headers */,
);
runOnlyForDeploymentPostprocessing = 0;
};
A80A9423149F225E00989291 /* JSDOMWindowWebAudioCustom.cpp in Sources */,
A80A9425149F227100989291 /* JSDOMWindowWebSocketCustom.cpp in Sources */,
52F52E1114A0134F00ACC397 /* NSScrollerImpDetails.mm in Sources */,
+ FDB1700514A2BAB200A2B5D9 /* MultiChannelResampler.cpp in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
--- /dev/null
+/*
+ * Copyright (C) 2011 Google Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of
+ * its contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "config.h"
+
+#if ENABLE(WEB_AUDIO)
+
+#include "MultiChannelResampler.h"
+
+#include "AudioBus.h"
+
+namespace WebCore {
+
+namespace {
+
+// ChannelProvider provides a single channel of audio data (one channel at a time) for each channel
+// of data provided to us in a multi-channel provider.
+
+class ChannelProvider : public AudioSourceProvider {
+public:
+ ChannelProvider(AudioSourceProvider* multiChannelProvider, unsigned numberOfChannels)
+ : m_multiChannelProvider(multiChannelProvider)
+ , m_numberOfChannels(numberOfChannels)
+ , m_currentChannel(0)
+ , m_framesToProcess(0)
+ {
+ }
+
+ // provideInput() will be called once for each channel, starting with the first channel.
+ // Each time it's called, it will provide the next channel of data.
+ virtual void provideInput(AudioBus* bus, size_t framesToProcess)
+ {
+ bool isBusGood = bus && bus->numberOfChannels() == 1;
+ ASSERT(isBusGood);
+ if (!isBusGood)
+ return;
+
+ // Get the data from the multi-channel provider when the first channel asks for it.
+ // For subsequent channels, we can just dish out the channel data from that (stored in m_multiChannelBus).
+ if (!m_currentChannel) {
+ m_framesToProcess = framesToProcess;
+ m_multiChannelBus = adoptPtr(new AudioBus(m_numberOfChannels, framesToProcess));
+ m_multiChannelProvider->provideInput(m_multiChannelBus.get(), framesToProcess);
+ }
+
+ // All channels must ask for the same amount. This should always be the case, but let's just make sure.
+ bool isGood = m_multiChannelBus.get() && framesToProcess == m_framesToProcess;
+ ASSERT(isGood);
+ if (!isGood)
+ return;
+
+ // Copy the channel data from what we received from m_multiChannelProvider.
+ ASSERT(m_currentChannel <= m_numberOfChannels);
+ if (m_currentChannel < m_numberOfChannels) {
+ memcpy(bus->channel(0)->data(), m_multiChannelBus->channel(m_currentChannel)->data(), sizeof(float) * framesToProcess);
+ ++m_currentChannel;
+ }
+ }
+
+private:
+ AudioSourceProvider* m_multiChannelProvider;
+ OwnPtr<AudioBus> m_multiChannelBus;
+ unsigned m_numberOfChannels;
+ unsigned m_currentChannel;
+ size_t m_framesToProcess; // Used to verify that all channels ask for the same amount.
+};
+
+} // namespace
+
+MultiChannelResampler::MultiChannelResampler(double scaleFactor, unsigned numberOfChannels)
+ : m_numberOfChannels(numberOfChannels)
+{
+ // Create each channel's resampler.
+ for (unsigned channelIndex = 0; channelIndex < numberOfChannels; ++channelIndex)
+ m_kernels.append(adoptPtr(new SincResampler(scaleFactor)));
+}
+
+void MultiChannelResampler::process(AudioSourceProvider* provider, AudioBus* destination, size_t framesToProcess)
+{
+ // The provider can provide us with multi-channel audio data. But each of our single-channel resamplers (kernels)
+ // below requires a provider which provides a single unique channel of data.
+ // channelProvider wraps the original multi-channel provider and dishes out one channel at a time.
+ ChannelProvider channelProvider(provider, m_numberOfChannels);
+
+ for (unsigned channelIndex = 0; channelIndex < m_numberOfChannels; ++channelIndex) {
+ // Depending on the sample-rate scale factor, and the internal buffering used in a SincResampler
+ // kernel, this call to process() will only sometimes call provideInput() on the channelProvider.
+ // However, if it calls provideInput() for the first channel, then it will call it for the remaining
+ // channels, since they all buffer in the same way and are processing the same number of frames.
+ m_kernels[channelIndex]->process(&channelProvider,
+ destination->channel(channelIndex)->data(),
+ framesToProcess);
+ }
+}
+
+} // namespace WebCore
+
+#endif // ENABLE(WEB_AUDIO)
--- /dev/null
+/*
+ * Copyright (C) 2011 Google Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of
+ * its contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef MultiChannelResampler_h
+#define MultiChannelResampler_h
+
+#include "SincResampler.h"
+#include <wtf/OwnPtr.h>
+
+namespace WebCore {
+
+class AudioBus;
+
+class MultiChannelResampler {
+public:
+ MultiChannelResampler(double scaleFactor, unsigned numberOfChannels);
+
+ // Process given AudioSourceProvider for streaming applications.
+ void process(AudioSourceProvider*, AudioBus* destination, size_t framesToProcess);
+
+private:
+ // FIXME: the mac port can have a more highly optimized implementation based on CoreAudio
+ // instead of SincResampler. For now the default implementation will be used on all ports.
+ // https://bugs.webkit.org/show_bug.cgi?id=75118
+
+ // Each channel will be resampled using a high-quality SincResampler.
+ Vector<OwnPtr<SincResampler> > m_kernels;
+
+ unsigned m_numberOfChannels;
+};
+
+} // namespace WebCore
+
+#endif // MultiChannelResampler_h
#include "AudioContext.h"
#include "AudioNodeOutput.h"
+#include "Locker.h"
+#include "Logging.h"
#include "MediaPlayer.h"
+// These are somewhat arbitrary limits, but we need to do some kind of sanity-checking.
+const unsigned minSampleRate = 8000;
+const unsigned maxSampleRate = 192000;
+
namespace WebCore {
PassRefPtr<MediaElementAudioSourceNode> MediaElementAudioSourceNode::create(AudioContext* context, HTMLMediaElement* mediaElement)
{
- return adoptRef(new MediaElementAudioSourceNode(context, mediaElement));
+ return adoptRef(new MediaElementAudioSourceNode(context, mediaElement));
}
MediaElementAudioSourceNode::MediaElementAudioSourceNode(AudioContext* context, HTMLMediaElement* mediaElement)
: AudioSourceNode(context, context->sampleRate())
, m_mediaElement(mediaElement)
+ , m_sourceNumberOfChannels(0)
+ , m_sourceSampleRate(0)
{
// Default to stereo. This could change depending on what the media element .src is set to.
addOutput(adoptPtr(new AudioNodeOutput(this, 2)));
-
+
setNodeType(NodeTypeMediaElementAudioSource);
initialize();
uninitialize();
}
-void MediaElementAudioSourceNode::setFormat(size_t, float)
+void MediaElementAudioSourceNode::setFormat(size_t numberOfChannels, float sourceSampleRate)
{
- // FIXME: setup a sample-rate converter if necessary to convert to the AudioContext sample-rate.
- ASSERT_NOT_REACHED();
+ if (numberOfChannels != m_sourceNumberOfChannels || sourceSampleRate != m_sourceSampleRate) {
+ // FIXME: implement multi-channel greater than stereo.
+ // https://bugs.webkit.org/show_bug.cgi?id=75119
+ if (!numberOfChannels || numberOfChannels > 2 || sourceSampleRate < minSampleRate || sourceSampleRate > maxSampleRate) {
+ // process() will generate silence for these uninitialized values.
+ LOG(Media, "MediaElementAudioSourceNode::setFormat(%u, %f) - unhandled format change", static_cast<unsigned>(numberOfChannels), sourceSampleRate);
+ m_sourceNumberOfChannels = 0;
+ m_sourceSampleRate = 0;
+ return;
+ }
+
+ m_sourceNumberOfChannels = numberOfChannels;
+ m_sourceSampleRate = sourceSampleRate;
+
+ // Synchronize with process().
+ Locker<MediaElementAudioSourceNode> locker(*this);
+
+ if (sourceSampleRate != sampleRate()) {
+ double scaleFactor = sourceSampleRate / sampleRate();
+ m_multiChannelResampler = adoptPtr(new MultiChannelResampler(scaleFactor, numberOfChannels));
+ } else {
+ // Bypass resampling.
+ m_multiChannelResampler.clear();
+ }
+
+ {
+ // The context must be locked when changing the number of output channels.
+ AudioContext::AutoLocker contextLocker(context());
+
+ // Do any necesssary re-configuration to the output's number of channels.
+ output(0)->setNumberOfChannels(numberOfChannels);
+ }
+ }
}
void MediaElementAudioSourceNode::process(size_t numberOfFrames)
{
AudioBus* outputBus = output(0)->bus();
- if (!mediaElement()) {
+ if (!mediaElement() || !m_sourceNumberOfChannels || !m_sourceSampleRate) {
outputBus->zero();
return;
}
-
+
// Use a tryLock() to avoid contention in the real-time audio thread.
// If we fail to acquire the lock then the HTMLMediaElement must be in the middle of
// reconfiguring its playback engine, so we output silence in this case.
if (m_processLock.tryLock()) {
- if (AudioSourceProvider* provider = mediaElement()->audioSourceProvider())
- provider->provideInput(outputBus, numberOfFrames);
- else
+ if (AudioSourceProvider* provider = mediaElement()->audioSourceProvider()) {
+ if (m_multiChannelResampler.get()) {
+ ASSERT(m_sourceSampleRate != sampleRate());
+ m_multiChannelResampler->process(provider, outputBus, numberOfFrames);
+ } else {
+ // Bypass the resampler completely if the source is at the context's sample-rate.
+ ASSERT(m_sourceSampleRate == sampleRate());
+ provider->provideInput(outputBus, numberOfFrames);
+ }
+ } else {
+ // Either this port doesn't yet support HTMLMediaElement audio stream access,
+ // or the stream is not yet available.
outputBus->zero();
+ }
m_processLock.unlock();
- } else
+ } else {
+ // We failed to acquire the lock.
outputBus->zero();
+ }
}
void MediaElementAudioSourceNode::reset()
#include "AudioSourceNode.h"
#include "AudioSourceProviderClient.h"
#include "HTMLMediaElement.h"
+#include "MultiChannelResampler.h"
+#include <wtf/OwnPtr.h>
#include <wtf/PassRefPtr.h>
#include <wtf/Threading.h>
RefPtr<HTMLMediaElement> m_mediaElement;
Mutex m_processLock;
+
+ unsigned m_sourceNumberOfChannels;
+ double m_sourceSampleRate;
+
+ OwnPtr<MultiChannelResampler> m_multiChannelResampler;
};
} // namespace WebCore
git://git.webkit.org / WebKit-https.git / commitdiff