[Mac][WebAudio] Update the AVAudioMix in the AudioSourceProviderAVFObjC when the...
authorjer.noble@apple.com <jer.noble@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 8 Apr 2015 18:12:56 +0000 (18:12 +0000)
committerjer.noble@apple.com <jer.noble@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Wed, 8 Apr 2015 18:12:56 +0000 (18:12 +0000)
https://bugs.webkit.org/show_bug.cgi?id=143332

Reviewed by Eric Carlson.

Some media assets (notably, mp3s) will not have an enabled audio track when the AVAsset is
first loaded, so the AVAudioMix will have no trackID in it's parameters. Whenever the list
of enabled tracks change, recreate the AVAudioMix with the new first enabled audio trackID.

To facilitate this, add a new setter to AudioSourceProviderAVFObjC taking an AVAssetTrack to
use with the AVAudioMix. Whenever this parameter changes, the AVAudioMix is destroyed and
recreated.

* html/HTMLMediaElement.cpp:
(WebCore::HTMLMediaElement::mediaPlayerEngineUpdated): Drive-by fix: when the media
    player switches engines, re-associate the audio source node with its provider.
* platform/graphics/avfoundation/AudioSourceProviderAVFObjC.h:
* platform/graphics/avfoundation/AudioSourceProviderAVFObjC.mm:
(WebCore::AudioSourceProviderAVFObjC::setPlayerItem): Only create the mix if there is a valid AVPlayerItem and AVAssetTrack
(WebCore::AudioSourceProviderAVFObjC::setAudioTrack): Ditto.
(WebCore::AudioSourceProviderAVFObjC::createMix): Don't iterate over the AVPlayerItem's tracks,
    just use the one passed in through setAudioTrack().
* platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm:
(WebCore::MediaPlayerPrivateAVFoundationObjC::cancelLoad): Clear the provider's track.
(WebCore::MediaPlayerPrivateAVFoundationObjC::createAVPlayerItem): Set the provider's track.
(WebCore::MediaPlayerPrivateAVFoundationObjC::tracksChanged): Ditto.
(WebCore::MediaPlayerPrivateAVFoundationObjC::audioSourceProvider): Ditto.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@182555 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WebCore/ChangeLog
Source/WebCore/html/HTMLMediaElement.cpp
Source/WebCore/platform/graphics/avfoundation/AudioSourceProviderAVFObjC.h
Source/WebCore/platform/graphics/avfoundation/AudioSourceProviderAVFObjC.mm
Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm

index 958b195213046ad3d4b28503b002715456a26ecf..dbc5c273afc4c489c754c7b07bb92a99bbdfecf2 100644 (file)
@@ -1,3 +1,33 @@
+2015-04-08  Jer Noble  <jer.noble@apple.com>
+
+        [Mac][WebAudio] Update the AVAudioMix in the AudioSourceProviderAVFObjC when the list of enabled audio tracks change.
+        https://bugs.webkit.org/show_bug.cgi?id=143332
+
+        Reviewed by Eric Carlson.
+
+        Some media assets (notably, mp3s) will not have an enabled audio track when the AVAsset is
+        first loaded, so the AVAudioMix will have no trackID in it's parameters. Whenever the list
+        of enabled tracks change, recreate the AVAudioMix with the new first enabled audio trackID.
+
+        To facilitate this, add a new setter to AudioSourceProviderAVFObjC taking an AVAssetTrack to
+        use with the AVAudioMix. Whenever this parameter changes, the AVAudioMix is destroyed and
+        recreated.
+
+        * html/HTMLMediaElement.cpp:
+        (WebCore::HTMLMediaElement::mediaPlayerEngineUpdated): Drive-by fix: when the media
+            player switches engines, re-associate the audio source node with its provider.
+        * platform/graphics/avfoundation/AudioSourceProviderAVFObjC.h:
+        * platform/graphics/avfoundation/AudioSourceProviderAVFObjC.mm:
+        (WebCore::AudioSourceProviderAVFObjC::setPlayerItem): Only create the mix if there is a valid AVPlayerItem and AVAssetTrack
+        (WebCore::AudioSourceProviderAVFObjC::setAudioTrack): Ditto.
+        (WebCore::AudioSourceProviderAVFObjC::createMix): Don't iterate over the AVPlayerItem's tracks,
+            just use the one passed in through setAudioTrack().
+        * platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm:
+        (WebCore::MediaPlayerPrivateAVFoundationObjC::cancelLoad): Clear the provider's track.
+        (WebCore::MediaPlayerPrivateAVFoundationObjC::createAVPlayerItem): Set the provider's track.
+        (WebCore::MediaPlayerPrivateAVFoundationObjC::tracksChanged): Ditto.
+        (WebCore::MediaPlayerPrivateAVFoundationObjC::audioSourceProvider): Ditto.
+
 2015-04-08  Anders Carlsson  <andersca@apple.com>
 
         Move some ApplicationCache static member functions to ApplicationCacheStorage
index 6a92d08168612573a12a97f3835c17241da0d372..15e4b838337824ee2836f0cd9d51ba51627a3f18 100644 (file)
@@ -4273,6 +4273,14 @@ void HTMLMediaElement::mediaPlayerEngineUpdated(MediaPlayer*)
 
     m_mediaSession->applyMediaPlayerRestrictions(*this);
 
+#if ENABLE(WEB_AUDIO)
+    if (m_audioSourceNode && audioSourceProvider()) {
+        m_audioSourceNode->lock();
+        audioSourceProvider()->setClient(m_audioSourceNode);
+        m_audioSourceNode->unlock();
+    }
+#endif
+
 #if PLATFORM(IOS)
     if (!m_player)
         return;
index 91ce163ac9a372b8eb32f4e206d19e65e33afa1d..2d17885f178cee3f42f2f621c55a1fbfda265fe5 100644 (file)
@@ -34,6 +34,7 @@
 #include <wtf/RefPtr.h>
 #include <wtf/RetainPtr.h>
 
+OBJC_CLASS AVAssetTrack;
 OBJC_CLASS AVPlayerItem;
 OBJC_CLASS AVMutableAudioMix;
 
@@ -54,6 +55,7 @@ public:
     virtual ~AudioSourceProviderAVFObjC();
 
     void setPlayerItem(AVPlayerItem *);
+    void setAudioTrack(AVAssetTrack *);
 
 private:
     AudioSourceProviderAVFObjC(AVPlayerItem *);
@@ -78,6 +80,7 @@ private:
     void process(CMItemCount numberFrames, MTAudioProcessingTapFlags flagsIn, AudioBufferList *bufferListInOut, CMItemCount *numberFramesOut, MTAudioProcessingTapFlags *flagsOut);
 
     RetainPtr<AVPlayerItem> m_avPlayerItem;
+    RetainPtr<AVAssetTrack> m_avAssetTrack;
     RetainPtr<AVMutableAudioMix> m_avAudioMix;
     RetainPtr<MTAudioProcessingTapRef> m_tap;
     RetainPtr<AudioConverterRef> m_converter;
index 419425b3e90840628cdc6df45ffcff16b6ad707b..57a1228d4fdf08bdd45a83fac49be7ec39d9f21b 100644 (file)
@@ -38,7 +38,6 @@
 #import <AVFoundation/AVAudioMix.h>
 #import <AVFoundation/AVMediaFormat.h>
 #import <AVFoundation/AVPlayerItem.h>
-#import <AVFoundation/AVPlayerItemTrack.h>
 #import <objc/runtime.h>
 #import <wtf/MainThread.h>
 
@@ -151,7 +150,21 @@ void AudioSourceProviderAVFObjC::setPlayerItem(AVPlayerItem *avPlayerItem)
 
     m_avPlayerItem = avPlayerItem;
 
-    if (m_client && m_avPlayerItem)
+    if (m_client && m_avPlayerItem && m_avAssetTrack)
+        createMix();
+}
+
+void AudioSourceProviderAVFObjC::setAudioTrack(AVAssetTrack *avAssetTrack)
+{
+    if (m_avAssetTrack == avAssetTrack)
+        return;
+
+    if (m_avAudioMix)
+        destroyMix();
+
+    m_avAssetTrack = avAssetTrack;
+
+    if (m_client && m_avPlayerItem && m_avAssetTrack)
         createMix();
 }
 
@@ -190,15 +203,8 @@ void AudioSourceProviderAVFObjC::createMix()
     RetainPtr<AVMutableAudioMixInputParameters> parameters = adoptNS([allocAVMutableAudioMixInputParametersInstance() init]);
     [parameters setAudioTapProcessor:m_tap.get()];
 
-    CMPersistentTrackID firstEnabledAudioTrackID = kCMPersistentTrackID_Invalid;
-    NSArray* tracks = [m_avPlayerItem tracks];
-    for (AVPlayerItemTrack* track in tracks) {
-        if ([track.assetTrack hasMediaCharacteristic:AVMediaCharacteristicAudible] && track.enabled) {
-            firstEnabledAudioTrackID = track.assetTrack.trackID;
-            break;
-        }
-    }
-    [parameters setTrackID:firstEnabledAudioTrackID];
+    CMPersistentTrackID trackID = m_avAssetTrack.get().trackID;
+    [parameters setTrackID:trackID];
     
     [m_avAudioMix setInputParameters:@[parameters.get()]];
     [m_avPlayerItem setAudioMix:m_avAudioMix.get()];
index ccec52c4250de4cad8df585c27cae8d427ae4607..7d0109321d8c413c2dee86b5ae72ddbff7bad2d8 100644 (file)
@@ -349,6 +349,7 @@ static NSArray *assetMetadataKeyNames();
 static NSArray *itemKVOProperties();
 static NSArray *assetTrackMetadataKeyNames();
 static NSArray *playerKVOProperties();
+static AVAssetTrack* firstEnabledTrack(NSArray* tracks);
 
 #if !LOG_DISABLED
 static const char *boolString(bool val)
@@ -566,8 +567,10 @@ void MediaPlayerPrivateAVFoundationObjC::cancelLoad()
     m_cachedTracks = nullptr;
 
 #if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
-    if (m_provider)
+    if (m_provider) {
         m_provider->setPlayerItem(nullptr);
+        m_provider->setAudioTrack(nullptr);
+    }
 #endif
 
     setIgnoreLoadStateChanges(false);
@@ -998,8 +1001,10 @@ void MediaPlayerPrivateAVFoundationObjC::createAVPlayerItem()
 #endif
 
 #if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
-    if (m_provider)
+    if (m_provider) {
         m_provider->setPlayerItem(m_avPlayerItem.get());
+        m_provider->setAudioTrack(firstEnabledTrack([m_avAsset tracksWithMediaCharacteristic:AVMediaCharacteristicAudible]));
+    }
 #endif
 
     setDelayCallbacks(false);
@@ -1917,6 +1922,11 @@ void MediaPlayerPrivateAVFoundationObjC::tracksChanged()
     if (primaryAudioTrackLanguage != languageOfPrimaryAudioTrack())
         characteristicsChanged();
 
+#if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
+    if (m_provider)
+        m_provider->setAudioTrack(firstEnabledTrack([m_avAsset tracksWithMediaCharacteristic:AVMediaCharacteristicAudible]));
+#endif
+
     setDelayCharacteristicsChangedNotification(false);
 }
 
@@ -2127,8 +2137,11 @@ void MediaPlayerPrivateAVFoundationObjC::setTextTrackRepresentation(TextTrackRep
 #if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
 AudioSourceProvider* MediaPlayerPrivateAVFoundationObjC::audioSourceProvider()
 {
-    if (!m_provider)
+    if (!m_provider) {
         m_provider = AudioSourceProviderAVFObjC::create(m_avPlayerItem.get());
+        m_provider->setAudioTrack(firstEnabledTrack([m_avAsset tracksWithMediaCharacteristic:AVMediaCharacteristicAudible]));
+    }
+
     return m_provider.get();
 }
 #endif