[MediaStream] Set correct audio session category when capturing audio
authoreric.carlson@apple.com <eric.carlson@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 11 Apr 2017 18:27:01 +0000 (18:27 +0000)
committereric.carlson@apple.com <eric.carlson@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Tue, 11 Apr 2017 18:27:01 +0000 (18:27 +0000)
https://bugs.webkit.org/show_bug.cgi?id=170736
<rdar://problem/31559405>

Reviewed by Jer Noble.

Source/WebCore:

No new tests yet, filed bug 170737.

* Modules/mediastream/MediaStream.cpp:
(WebCore::MediaStream::MediaStream): Initialize m_mediaSession.
(WebCore::MediaStream::statusDidChange): Call canProduceAudioChanged.
(WebCore::MediaStream::mediaType): New. Return MediaStreamCapturingAudio when actively
capturing audio.
(WebCore::MediaStream::presentationType):
(WebCore::MediaStream::characteristics):
(WebCore::MediaStream::mayResumePlayback):
(WebCore::MediaStream::suspendPlayback):
(WebCore::MediaStream::sourceApplicationIdentifier):
(WebCore::MediaStream::canProduceAudio):
* Modules/mediastream/MediaStream.h:

* platform/audio/PlatformMediaSession.h: Add MediaStreamCapturingAudio.
* platform/audio/PlatformMediaSessionManager.cpp:
(WebCore::PlatformMediaSessionManager::has): Adjust assert for MediaStreamCapturingAudio.
(WebCore::PlatformMediaSessionManager::count): Ditto.
* platform/audio/cocoa/MediaSessionManagerCocoa.cpp:
(PlatformMediaSessionManager::updateSessionState): Set small preferred buffer size when
capturing audio. Set audio session category to PlayAndRecord whenever there is a media
stream that is capturing audio.

Source/WebKit2:

* UIProcess/WebPageProxy.cpp:
(WebKit::WebPageProxy::isPlayingMediaDidChange):

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@215242 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/WebCore/ChangeLog
Source/WebCore/Modules/mediastream/MediaStream.cpp
Source/WebCore/Modules/mediastream/MediaStream.h
Source/WebCore/platform/audio/PlatformMediaSession.h
Source/WebCore/platform/audio/PlatformMediaSessionManager.cpp
Source/WebCore/platform/audio/cocoa/MediaSessionManagerCocoa.cpp
Source/WebCore/testing/Internals.cpp
Source/WebKit2/ChangeLog
Source/WebKit2/UIProcess/WebPageProxy.cpp

index bb15232ee905ebfd1562277e49b4331be6bd5f69..a7c7c9e172f2f0acaa8e683961b2a26857fdf92a 100644 (file)
@@ -1,3 +1,35 @@
+2017-04-11  Eric Carlson  <eric.carlson@apple.com>
+
+        [MediaStream] Set correct audio session category when capturing audio
+        https://bugs.webkit.org/show_bug.cgi?id=170736
+        <rdar://problem/31559405>
+
+        Reviewed by Jer Noble.
+
+        No new tests yet, filed bug 170737.
+
+        * Modules/mediastream/MediaStream.cpp:
+        (WebCore::MediaStream::MediaStream): Initialize m_mediaSession.
+        (WebCore::MediaStream::statusDidChange): Call canProduceAudioChanged.
+        (WebCore::MediaStream::mediaType): New. Return MediaStreamCapturingAudio when actively
+        capturing audio.
+        (WebCore::MediaStream::presentationType):
+        (WebCore::MediaStream::characteristics):
+        (WebCore::MediaStream::mayResumePlayback):
+        (WebCore::MediaStream::suspendPlayback):
+        (WebCore::MediaStream::sourceApplicationIdentifier):
+        (WebCore::MediaStream::canProduceAudio):
+        * Modules/mediastream/MediaStream.h:
+
+        * platform/audio/PlatformMediaSession.h: Add MediaStreamCapturingAudio.
+        * platform/audio/PlatformMediaSessionManager.cpp:
+        (WebCore::PlatformMediaSessionManager::has): Adjust assert for MediaStreamCapturingAudio.
+        (WebCore::PlatformMediaSessionManager::count): Ditto.
+        * platform/audio/cocoa/MediaSessionManagerCocoa.cpp:
+        (PlatformMediaSessionManager::updateSessionState): Set small preferred buffer size when
+        capturing audio. Set audio session category to PlayAndRecord whenever there is a media
+        stream that is capturing audio.
+
 2017-04-11  Yusuke Suzuki  <utatane.tea@gmail.com>
 
         [WebCore][JSC] ResourceUsageData.{timeOfNextEdenCollection,timeOfNextFullCollection} should be MonotonicTime
index 276875d273518509a78aa3305cba15e3014a8347..b4811ea5a4152a19d15fd6509ee1d768831ddd9d 100644 (file)
 #include "Document.h"
 #include "Event.h"
 #include "EventNames.h"
+#include "Frame.h"
 #include "Logging.h"
 #include "MediaStreamRegistry.h"
 #include "MediaStreamTrackEvent.h"
+#include "NetworkingContext.h"
 #include "Page.h"
 #include "RealtimeMediaSource.h"
 #include "URL.h"
@@ -75,6 +77,7 @@ MediaStream::MediaStream(ScriptExecutionContext& context, const MediaStreamTrack
     : ContextDestructionObserver(&context)
     , m_private(MediaStreamPrivate::create(createTrackPrivateVector(tracks)))
     , m_activityEventTimer(*this, &MediaStream::activityEventTimerFired)
+    , m_mediaSession(PlatformMediaSession::create(*this))
 {
     // This constructor preserves MediaStreamTrack instances and must be used by calls originating
     // from the JavaScript MediaStream constructor.
@@ -94,6 +97,7 @@ MediaStream::MediaStream(ScriptExecutionContext& context, Ref<MediaStreamPrivate
     : ContextDestructionObserver(&context)
     , m_private(WTFMove(streamPrivate))
     , m_activityEventTimer(*this, &MediaStream::activityEventTimerFired)
+    , m_mediaSession(PlatformMediaSession::create(*this))
 {
     setIsActive(m_private->active());
     if (document()->page() && document()->page()->isMediaCaptureMuted())
@@ -342,6 +346,8 @@ MediaProducer::MediaStateFlags MediaStream::mediaState() const
 
 void MediaStream::statusDidChange()
 {
+    m_mediaSession->canProduceAudioChanged();
+
     if (Document* document = this->document()) {
         if (m_isActive)
             document->setHasActiveMediaStreamTrack();
@@ -423,6 +429,63 @@ Document* MediaStream::document() const
     return downcast<Document>(scriptExecutionContext());
 }
 
+PlatformMediaSession::MediaType MediaStream::mediaType() const
+{
+    // We only need to override the type when capturing audio, HTMLMediaElement and/or WebAudio
+    // will do the right thing when a stream is attached to a media element or an audio context.
+    if (m_private->hasAudio() && m_private->isProducingData() && m_private->hasCaptureAudioSource())
+        return PlatformMediaSession::MediaStreamCapturingAudio;
+
+    return PlatformMediaSession::None;
+}
+
+PlatformMediaSession::MediaType MediaStream::presentationType() const
+{
+    return mediaType();
+}
+
+PlatformMediaSession::CharacteristicsFlags MediaStream::characteristics() const
+{
+    PlatformMediaSession::CharacteristicsFlags state = PlatformMediaSession::HasNothing;
+
+    if (!m_private->isProducingData())
+        return state;
+
+    if (m_private->hasAudio())
+        state |= PlatformMediaSession::HasAudio;
+
+    if (m_private->hasVideo())
+        state |= PlatformMediaSession::HasVideo;
+
+    return state;
+}
+
+void MediaStream::mayResumePlayback(bool)
+{
+    // FIXME: should a media stream pay attention to this directly, or only when attached to a media element?
+}
+
+void MediaStream::suspendPlayback()
+{
+    // FIXME: should a media stream pay attention to this directly, or only when attached to a media element?
+}
+
+String MediaStream::sourceApplicationIdentifier() const
+{
+    Document* document = this->document();
+    if (document && document->frame()) {
+        if (NetworkingContext* networkingContext = document->frame()->loader().networkingContext())
+            return networkingContext->sourceApplicationIdentifier();
+    }
+
+    return emptyString();
+}
+
+bool MediaStream::canProduceAudio() const
+{
+    return !muted() && active() && m_private->hasAudio() && m_private->isProducingData();
+}
+
 } // namespace WebCore
 
 #endif // ENABLE(MEDIA_STREAM)
index 940048035efab9f9e4bfdc3c614a0717e5141938..525771d3da1fc3103b7d6603485d67ecb3894799 100644 (file)
@@ -1,7 +1,7 @@
 /*
  * Copyright (C) 2011 Google Inc. All rights reserved.
  * Copyright (C) 2011, 2015 Ericsson AB. All rights reserved.
- * Copyright (C) 2013-2015 Apple Inc. All rights reserved.
+ * Copyright (C) 2013-2017 Apple Inc. All rights reserved.
  * Copyright (C) 2013 Nokia Corporation and/or its subsidiary(-ies).
  *
  * Redistribution and use in source and binary forms, with or without
@@ -36,6 +36,7 @@
 #include "MediaProducer.h"
 #include "MediaStreamPrivate.h"
 #include "MediaStreamTrack.h"
+#include "PlatformMediaSession.h"
 #include "ScriptWrappable.h"
 #include "Timer.h"
 #include "URLRegistry.h"
@@ -55,6 +56,7 @@ class MediaStream final
     , public MediaStreamPrivate::Observer
     , private MediaProducer
     , private MediaCanStartListener
+    , private PlatformMediaSessionClient
     , public RefCounted<MediaStream> {
 public:
     class Observer {
@@ -134,6 +136,20 @@ private:
     // MediaCanStartListener
     void mediaCanStart(Document&) final;
 
+    // PlatformMediaSessionClient
+    PlatformMediaSession::MediaType mediaType() const final;
+    PlatformMediaSession::MediaType presentationType() const final;
+    PlatformMediaSession::CharacteristicsFlags characteristics() const final;
+    void mayResumePlayback(bool shouldResume) final;
+    void suspendPlayback() final;
+    bool canReceiveRemoteControlCommands() const final { return false; }
+    void didReceiveRemoteControlCommand(PlatformMediaSession::RemoteControlCommandType, const PlatformMediaSession::RemoteCommandArgument*) final { }
+    bool supportsSeeking() const final { return false; }
+    bool shouldOverrideBackgroundPlaybackRestriction(PlatformMediaSession::InterruptionType) const final { return false; }
+    String sourceApplicationIdentifier() const final;
+    bool canProduceAudio() const final;
+    const Document* hostingDocument() const final { return document(); }
+
     bool internalAddTrack(Ref<MediaStreamTrack>&&, StreamModifier);
     bool internalRemoveTrack(const String&, StreamModifier);
 
@@ -154,6 +170,7 @@ private:
     Vector<Ref<Event>> m_scheduledActivityEvents;
 
     Vector<Observer*> m_observers;
+    std::unique_ptr<PlatformMediaSession> m_mediaSession;
 
     bool m_isActive { false };
     bool m_isMuted { true };
index fd5294c36acb55826a29813d2deffcbf7921cd2b..5b9bd83f329ac26af5594b05e45724669d4857e4 100644 (file)
@@ -58,6 +58,7 @@ public:
         VideoAudio,
         Audio,
         WebAudio,
+        MediaStreamCapturingAudio,
     };
     MediaType mediaType() const;
     MediaType presentationType() const;
index b58902c6608818416149b7d043c3db729db8f72c..1317894851e422a236565e48932fb43a02e1344a 100644 (file)
@@ -73,7 +73,7 @@ void PlatformMediaSessionManager::resetRestrictions()
 
 bool PlatformMediaSessionManager::has(PlatformMediaSession::MediaType type) const
 {
-    ASSERT(type >= PlatformMediaSession::None && type <= PlatformMediaSession::WebAudio);
+    ASSERT(type >= PlatformMediaSession::None && type <= PlatformMediaSession::MediaStreamCapturingAudio);
 
     return anyOfSessions([type] (PlatformMediaSession& session, size_t) {
         return session.mediaType() == type;
@@ -96,7 +96,7 @@ bool PlatformMediaSessionManager::canProduceAudio() const
 
 int PlatformMediaSessionManager::count(PlatformMediaSession::MediaType type) const
 {
-    ASSERT(type >= PlatformMediaSession::None && type <= PlatformMediaSession::WebAudio);
+    ASSERT(type >= PlatformMediaSession::None && type <= PlatformMediaSession::MediaStreamCapturingAudio);
 
     int count = 0;
     for (auto* session : m_sessions) {
index b3aa46b9e99fd1871703cd09f0f99d4a07a5b6b0..65ae9b182b3ca7cbde58109018380824cb6b4f44 100644 (file)
@@ -42,7 +42,7 @@ void PlatformMediaSessionManager::updateSessionState()
 {
     LOG(Media, "PlatformMediaSessionManager::updateSessionState() - types: Video(%d), Audio(%d), WebAudio(%d)", count(PlatformMediaSession::Video), count(PlatformMediaSession::Audio), count(PlatformMediaSession::WebAudio));
 
-    if (has(PlatformMediaSession::WebAudio))
+    if (has(PlatformMediaSession::WebAudio) || has(PlatformMediaSession::MediaStreamCapturingAudio))
         AudioSession::sharedSession().setPreferredBufferSize(kWebAudioBufferSize);
     else if ((has(PlatformMediaSession::Video) || has(PlatformMediaSession::Audio)) && Settings::lowPowerVideoAudioBufferSizeEnabled()) {
         // FIXME: <http://webkit.org/b/116725> Figure out why enabling the code below
@@ -62,14 +62,19 @@ void PlatformMediaSessionManager::updateSessionState()
         return;
 
     bool hasAudioMediaType = false;
-    bool hasAudibleAudioOrVideoMediaType = anyOfSessions([this, hasAudioMediaType] (PlatformMediaSession& session, size_t) mutable {
+    bool hasAudibleAudioOrVideoMediaType = false;
+    bool hasAudioCapture = anyOfSessions([this, hasAudioMediaType, hasAudibleAudioOrVideoMediaType] (PlatformMediaSession& session, size_t) mutable {
         auto type = session.mediaType();
         if (type == PlatformMediaSession::VideoAudio || type == PlatformMediaSession::Audio || type == PlatformMediaSession::WebAudio)
             hasAudioMediaType = true;
-        return (type == PlatformMediaSession::VideoAudio || type == PlatformMediaSession::Audio) && session.canProduceAudio();
+        if ((type == PlatformMediaSession::VideoAudio || type == PlatformMediaSession::Audio) && session.canProduceAudio())
+            hasAudibleAudioOrVideoMediaType = true;
+        return (type == PlatformMediaSession::MediaStreamCapturingAudio);
     });
 
-    if (hasAudibleAudioOrVideoMediaType)
+    if (hasAudioCapture)
+        AudioSession::sharedSession().setCategory(AudioSession::PlayAndRecord);
+    else if (hasAudibleAudioOrVideoMediaType)
         AudioSession::sharedSession().setCategory(AudioSession::MediaPlayback);
     else if (hasAudioMediaType)
         AudioSession::sharedSession().setCategory(AudioSession::AmbientSound);
index 301a87a12008a6df2e7db24f72b8fa170f5a3281..e5010793486edf1ad205c8863b3450faf8709ed2 100644 (file)
@@ -3201,6 +3201,8 @@ static PlatformMediaSession::MediaType mediaTypeFromString(const String& mediaTy
         return PlatformMediaSession::VideoAudio;
     if (equalLettersIgnoringASCIICase(mediaTypeString, "webaudio"))
         return PlatformMediaSession::WebAudio;
+    if (equalLettersIgnoringASCIICase(mediaTypeString, "mediastreamcapturingaudio"))
+        return PlatformMediaSession::MediaStreamCapturingAudio;
 
     return PlatformMediaSession::None;
 }
index ff38ba3c28a705d367f81b44333468d52f061372..f21f76902e408e5656d3270389c0c17be5c73a81 100644 (file)
@@ -1,3 +1,14 @@
+2017-04-11  Eric Carlson  <eric.carlson@apple.com>
+
+        [MediaStream] Set correct audio session category when capturing audio
+        https://bugs.webkit.org/show_bug.cgi?id=170736
+        <rdar://problem/31559405>
+
+        Reviewed by Jer Noble.
+
+        * UIProcess/WebPageProxy.cpp:
+        (WebKit::WebPageProxy::isPlayingMediaDidChange):
+
 2017-04-11  Youenn Fablet  <youenn@apple.com>
 
         Activate WebRTC data channel tests for WK1
index a5fc2c9a067348503e5b74b2eebb0c2dc0494f73..d77d407232233a5ce989f3f9b47daaa2cbe400da 100644 (file)
@@ -6446,7 +6446,7 @@ void WebPageProxy::isPlayingMediaDidChange(MediaProducer::MediaStateFlags state,
     if (state == m_mediaState)
         return;
 
-    WebCore::MediaProducer::MediaStateFlags activeCaptureMask = WebCore::MediaProducer::HasActiveAudioCaptureDevice | WebCore::MediaProducer::HasActiveVideoCaptureDevice;
+    WebCore::MediaProducer::MediaStateFlags activeCaptureMask = WebCore::MediaProducer::HasActiveAudioCaptureDevice | WebCore::MediaProducer::HasActiveVideoCaptureDevice | WebCore::MediaProducer::HasMutedAudioCaptureDevice | WebCore::MediaProducer::HasMutedVideoCaptureDevice;
 #if ENABLE(MEDIA_STREAM)
     WebCore::MediaProducer::MediaStateFlags oldMediaStateHasActiveCapture = m_mediaState & activeCaptureMask;
     WebCore::MediaProducer::MediaStateFlags newMediaStateHasActiveCapture = state & activeCaptureMask;