[MediaStream] push media stream state to the UI process
authoreric.carlson@apple.com <eric.carlson@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 10 Mar 2016 12:05:59 +0000 (12:05 +0000)
committereric.carlson@apple.com <eric.carlson@apple.com@268f45cc-cd09-0410-ab3c-d52691b4dbfc>
Thu, 10 Mar 2016 12:05:59 +0000 (12:05 +0000)
https://bugs.webkit.org/show_bug.cgi?id=155281

Reviewed by Darin Adler.

Source/WebCore:

* Modules/mediastream/MediaStream.cpp:
(WebCore::MediaStream::MediaStream): Register with the document as an audio producer.
(WebCore::MediaStream::~MediaStream): Unregister.
(WebCore::MediaStream::setIsActive): Update document status.
(WebCore::MediaStream::pageMutedStateDidChange): Mute/unmute according to the page mute setting.
(WebCore::MediaStream::mediaState): Return state.
(WebCore::MediaStream::statusDidChange): Call document.updateIsPlayingMedia.
(WebCore::MediaStream::characteristicsChanged): Track stream mute state.
(WebCore::MediaStream::scheduleActiveStateChange): m_isActive -> m_active.
* Modules/mediastream/MediaStream.h:

* page/MediaProducer.h: Add HasActiveMediaCaptureDevice.

* platform/mediastream/MediaStreamPrivate.cpp:
(WebCore::MediaStreamPrivate::hasVideo): Make const.
(WebCore::MediaStreamPrivate::hasAudio): Ditto.
(WebCore::MediaStreamPrivate::muted): New.
* platform/mediastream/MediaStreamPrivate.h:

* platform/mediastream/mac/AVMediaCaptureSource.mm:
(WebCore::AVMediaCaptureSource::AVMediaCaptureSource): Initialize muted to true.
(WebCore::AVMediaCaptureSource::captureSessionIsRunningDidChange): Set muted.

Source/WebKit2:

* UIProcess/API/C/WKPage.cpp:
(WKPageGetMediaState): New.
* UIProcess/API/C/WKPagePrivate.h:

* UIProcess/WebPageProxy.cpp:
(WebKit::WebPageProxy::didCommitLoadForFrame): Clear m_mediaState.
(WebKit::WebPageProxy::isPlayingMediaDidChange): Call m_pageClient.isPlayingMediaDidChange when
  audio or video state changes, call m_uiClient->isPlayingAudioDidChange when audio, vidoe,
  or media stream state changes.
* UIProcess/WebPageProxy.h:
(WebKit::WebPageProxy::mediaStateFlags): New.

git-svn-id: https://svn.webkit.org/repository/webkit/trunk@197929 268f45cc-cd09-0410-ab3c-d52691b4dbfc

13 files changed:
Source/WebCore/ChangeLog
Source/WebCore/Modules/mediastream/MediaStream.cpp
Source/WebCore/Modules/mediastream/MediaStream.h
Source/WebCore/page/MediaProducer.h
Source/WebCore/platform/mediastream/MediaStreamPrivate.cpp
Source/WebCore/platform/mediastream/MediaStreamPrivate.h
Source/WebCore/platform/mediastream/mac/AVMediaCaptureSource.mm
Source/WebKit2/ChangeLog
Source/WebKit2/UIProcess/API/C/WKPage.cpp
Source/WebKit2/UIProcess/API/C/WKPagePrivate.h
Source/WebKit2/UIProcess/WebPageProxy.cpp
Source/WebKit2/UIProcess/WebPageProxy.h
Source/WebKit2/WebProcess/com.apple.WebProcess.sb.in

index 7780f73..5639a4e 100644 (file)
@@ -1,3 +1,33 @@
+2016-03-10  Eric Carlson  <eric.carlson@apple.com>
+
+        [MediaStream] push media stream state to the UI process
+        https://bugs.webkit.org/show_bug.cgi?id=155281
+
+        Reviewed by Darin Adler.
+
+        * Modules/mediastream/MediaStream.cpp:
+        (WebCore::MediaStream::MediaStream): Register with the document as an audio producer.
+        (WebCore::MediaStream::~MediaStream): Unregister.
+        (WebCore::MediaStream::setIsActive): Update document status.
+        (WebCore::MediaStream::pageMutedStateDidChange): Mute/unmute according to the page mute setting.
+        (WebCore::MediaStream::mediaState): Return state.
+        (WebCore::MediaStream::statusDidChange): Call document.updateIsPlayingMedia.
+        (WebCore::MediaStream::characteristicsChanged): Track stream mute state.
+        (WebCore::MediaStream::scheduleActiveStateChange): m_isActive -> m_active.
+        * Modules/mediastream/MediaStream.h:
+
+        * page/MediaProducer.h: Add HasActiveMediaCaptureDevice.
+
+        * platform/mediastream/MediaStreamPrivate.cpp:
+        (WebCore::MediaStreamPrivate::hasVideo): Make const.
+        (WebCore::MediaStreamPrivate::hasAudio): Ditto.
+        (WebCore::MediaStreamPrivate::muted): New.
+        * platform/mediastream/MediaStreamPrivate.h:
+
+        * platform/mediastream/mac/AVMediaCaptureSource.mm:
+        (WebCore::AVMediaCaptureSource::AVMediaCaptureSource): Initialize muted to true.
+        (WebCore::AVMediaCaptureSource::captureSessionIsRunningDidChange): Set muted.
+
 2016-03-09  Ryosuke Niwa  <rniwa@webkit.org>
 
         Extract EventPath.h/cpp out of EventDispatcher.cpp
index 16ba389..e148cab 100644 (file)
@@ -35,6 +35,7 @@
 #include "ExceptionCode.h"
 #include "MediaStreamRegistry.h"
 #include "MediaStreamTrackEvent.h"
+#include "Page.h"
 #include "RealtimeMediaSource.h"
 #include "URL.h"
 #include <wtf/NeverDestroyed.h>
@@ -82,6 +83,7 @@ MediaStream::MediaStream(ScriptExecutionContext& context, const MediaStreamTrack
     setIsActive(m_private->active());
     m_private->addObserver(*this);
     MediaStreamRegistry::shared().registerStream(*this);
+    document()->addAudioProducer(this);
 }
 
 MediaStream::MediaStream(ScriptExecutionContext& context, RefPtr<MediaStreamPrivate>&& streamPrivate)
@@ -99,14 +101,20 @@ MediaStream::MediaStream(ScriptExecutionContext& context, RefPtr<MediaStreamPriv
         track->addObserver(this);
         m_trackSet.add(track->id(), WTFMove(track));
     }
+    document()->addAudioProducer(this);
 }
 
 MediaStream::~MediaStream()
 {
+    // Set isActive to false immediately so an callbacks triggered by shutting down, e.g.
+    // mediaState(), are short circuited.
+    m_isActive = false;
     MediaStreamRegistry::shared().unregisterStream(*this);
     m_private->removeObserver(*this);
     for (auto& track : m_trackSet.values())
         track->removeObserver(this);
+    if (Document* document = this->document())
+        document->removeAudioProducer(this);
 }
 
 RefPtr<MediaStream> MediaStream::clone()
@@ -233,12 +241,65 @@ bool MediaStream::internalRemoveTrack(RefPtr<MediaStreamTrack>&& track, StreamMo
 
 void MediaStream::setIsActive(bool active)
 {
+    if (m_isActive == active)
+        return;
+
     m_isActive = active;
-    if (!active)
+    statusDidChange();
+}
+
+void MediaStream::pageMutedStateDidChange()
+{
+    if (!m_isActive)
+        return;
+
+    Document* document = this->document();
+    if (!document)
         return;
 
-    if (Document* document = downcast<Document>(scriptExecutionContext()))
-        document->setHasActiveMediaStreamTrack();
+    bool pageMuted = document->page()->isMuted();
+    if (m_externallyMuted == pageMuted)
+        return;
+
+    m_externallyMuted = pageMuted;
+    if (pageMuted)
+        m_private->stopProducingData();
+    else
+        m_private->startProducingData();
+}
+
+MediaProducer::MediaStateFlags MediaStream::mediaState() const
+{
+    MediaStateFlags state = IsNotPlaying;
+
+    if (!m_isActive)
+        return state;
+
+    if (m_externallyMuted || m_private->isProducingData())
+        state |= HasActiveMediaCaptureDevice;
+
+    if (m_private->hasAudio() || m_private->hasVideo())
+        state |= HasAudioOrVideo;
+
+    return state;
+}
+
+void MediaStream::statusDidChange()
+{
+    if (Document* document = this->document()) {
+        if (m_isActive)
+            document->setHasActiveMediaStreamTrack();
+        document->updateIsPlayingMedia();
+    }
+}
+
+void MediaStream::characteristicsChanged()
+{
+    bool muted = m_private->muted();
+    if (m_isMuted != muted) {
+        m_isMuted = muted;
+        statusDidChange();
+    }
 }
 
 void MediaStream::scheduleActiveStateChange()
@@ -301,6 +362,11 @@ void MediaStream::removeObserver(MediaStream::Observer* observer)
         m_observers.remove(pos);
 }
 
+Document* MediaStream::document() const
+{
+    return downcast<Document>(scriptExecutionContext());
+}
+
 } // namespace WebCore
 
 #endif // ENABLE(MEDIA_STREAM)
index 901704d..7e9dbf3 100644 (file)
@@ -33,6 +33,7 @@
 #include "ContextDestructionObserver.h"
 #include "EventTarget.h"
 #include "ExceptionBase.h"
+#include "MediaProducer.h"
 #include "MediaStreamPrivate.h"
 #include "MediaStreamTrack.h"
 #include "ScriptWrappable.h"
@@ -44,7 +45,9 @@
 
 namespace WebCore {
 
-class MediaStream final : public URLRegistrable, public EventTargetWithInlineData, public ContextDestructionObserver, public MediaStreamTrack::Observer, public MediaStreamPrivate::Observer, public RefCounted<MediaStream> {
+class Document;
+
+class MediaStream final : public URLRegistrable, public EventTargetWithInlineData, public ContextDestructionObserver, public MediaStreamTrack::Observer, public MediaStreamPrivate::Observer, private MediaProducer, public RefCounted<MediaStream> {
 public:
     class Observer {
     public:
@@ -71,6 +74,7 @@ public:
     RefPtr<MediaStream> clone();
 
     bool active() const { return m_isActive; }
+    bool muted() const { return m_isMuted; }
 
     MediaStreamPrivate* privateStream() const { return m_private.get(); }
 
@@ -108,6 +112,11 @@ private:
     void activeStatusChanged() final;
     void didAddTrack(MediaStreamTrackPrivate&) final;
     void didRemoveTrack(MediaStreamTrackPrivate&) final;
+    void characteristicsChanged() final;
+
+    // MediaProducer
+    void pageMutedStateDidChange() final;
+    MediaProducer::MediaStateFlags mediaState() const final;
 
     bool internalAddTrack(RefPtr<MediaStreamTrack>&&, StreamModifier);
     bool internalRemoveTrack(RefPtr<MediaStreamTrack>&&, StreamModifier);
@@ -115,18 +124,24 @@ private:
     void scheduleActiveStateChange();
     void activityEventTimerFired();
     void setIsActive(bool);
+    void statusDidChange();
+
+    Document* document() const;
 
     MediaStreamTrackVector trackVectorForType(RealtimeMediaSource::Type) const;
 
     RefPtr<MediaStreamPrivate> m_private;
 
-    bool m_isActive;
     HashMap<String, RefPtr<MediaStreamTrack>> m_trackSet;
 
     Timer m_activityEventTimer;
     Vector<Ref<Event>> m_scheduledActivityEvents;
 
     Vector<Observer*> m_observers;
+
+    bool m_isActive { false };
+    bool m_isMuted { true };
+    bool m_externallyMuted { false };
 };
 
 } // namespace WebCore
index 1fbb791..9a66c88 100644 (file)
@@ -43,6 +43,7 @@ public:
         IsPreviousTrackControlEnabled = 1 << 8,
         HasPlaybackTargetAvailabilityListener = 1 << 9,
         HasAudioOrVideo = 1 << 10,
+        HasActiveMediaCaptureDevice = 1 << 11,
     };
     typedef unsigned MediaStateFlags;
 
index 994ac4b..df66070 100644 (file)
@@ -181,7 +181,7 @@ bool MediaStreamPrivate::isProducingData() const
     return false;
 }
 
-bool MediaStreamPrivate::hasVideo()
+bool MediaStreamPrivate::hasVideo() const
 {
     for (auto& track : m_trackSet.values()) {
         if (track->type() == RealtimeMediaSource::Type::Video && track->enabled() && !track->ended())
@@ -190,7 +190,7 @@ bool MediaStreamPrivate::hasVideo()
     return false;
 }
 
-bool MediaStreamPrivate::hasAudio()
+bool MediaStreamPrivate::hasAudio() const
 {
     for (auto& track : m_trackSet.values()) {
         if (track->type() == RealtimeMediaSource::Type::Audio && track->enabled() && !track->ended())
@@ -199,6 +199,15 @@ bool MediaStreamPrivate::hasAudio()
     return false;
 }
 
+bool MediaStreamPrivate::muted() const
+{
+    for (auto& track : m_trackSet.values()) {
+        if (!track->muted())
+            return false;
+    }
+    return true;
+}
+
 FloatSize MediaStreamPrivate::intrinsicSize() const
 {
     FloatSize size;
index b982bb4..6bae73c 100644 (file)
@@ -91,8 +91,9 @@ public:
     RefPtr<Image> currentFrameImage();
     void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&);
 
-    bool hasVideo();
-    bool hasAudio();
+    bool hasVideo() const;
+    bool hasAudio() const;
+    bool muted() const;
 
     FloatSize intrinsicSize() const;
 
index 469c250..c0612c5 100644 (file)
@@ -132,6 +132,7 @@ AVMediaCaptureSource::AVMediaCaptureSource(AVCaptureDeviceType* device, const At
 {
     setName(device.localizedName);
     setPersistentID(device.uniqueID);
+    setMuted(true);
 }
 
 AVMediaCaptureSource::~AVMediaCaptureSource()
@@ -225,6 +226,7 @@ void AVMediaCaptureSource::captureSessionIsRunningDidChange(bool state)
             return;
 
         m_isRunning = state;
+        setMuted(!m_isRunning);
     });
 }
 
index 4b78cf8..430d63f 100644 (file)
@@ -1,3 +1,22 @@
+2016-03-10  Eric Carlson  <eric.carlson@apple.com>
+
+        [MediaStream] push media stream state to the UI process
+        https://bugs.webkit.org/show_bug.cgi?id=155281
+
+        Reviewed by Darin Adler.
+
+        * UIProcess/API/C/WKPage.cpp:
+        (WKPageGetMediaState): New.
+        * UIProcess/API/C/WKPagePrivate.h:
+
+        * UIProcess/WebPageProxy.cpp:
+        (WebKit::WebPageProxy::didCommitLoadForFrame): Clear m_mediaState.
+        (WebKit::WebPageProxy::isPlayingMediaDidChange): Call m_pageClient.isPlayingMediaDidChange when
+          audio or video state changes, call m_uiClient->isPlayingAudioDidChange when audio, vidoe,
+          or media stream state changes.
+        * UIProcess/WebPageProxy.h:
+        (WebKit::WebPageProxy::mediaStateFlags): New.
+
 2016-03-09  Carlos Garcia Campos  <cgarcia@igalia.com>
 
         [GTK] Artifacts when using web view background color
index c4042ba..18f47b2 100644 (file)
@@ -2697,6 +2697,21 @@ bool WKPageIsPlayingAudio(WKPageRef page)
     return toImpl(page)->isPlayingAudio();
 }
 
+WKMediaState WKPageGetMediaState(WKPageRef page)
+{
+    WebCore::MediaProducer::MediaStateFlags coreState = toImpl(page)->mediaStateFlags();
+    WKMediaState state = kWKMediaIsNotPlaying;
+
+    if (coreState & WebCore::MediaProducer::IsPlayingAudio)
+        state |= kWKMediaIsPlayingAudio;
+    if (coreState & WebCore::MediaProducer::IsPlayingVideo)
+        state |= kWKMediaIsPlayingVideo;
+    if (coreState & WebCore::MediaProducer::HasActiveMediaCaptureDevice)
+        state |= kWKMediaHasActiveCaptureDevice;
+
+    return state;
+}
+
 void WKPageClearWheelEventTestTrigger(WKPageRef pageRef)
 {
     toImpl(pageRef)->clearWheelEventTestTrigger();
index 43af7f7..5886ba2 100644 (file)
@@ -118,6 +118,16 @@ WK_EXPORT bool WKPageIsPlayingAudio(WKPageRef page);
 WK_EXPORT void WKPageSetMuted(WKPageRef page, bool muted);
 
 enum {
+    kWKMediaIsNotPlaying = 0,
+    kWKMediaIsPlayingAudio = 1 << 0,
+    kWKMediaIsPlayingVideo = 1 << 1,
+    kWKMediaHasActiveCaptureDevice = 1 << 2,
+};
+typedef uint32_t WKMediaState;
+
+WK_EXPORT WKMediaState WKPageGetMediaState(WKPageRef page);
+
+enum {
     kWKMediaEventTypePlayPause,
     kWKMediaEventTypeTrackNext,
     kWKMediaEventTypeTrackPrevious
index 624412b..42efb87 100644 (file)
@@ -3088,6 +3088,7 @@ void WebPageProxy::didCommitLoadForFrame(uint64_t frameID, uint64_t navigationID
     }
 
     m_pageLoadState.commitChanges();
+    m_mediaState = MediaProducer::IsNotPlaying;
     if (m_navigationClient) {
         if (frame->isMainFrame())
             m_navigationClient->didCommitNavigation(*this, navigation.get(), m_process->transformHandlesToObjects(userData.object()).get());
@@ -6000,16 +6001,18 @@ void WebPageProxy::isPlayingMediaDidChange(MediaProducer::MediaStateFlags state,
     if (state == m_mediaState)
         return;
 
+    MediaProducer::MediaStateFlags playingMediaMask = MediaProducer::IsPlayingAudio | MediaProducer::IsPlayingVideo;
     MediaProducer::MediaStateFlags oldState = m_mediaState;
     m_mediaState = state;
 
-    if ((oldState & MediaProducer::IsPlayingAudio) == (m_mediaState & MediaProducer::IsPlayingAudio))
-        return;
-
 #if PLATFORM(MAC)
-    m_pageClient.isPlayingMediaDidChange();
+    if ((oldState & playingMediaMask) != (m_mediaState & playingMediaMask))
+        m_pageClient.isPlayingMediaDidChange();
 #endif
-    m_uiClient->isPlayingAudioDidChange(*this);
+
+    playingMediaMask |= MediaProducer::HasActiveMediaCaptureDevice;
+    if ((oldState & playingMediaMask) != (m_mediaState & playingMediaMask))
+        m_uiClient->isPlayingAudioDidChange(*this);
 }
 
 bool WebPageProxy::isPlayingVideoWithAudio() const
index e8954e2..7c09d7f 100644 (file)
@@ -1019,6 +1019,7 @@ public:
 
     bool isPlayingAudio() const { return !!(m_mediaState & WebCore::MediaProducer::IsPlayingAudio); }
     void isPlayingMediaDidChange(WebCore::MediaProducer::MediaStateFlags, uint64_t);
+    WebCore::MediaProducer::MediaStateFlags mediaStateFlags() const { return m_mediaState; }
 
     bool isPlayingVideoWithAudio() const;
 
index 0fd67ef..fdf7e9a 100644 (file)
     (global-name "com.apple.coremedia.endpointstream.xpc")
     (global-name "com.apple.coremedia.endpointplaybacksession.xpc")
     (global-name "com.apple.coremedia.endpointpicker.xpc"))
+
+
+
+
+
+;; Camera access
+(allow device-camera)
+(allow file-read*
+    (literal "/Library/Preferences/com.apple.coremedia")
+    (home-literal "/Library/Preferences/com.apple.coremedia"))
+(allow mach-lookup (extension "com.apple.app-sandbox.mach"))
+(allow mach-lookup
+    (global-name "com.apple.cmio.AppleCameraAssistant")
+    ;; Apple DAL assistants
+    (global-name "com.apple.cmio.VDCAssistant")
+    (global-name "com.apple.cmio.AVCAssistant")
+    (global-name "com.apple.cmio.IIDCVideoAssistant")
+    ;; QuickTimeIIDCDigitizer assistant
+    (global-name "com.apple.IIDCAssistant"))
+(allow iokit-open
+    ;; QuickTimeUSBVDCDigitizer
+    (iokit-user-client-class "IOUSBDeviceUserClientV2")
+    (iokit-user-client-class "IOUSBInterfaceUserClientV2"))
+
+;; Microphone
+(allow device-microphone)